From 525159f943f8195ff25068719c3193e6628efc11 Mon Sep 17 00:00:00 2001 From: Bhupendra Raut Date: Sun, 20 Sep 2026 15:20:24 -0500 Subject: [PATCH 01/12] ADD:seed carry; FIX:track-breaks; ADD:decisions.db --- .../schemas/collection_decisions.sql | 153 +++++++ src/adapt/configuration/schemas/internal.py | 4 + src/adapt/configuration/schemas/param.py | 26 +- src/adapt/configuration/schemas/user.py | 3 + src/adapt/contracts/__init__.py | 33 +- src/adapt/contracts/decisions.py | 216 ++++++++++ src/adapt/contracts/history.py | 13 + src/adapt/contracts/persistence.py | 10 +- src/adapt/contracts/segmentation.py | 15 + src/adapt/execution/nodes/detection.py | 117 ++++- src/adapt/execution/nodes/tracking.py | 10 +- src/adapt/modules/detection/config.py | 3 + src/adapt/modules/detection/module.py | 403 +++++++++++++++--- src/adapt/modules/tracking/README.md | 14 +- src/adapt/modules/tracking/config.py | 1 + src/adapt/modules/tracking/decision_log.py | 190 +++++++++ src/adapt/modules/tracking/models.py | 11 +- src/adapt/modules/tracking/module.py | 184 ++++++-- src/adapt/modules/tracking/motion.py | 25 +- src/adapt/modules/tracking/projection.py | 30 -- src/adapt/persistence/decision_store.py | 82 ++++ src/adapt/persistence/output_router.py | 16 + src/adapt/persistence/store.py | 9 +- src/adapt/runtime/processor.py | 41 +- tests/configuration/test_seed_carry_config.py | 57 +++ tests/contracts/test_decisions_contract.py | 61 +++ tests/contracts/test_persistence_specs.py | 1 + tests/contracts/test_seed_carry_contract.py | 36 ++ .../nodes/test_detection_node_seed_carry.py | 134 ++++++ tests/execution/nodes/test_tracking_node.py | 7 + .../detection/test_detection_config.py | 3 + .../detection/test_segmentation_decisions.py | 115 +++++ .../detection/test_segmenter_carried_seeds.py | 142 ++++++ .../test_segmenter_pyart_conv_strat.py | 3 + .../detection/test_segmenter_size_ranking.py | 45 ++ tests/modules/tracking/test_decision_log.py | 154 +++++++ tests/modules/tracking/test_motion.py | 38 ++ .../modules/tracking/test_tracker_quality.py | 106 ++++- .../test_collection_provisioning.py | 29 +- tests/persistence/test_decision_store.py | 177 ++++++++ tests/runtime/test_processor_prior_scan.py | 98 +++++ tests/stories/test_seed_carry_story.py | 81 ++++ tests/test_architecture.py | 1 + tests/unit/test_base_module_persistence.py | 17 +- 44 files changed, 2711 insertions(+), 203 deletions(-) create mode 100644 src/adapt/configuration/schemas/collection_decisions.sql create mode 100644 src/adapt/contracts/decisions.py create mode 100644 src/adapt/modules/tracking/decision_log.py delete mode 100644 src/adapt/modules/tracking/projection.py create mode 100644 src/adapt/persistence/decision_store.py create mode 100644 tests/configuration/test_seed_carry_config.py create mode 100644 tests/contracts/test_decisions_contract.py create mode 100644 tests/contracts/test_seed_carry_contract.py create mode 100644 tests/execution/nodes/test_detection_node_seed_carry.py create mode 100644 tests/modules/detection/test_segmentation_decisions.py create mode 100644 tests/modules/detection/test_segmenter_carried_seeds.py create mode 100644 tests/modules/detection/test_segmenter_size_ranking.py create mode 100644 tests/modules/tracking/test_decision_log.py create mode 100644 tests/modules/tracking/test_motion.py create mode 100644 tests/persistence/test_decision_store.py create mode 100644 tests/runtime/test_processor_prior_scan.py create mode 100644 tests/stories/test_seed_carry_story.py diff --git a/src/adapt/configuration/schemas/collection_decisions.sql b/src/adapt/configuration/schemas/collection_decisions.sql new file mode 100644 index 0000000..de0736f --- /dev/null +++ b/src/adapt/configuration/schemas/collection_decisions.sql @@ -0,0 +1,153 @@ +-- Analysis-only decision log: collections//decisions.db +-- +-- Written by the output router from the tracking and detection modules' +-- decision records (adapt.contracts.decisions); never read by the pipeline or +-- the public API. Rows mirror the frozen record fields plus run/scan stamps. +-- Config thresholds are not repeated here: join run_id to the run registry. + +PRAGMA user_version = 1; + +CREATE TABLE IF NOT EXISTS tracking_frames ( + run_id TEXT NOT NULL, + scan_id TEXT NOT NULL, + scan_time TEXT NOT NULL, + dt_s REAL, + reset_code TEXT, + n_prev INTEGER NOT NULL, + n_curr INTEGER NOT NULL, + n_pairs INTEGER NOT NULL, + n_pass_overlap INTEGER NOT NULL, + n_pass_kinematic INTEGER NOT NULL, + n_propagated INTEGER NOT NULL, + n_hungarian INTEGER NOT NULL, + n_split INTEGER NOT NULL, + n_merge INTEGER NOT NULL, + n_initiation INTEGER NOT NULL, + n_termination INTEGER NOT NULL, + PRIMARY KEY (run_id, scan_id) +); + +CREATE TABLE IF NOT EXISTS tracking_candidates ( + run_id TEXT NOT NULL, + scan_id TEXT NOT NULL, + scan_time TEXT NOT NULL, + prev_cell_uid TEXT NOT NULL, + prev_cell_label INTEGER NOT NULL, + curr_cell_label INTEGER NOT NULL, + track_steps_before INTEGER NOT NULL, + hull_area_px INTEGER NOT NULL, + hull_centroid_x REAL NOT NULL, + hull_centroid_y REAL NOT NULL, + curr_area_px INTEGER NOT NULL, + curr_centroid_x REAL NOT NULL, + curr_centroid_y REAL NOT NULL, + intersection_px INTEGER NOT NULL, + opc REAL NOT NULL, + ocp REAL NOT NULL, + overlap_passed INTEGER NOT NULL, + speed_ms REAL, + previous_speed_ms REAL, + accel_cap_ms REAL, + kinematic_code TEXT, + displacement_m REAL, + length_scale_m REAL, + heading_change_deg REAL, + cost REAL, + component_n_prev INTEGER, + component_n_curr INTEGER, + cost_rank INTEGER, + match_method TEXT, + last_stage TEXT NOT NULL, + outcome TEXT NOT NULL, + PRIMARY KEY (run_id, scan_id, prev_cell_uid, curr_cell_label) +); +CREATE INDEX IF NOT EXISTS idx_tracking_candidates_prev + ON tracking_candidates (run_id, prev_cell_uid); + +CREATE TABLE IF NOT EXISTS tracking_unmatched ( + run_id TEXT NOT NULL, + scan_id TEXT NOT NULL, + scan_time TEXT NOT NULL, + side TEXT NOT NULL, + cell_uid TEXT NOT NULL, + cell_label INTEGER NOT NULL, + n_candidates INTEGER NOT NULL, + best_candidate_label INTEGER, + best_stage TEXT, + reason TEXT NOT NULL, + PRIMARY KEY (run_id, scan_id, side, cell_uid) +); +CREATE INDEX IF NOT EXISTS idx_tracking_unmatched_uid + ON tracking_unmatched (run_id, cell_uid); + +CREATE TABLE IF NOT EXISTS tracking_split_merge_tests ( + run_id TEXT NOT NULL, + scan_id TEXT NOT NULL, + scan_time TEXT NOT NULL, + kind TEXT NOT NULL, + continuing_cell_uid TEXT NOT NULL, + tested_cell_label INTEGER NOT NULL, + overlap_fraction REAL NOT NULL, + threshold REAL NOT NULL, + passed INTEGER NOT NULL, + PRIMARY KEY (run_id, scan_id, kind, continuing_cell_uid, tested_cell_label) +); + +CREATE TABLE IF NOT EXISTS segmentation_frames ( + run_id TEXT NOT NULL, + scan_id TEXT NOT NULL, + scan_time TEXT NOT NULL, + mask_px INTEGER NOT NULL, + closed_mask_px INTEGER NOT NULL, + n_components INTEGER NOT NULL, + n_independent_seeds INTEGER NOT NULL, + n_carried_claims INTEGER NOT NULL, + n_carried_admitted INTEGER NOT NULL, + n_basins INTEGER NOT NULL, + n_dropped_small INTEGER NOT NULL, + n_dropped_large INTEGER NOT NULL, + n_cells INTEGER NOT NULL, + PRIMARY KEY (run_id, scan_id) +); + +CREATE TABLE IF NOT EXISTS segmentation_seeds ( + run_id TEXT NOT NULL, + scan_id TEXT NOT NULL, + scan_time TEXT NOT NULL, + seq INTEGER NOT NULL, + marker_id INTEGER, + origin TEXT NOT NULL, + row INTEGER NOT NULL, + col INTEGER NOT NULL, + peak_value REAL, + plateau_px INTEGER, + component_id INTEGER, + prior_label INTEGER, + prior_age INTEGER, + footprint_px INTEGER, + clearance_px REAL, + footprint_holds_seed INTEGER, + component_claims INTEGER, + component_seeds INTEGER, + component_deficit INTEGER, + decision TEXT NOT NULL, + basin_px INTEGER, + final_label INTEGER, + PRIMARY KEY (run_id, scan_id, seq) +); + +CREATE TABLE IF NOT EXISTS segmentation_components ( + run_id TEXT NOT NULL, + scan_id TEXT NOT NULL, + scan_time TEXT NOT NULL, + component_id INTEGER NOT NULL, + area_px INTEGER NOT NULL, + field_max REAL NOT NULL, + field_min REAL NOT NULL, + n_seeds INTEGER NOT NULL, + n_claims INTEGER NOT NULL, + deficit INTEGER NOT NULL, + n_admitted INTEGER NOT NULL, + n_final_labels INTEGER NOT NULL, + PRIMARY KEY (run_id, scan_id, component_id) +); diff --git a/src/adapt/configuration/schemas/internal.py b/src/adapt/configuration/schemas/internal.py index 70d4af9..ccb8554 100644 --- a/src/adapt/configuration/schemas/internal.py +++ b/src/adapt/configuration/schemas/internal.py @@ -76,6 +76,9 @@ class InternalSegmenterConfig(AdaptBaseModel): closing_kernel: tuple[int, int] filter_by_size: bool h_maxima: float + seed_carry: bool + seed_carry_max_frames: int + seed_carry_min_separation: int threshold_params: ThresholdParams conv_strat_raut_params: ConvStratRautParams conv_strat_yuter_params: ConvStratYuterParams @@ -170,6 +173,7 @@ class InternalCellUidConfig(AdaptBaseModel): max_tracking_gap_minutes: float = Field(default=20.0, gt=0.0) max_speed_ms: float = Field(default=40.0, gt=0.0) max_speed_multiplier: float = Field(default=3.0, gt=0.0) + acceleration_floor_ms: float = Field(default=10.0, ge=0.0) heading_change_penalty_weight: float = Field(default=0.0, ge=0.0) projected_hull_buffer_km: float = Field(default=1.0, gt=0.0) minimum_candidate_overlap: float = Field(default=0.20, ge=0.0, le=1.0) diff --git a/src/adapt/configuration/schemas/param.py b/src/adapt/configuration/schemas/param.py index 99cbdc9..07faa83 100644 --- a/src/adapt/configuration/schemas/param.py +++ b/src/adapt/configuration/schemas/param.py @@ -105,6 +105,24 @@ class SegmenterConfig(AdaptBaseModel): closing_kernel: tuple[int, int] = (1, 1) filter_by_size: bool = True h_maxima: float = Field(5.0, gt=0, description="h-maxima height for cell seeding (dBZ)") + # Previous-frame seeding: the projection module's one-step-ahead labels + # become extra watershed markers (union with h-maxima seeds, never a + # replacement). Off by default; off is byte-identical to no seeding. + seed_carry: bool = Field( + False, description="Seed the watershed with the previous frame's advected cell cores" + ) + seed_carry_max_frames: int = Field( + 2, + ge=1, + description="Drop a carried core after this many consecutive frames without " + "independent detection", + ) + seed_carry_min_separation: int = Field( + 3, + ge=1, + description="Minimum pixel distance from any h-maxima seed for a carried core " + "to be admitted", + ) # Per-method scientific parameters (one block per method; defaults = algorithm defaults) threshold_params: ThresholdParams = Field(default_factory=ThresholdParams) # type: ignore[arg-type] conv_strat_raut_params: ConvStratRautParams = Field(default_factory=ConvStratRautParams) # type: ignore[arg-type] @@ -265,7 +283,13 @@ class CellUidConfig(AdaptBaseModel): 3.0, gt=0.0, description="Hard acceleration cap: reject if candidate speed exceeds this times the " - "track's previous speed", + "track's mean step speed", + ) + acceleration_floor_ms: float = Field( + 10.0, + ge=0.0, + description="The acceleration cap never falls below this speed (m/s): a slow prior " + "step from centroid jitter must not veto ordinary storm motion", ) heading_change_penalty_weight: float = Field( 0.0, diff --git a/src/adapt/configuration/schemas/user.py b/src/adapt/configuration/schemas/user.py index 489d6b5..c1b7d49 100644 --- a/src/adapt/configuration/schemas/user.py +++ b/src/adapt/configuration/schemas/user.py @@ -52,6 +52,9 @@ class UserSegmenterConfig(_UserSection): closing_kernel: tuple[int, int] | None = None filter_by_size: bool | None = None h_maxima: float | None = None + seed_carry: bool | None = None + seed_carry_max_frames: int | None = None + seed_carry_min_separation: int | None = None @field_validator("method", mode="before") @classmethod diff --git a/src/adapt/contracts/__init__.py b/src/adapt/contracts/__init__.py index ed565e6..980c9b5 100644 --- a/src/adapt/contracts/__init__.py +++ b/src/adapt/contracts/__init__.py @@ -25,9 +25,23 @@ check_cell_volume_stats, ) from adapt.contracts.columns import CELL_LABELS_VAR, stat_column +from adapt.contracts.decisions import ( + SegmentationComponent, + SegmentationDecisions, + SegmentationFrame, + SegmentationSeed, + TrackingCandidate, + TrackingDecisions, + TrackingFrame, + TrackingSplitMergeTest, + TrackingUnmatched, + check_segmentation_decisions, + check_tracking_decisions, +) from adapt.contracts.grid import assert_gridded, check_grid_ds_2d -from adapt.contracts.history import check_scan_history +from adapt.contracts.history import check_prior_scan, check_scan_history from adapt.contracts.persistence import ( + DecisionTableWrite, NetcdfArtifact, PersistenceMeta, PersistenceSpec, @@ -37,7 +51,7 @@ ) from adapt.contracts.pipeline import ContractViolation, require from adapt.contracts.projection import assert_projected, check_projected_ds -from adapt.contracts.segmentation import assert_segmented, check_segmented_ds +from adapt.contracts.segmentation import assert_segmented, check_seed_carry, check_segmented_ds from adapt.contracts.time import assert_time_normalized, check_time_normalized from adapt.contracts.tracking import ( assert_cell_events, @@ -63,9 +77,20 @@ "ProductTableWrite", "NetcdfArtifact", "TrackTablesWrite", + "DecisionTableWrite", "PersistenceSpec", "PersistenceMeta", "ScanRecord", + # decision records — analysis-only, written to decisions.db + "TrackingFrame", + "TrackingCandidate", + "TrackingUnmatched", + "TrackingSplitMergeTest", + "TrackingDecisions", + "SegmentationFrame", + "SegmentationSeed", + "SegmentationComponent", + "SegmentationDecisions", "assert_gridded", "assert_segmented", "assert_projected", @@ -80,7 +105,9 @@ # bound checks — register these in input_contracts / output_contracts "check_grid_ds_2d", "check_scan_history", + "check_prior_scan", "check_segmented_ds", + "check_seed_carry", "check_projected_ds", "check_cell_stats", "check_cell_adjacency", @@ -90,4 +117,6 @@ "check_time_normalized", "check_xlma_stat_minutes", "check_xlma_stat_scan", + "check_tracking_decisions", + "check_segmentation_decisions", ] diff --git a/src/adapt/contracts/decisions.py b/src/adapt/contracts/decisions.py new file mode 100644 index 0000000..63e74aa --- /dev/null +++ b/src/adapt/contracts/decisions.py @@ -0,0 +1,216 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Decision records: what the segmenter and the tracker considered, and why. + +These are analysis-only records written to a separate database +(``decisions.db``). The pipeline never reads them and the public API does not +expose them; they exist so a run can be explained after the fact and gate +thresholds re-tuned offline from stored numbers. Identity (run, scan, time) +is stamped by persistence, never carried here. +""" + +from dataclasses import dataclass + +from adapt.contracts.pipeline import require + +# ── Tracking ───────────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class TrackingFrame: + """One scan pair: how the frame was classified and what each stage yielded.""" + + dt_s: float | None + reset_code: str | None # FIRST_SCAN | NON_MONOTONIC_TIME | TRACK_GAP_EXCEEDED | NO_PROJECTIONS + n_prev: int + n_curr: int + n_pairs: int + n_pass_overlap: int + n_pass_kinematic: int + n_propagated: int + n_hungarian: int + n_split: int + n_merge: int + n_initiation: int + n_termination: int + + +@dataclass(frozen=True) +class TrackingCandidate: + """One (previous cell, current cell) pair that candidate generation produced. + + ``last_stage`` is the furthest stage the pair reached (OVERLAP, KINEMATIC, + ASSIGNMENT); ``outcome`` is CONTINUE, REJECTED_OVERLAP, REJECTED_KINEMATIC + or LOST_ASSIGNMENT. Fields of stages the pair never reached are None. + """ + + prev_cell_uid: str + prev_cell_label: int + curr_cell_label: int + track_steps_before: int + hull_area_px: int + hull_centroid_x: float + hull_centroid_y: float + curr_area_px: int + curr_centroid_x: float + curr_centroid_y: float + intersection_px: int + opc: float + ocp: float + overlap_passed: bool + speed_ms: float | None + previous_speed_ms: float | None + accel_cap_ms: float | None + kinematic_code: str | None + displacement_m: float | None + length_scale_m: float | None + heading_change_deg: float | None + cost: float | None + component_n_prev: int | None + component_n_curr: int | None + cost_rank: int | None + match_method: str | None + last_stage: str + outcome: str + + +@dataclass(frozen=True) +class TrackingUnmatched: + """A previous cell that did not continue, or a current cell that was born.""" + + side: str # prev | curr + cell_uid: str + cell_label: int + n_candidates: int + best_candidate_label: int | None + best_stage: str | None + reason: str # NO_CANDIDATE | ALL_REJECTED_OVERLAP | ALL_REJECTED_KINEMATIC | + # LOST_ASSIGNMENT | SPLIT_CHILD | MERGE_SOURCE | RESET | FIRST_SCAN + + +@dataclass(frozen=True) +class TrackingSplitMergeTest: + """Every hull overlap tested against ``split_overlap`` — pass or fail.""" + + kind: str # SPLIT | MERGE + continuing_cell_uid: str + tested_cell_label: int + overlap_fraction: float + threshold: float + passed: bool + + +@dataclass(frozen=True) +class TrackingDecisions: + frame: TrackingFrame + candidates: tuple[TrackingCandidate, ...] + unmatched: tuple[TrackingUnmatched, ...] + split_merge_tests: tuple[TrackingSplitMergeTest, ...] + + +# ── Segmentation ───────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class SegmentationFrame: + """One scan: mask, seeds, carried claims and what survived the size filter.""" + + mask_px: int + closed_mask_px: int + n_components: int + n_independent_seeds: int + n_carried_claims: int + n_carried_admitted: int + n_basins: int + n_dropped_small: int + n_dropped_large: int + n_cells: int + + +@dataclass(frozen=True) +class SegmentationSeed: + """One watershed marker — independent or carried, admitted or not. + + ``decision`` is SEEDED for markers used, else why a carried claim was not: + SURVIVOR (its footprint already holds a seed), EXPIRED (past the carry + bound), OUTSIDE_MASK, NO_DEFICIT, TOO_CLOSE_TO_SEED, TOO_CLOSE_TO_MARKER. + ``final_label`` is 0 when the size filter dropped the basin. + """ + + marker_id: int | None + origin: str # HMAXIMA | CARRIED + row: int + col: int + peak_value: float | None + plateau_px: int | None + component_id: int | None + prior_label: int | None + prior_age: int | None + footprint_px: int | None + clearance_px: float | None + footprint_holds_seed: bool | None + component_claims: int | None + component_seeds: int | None + component_deficit: int | None + decision: str + basin_px: int | None + final_label: int | None + + +@dataclass(frozen=True) +class SegmentationComponent: + """One 8-connected component of the closed convective mask.""" + + component_id: int + area_px: int + field_max: float + field_min: float + n_seeds: int + n_claims: int + deficit: int + n_admitted: int + n_final_labels: int + + +@dataclass(frozen=True) +class SegmentationDecisions: + frame: SegmentationFrame + seeds: tuple[SegmentationSeed, ...] + components: tuple[SegmentationComponent, ...] + + +# ── Bound checks ───────────────────────────────────────────────────────────── + + +def check_tracking_decisions(decisions: TrackingDecisions) -> None: + require( + isinstance(decisions, TrackingDecisions), "tracking_decisions must be TrackingDecisions" + ) + frame = decisions.frame + require( + frame.n_termination <= frame.n_prev, + "tracking_decisions: more terminations than previous cells", + ) + require( + len(decisions.candidates) == frame.n_pairs, + f"tracking_decisions: {len(decisions.candidates)} candidate rows for " + f"{frame.n_pairs} generated pairs", + ) + + +def check_segmentation_decisions(decisions: SegmentationDecisions) -> None: + require( + isinstance(decisions, SegmentationDecisions), + "detection_decisions must be SegmentationDecisions", + ) + frame = decisions.frame + require( + len(decisions.components) == frame.n_components, + f"detection_decisions: {len(decisions.components)} component rows for " + f"{frame.n_components} components", + ) + require( + frame.n_cells <= frame.n_basins, + "detection_decisions: more cells than basins", + ) diff --git a/src/adapt/contracts/history.py b/src/adapt/contracts/history.py index f59b433..1cfd96c 100644 --- a/src/adapt/contracts/history.py +++ b/src/adapt/contracts/history.py @@ -28,3 +28,16 @@ def check_scan_history(history: list) -> None: require(isinstance(ctx, dict), f"scan_history[{i}] must be a dict") require("segmented_ds" in ctx, f"scan_history[{i}] missing 'segmented_ds'") require("scan_time" in ctx, f"scan_history[{i}] missing 'scan_time'") + + +def check_prior_scan(prior: dict | None) -> None: + """Validate the prior_scan context key. + + ``None`` when no usable predecessor exists (first scan, or the time gap to + the previous scan is too large); otherwise the previous completed scan's + context dict, which must carry ``scan_time``. + """ + if prior is None: + return + require(isinstance(prior, dict), "prior_scan must be a dict or None") + require("scan_time" in prior, "prior_scan missing 'scan_time'") diff --git a/src/adapt/contracts/persistence.py b/src/adapt/contracts/persistence.py index 44ac07b..8fb7e6f 100644 --- a/src/adapt/contracts/persistence.py +++ b/src/adapt/contracts/persistence.py @@ -66,7 +66,15 @@ class ProductTableWrite: index_columns: tuple[str, ...] = () -PersistenceSpec = NetcdfArtifact | TrackTablesWrite | ProductTableWrite +@dataclass(frozen=True) +class DecisionTableWrite: + """Write a decision bundle (context key) to the collection's analysis-only + decisions.db. Never a scan product: no catalog link, no API exposure.""" + + key: str + + +PersistenceSpec = NetcdfArtifact | TrackTablesWrite | ProductTableWrite | DecisionTableWrite @dataclass(frozen=True) diff --git a/src/adapt/contracts/segmentation.py b/src/adapt/contracts/segmentation.py index 961f917..0df17fb 100644 --- a/src/adapt/contracts/segmentation.py +++ b/src/adapt/contracts/segmentation.py @@ -53,3 +53,18 @@ def assert_segmented(ds: xr.Dataset, labels_name: str) -> None: def check_segmented_ds(ds: xr.Dataset) -> None: """Bound contract for the standard segmented dataset (cell_labels variable name fixed).""" assert_segmented(ds, "cell_labels") + + +def check_seed_carry(ages: tuple) -> None: + """Carry age per cell label: a tuple of ints >= 0, index i for label i + 1. + + 0 means the cell was seeded independently this scan; k means it has been + carried k consecutive scans from the previous frame's cores. + """ + require(isinstance(ages, tuple), f"seed_carry must be a tuple, got {type(ages).__name__}") + for i, age in enumerate(ages): + require( + isinstance(age, int) and not isinstance(age, bool), + f"seed_carry[{i}] must be an int, got {type(age).__name__}", + ) + require(age >= 0, f"seed_carry[{i}] must be >= 0, got {age}") diff --git a/src/adapt/execution/nodes/detection.py b/src/adapt/execution/nodes/detection.py index 7fd5d10..d642e37 100644 --- a/src/adapt/execution/nodes/detection.py +++ b/src/adapt/execution/nodes/detection.py @@ -1,11 +1,39 @@ # Copyright © 2026, UChicago Argonne, LLC # See LICENSE for terms and disclaimer. -from adapt.contracts import CELL_LABELS_VAR, check_grid_ds_2d, check_segmented_ds +import logging + +import numpy as np + +from adapt.contracts import ( + CELL_LABELS_VAR, + DecisionTableWrite, + check_grid_ds_2d, + check_prior_scan, + check_seed_carry, + check_segmentation_decisions, + check_segmented_ds, + require, +) from adapt.execution.module_registry import registry from adapt.modules.base import BaseModule from adapt.modules.detection.config import DetectionConfig -from adapt.modules.detection.module import RadarCellSegmenter +from adapt.modules.detection.module import CarriedLabels, RadarCellSegmenter + +logger = logging.getLogger(__name__) + + +def _projection_enabled(cfg) -> bool: + """Whether the run enables ``projection``, by ``resolve_enabled_modules``' precedence. + + Mirrored here rather than imported: the resolver needs the module + registry, which this file is being registered into. + """ + if cfg.only_modules: + return "projection" in cfg.only_modules + if cfg.exclude_modules and "projection" in cfg.exclude_modules: + return False + return cfg.modules is None or "projection" in cfg.modules class DetectModule(BaseModule): @@ -21,6 +49,12 @@ class DetectModule(BaseModule): grid_ds : xr.Dataset Full 3D Cartesian dataset (also an ingest output). The pyart convective/stratiform methods classify it; ``threshold`` ignores it. + prior_scan : dict | None + The previous completed scan's context, seeded by the processor (None + on the first scan or after a too-large time gap). With + ``segmenter.seed_carry`` on, its ``projected_ds`` (the previous + frame's labels advected one flow step forward) and ``seed_carry`` + are handed to the segmenter as carried labels. config : InternalConfig Runtime configuration. @@ -30,21 +64,40 @@ class DetectModule(BaseModule): 2D dataset with cell_labels variable added. num_cells : int Number of detected cells. + seed_carry : tuple[int, ...] + Carry age per cell label (index i -> label i + 1): 0 when seeded + independently this scan, k when carried k consecutive scans. Empty + with the flag off. """ name = "detection" summary = "segment cells from the grid" required_history = 1 pipeline_phase = 0 - inputs = ["grid_ds_2d", "grid_ds", "detection_config"] - outputs = ["segmented_ds", "num_cells"] - input_contracts = {"grid_ds_2d": check_grid_ds_2d} - output_contracts = {"segmented_ds": check_segmented_ds} + inputs = ["grid_ds_2d", "grid_ds", "prior_scan", "detection_config"] + # detection_decisions: every seed and carried claim this scan and what + # became of it — analysis-only, written to decisions.db. + outputs = ["segmented_ds", "num_cells", "seed_carry", "detection_decisions"] + input_contracts = {"grid_ds_2d": check_grid_ds_2d, "prior_scan": check_prior_scan} + output_contracts = { + "segmented_ds": check_segmented_ds, + "seed_carry": check_seed_carry, + "detection_decisions": check_segmentation_decisions, + } config_class = DetectionConfig + persistence = (DecisionTableWrite(key="detection_decisions"),) @classmethod def build_config(cls, cfg) -> DetectionConfig: seg = cfg.segmenter + # Checked at startup: at run time "prior scan has no projected_ds" is + # also the legitimate state of the second scan, so it cannot be told + # apart from a disabled projection module there. + if seg.seed_carry and not _projection_enabled(cfg): + raise ValueError( + "segmenter.seed_carry needs the projection module (its advected labels " + "are the seeds carried forward) — enable projection or turn seed_carry off" + ) method_params = getattr(seg, f"{seg.method}_params").model_dump() return DetectionConfig( method=seg.method, @@ -54,6 +107,9 @@ def build_config(cls, cfg) -> DetectionConfig: min_cellsize_gridpoint=seg.min_cellsize_gridpoint, max_cellsize_gridpoint=seg.max_cellsize_gridpoint, h_maxima=seg.h_maxima, + seed_carry=seg.seed_carry, + seed_carry_max_frames=seg.seed_carry_max_frames, + seed_carry_min_separation=seg.seed_carry_min_separation, reflectivity_var=cfg.global_.tracking_field, labels_var=CELL_LABELS_VAR, z_level=cfg.global_.z_level, @@ -69,10 +125,53 @@ def run(self, context: dict) -> dict: if self._segmenter is None: self._segmenter = RadarCellSegmenter(config) - segmented = self._segmenter.segment(ds_2d, context["grid_ds"]) - num_cells = int(segmented[config.labels_var].max().item()) + carried = self._carried(context["prior_scan"], config) + segmented = self._segmenter.segment(ds_2d, context["grid_ds"], carried) + labels = segmented[config.labels_var] + num_cells = int(labels.max().item()) + seed_carry = ( + tuple(int(age) for age in np.atleast_1d(labels.attrs["carry_age"])) + if config.seed_carry and num_cells + else () + ) + + return { + "segmented_ds": segmented, + "num_cells": num_cells, + "seed_carry": seed_carry, + "detection_decisions": self._segmenter.decisions(), + } + + @staticmethod + def _carried(prior: dict | None, config: DetectionConfig) -> CarriedLabels | None: + """The previous scan's one-step-ahead projected labels with their carry ages. + + None with the flag off, without a prior scan, or when the prior has + no motion field yet (projection first runs at the second scan and is + skipped after a too-large gap): the carry is undefined without one. + """ + if not config.seed_carry or prior is None: + return None + if "projected_ds" not in prior: + logger.debug("seed_carry: prior scan has no projected_ds; nothing carried") + return None - return {"segmented_ds": segmented, "num_cells": num_cells} + ages = prior["seed_carry"] + require( + len(ages) == prior["num_cells"], + f"seed_carry: prior scan carries {len(ages)} ages for {prior['num_cells']} cells", + ) + projections = prior["projected_ds"]["cell_projections"] + require( + projections.sizes["frame_offset"] >= 2, + "seed_carry needs projector.max_projection_steps >= 1 (no one-step-ahead labels)", + ) + advected = np.asarray(projections.isel(frame_offset=1).values) + require( + int(advected.max()) <= len(ages), + f"seed_carry: advected label {int(advected.max())} exceeds prior cell count", + ) + return CarriedLabels(labels=advected, ages=tuple(ages)) registry.register(DetectModule) diff --git a/src/adapt/execution/nodes/tracking.py b/src/adapt/execution/nodes/tracking.py index 081debc..3dd22c3 100644 --- a/src/adapt/execution/nodes/tracking.py +++ b/src/adapt/execution/nodes/tracking.py @@ -3,11 +3,13 @@ from adapt.contracts import ( CELL_LABELS_VAR, + DecisionTableWrite, NetcdfArtifact, TrackTablesWrite, check_cell_events, check_projected_ds, check_tracked_cells, + check_tracking_decisions, ) from adapt.execution.module_registry import registry from adapt.modules.base import BaseModule @@ -50,12 +52,15 @@ class TrackingModule(BaseModule): required_history = 2 pipeline_phase = 0 inputs = ["projected_ds", "cell_stats", "tracking_config", "scan_time", "scan_id"] - outputs = ["tracked_cells", "cell_events", "analysis_ds"] + # tracking_decisions: every candidate considered this scan and why each + # cell continued or ended — analysis-only, written to decisions.db. + outputs = ["tracked_cells", "cell_events", "analysis_ds", "tracking_decisions"] input_contracts = {"projected_ds": check_projected_ds} output_contracts = { "tracked_cells": check_tracked_cells, "cell_events": check_cell_events, "analysis_ds": check_projected_ds, + "tracking_decisions": check_tracking_decisions, } config_class = TrackingConfig # DEBT: TrackTablesWrite encodes tracking science that lives in @@ -74,6 +79,7 @@ class TrackingModule(BaseModule): stats_key="cell_stats", adjacency_key="cell_adjacency", ), + DecisionTableWrite(key="tracking_decisions"), ) @classmethod @@ -87,6 +93,7 @@ def build_config(cls, cfg) -> TrackingConfig: max_tracking_gap_minutes=cfg.tracker.max_tracking_gap_minutes, max_speed_ms=cfg.tracker.max_speed_ms, max_speed_multiplier=cfg.tracker.max_speed_multiplier, + acceleration_floor_ms=cfg.tracker.acceleration_floor_ms, heading_change_penalty_weight=cfg.tracker.heading_change_penalty_weight, projected_hull_buffer_km=cfg.tracker.projected_hull_buffer_km, minimum_candidate_overlap=cfg.tracker.minimum_candidate_overlap, @@ -124,6 +131,7 @@ def run(self, context: dict) -> dict: "tracked_cells": tracked_cells, "cell_events": cell_events, "analysis_ds": analysis_ds, + "tracking_decisions": self._tracker.decisions(), } diff --git a/src/adapt/modules/detection/config.py b/src/adapt/modules/detection/config.py index e3a02a7..30969cf 100644 --- a/src/adapt/modules/detection/config.py +++ b/src/adapt/modules/detection/config.py @@ -27,6 +27,9 @@ class DetectionConfig(BaseModel): min_cellsize_gridpoint: int max_cellsize_gridpoint: int | None h_maxima: float + seed_carry: bool + seed_carry_max_frames: int + seed_carry_min_separation: int reflectivity_var: str labels_var: str z_level: float diff --git a/src/adapt/modules/detection/module.py b/src/adapt/modules/detection/module.py index c2d784c..e2390a8 100644 --- a/src/adapt/modules/detection/module.py +++ b/src/adapt/modules/detection/module.py @@ -27,49 +27,81 @@ """ import logging +from dataclasses import dataclass, fields import numpy as np import pyart import xarray as xr -from scipy.ndimage import label +from scipy.ndimage import center_of_mass, distance_transform_edt, label from skimage.morphology import h_maxima from skimage.segmentation import watershed -from adapt.contracts import ContractViolation +from adapt.contracts import ( + ContractViolation, + SegmentationComponent, + SegmentationDecisions, + SegmentationFrame, + SegmentationSeed, + require, +) -__all__ = ["RadarCellSegmenter"] +__all__ = ["CarriedLabels", "RadarCellSegmenter", "SeedCandidate"] logger = logging.getLogger(__name__) +# A watershed marker carried from the previous frame: (row, col, carry_age). +SeedCandidate = tuple[int, int, int] -def _label_maxtree(binary: np.ndarray, field: np.ndarray, h: float = 5.0) -> np.ndarray: - """Replace connected-component labeling: h-maxima seeding + watershed. +# Mask components are 8-connected, matching h_maxima's default footprint. +_COMPONENT_CONNECTIVITY = np.ones((3, 3), dtype=bool) - Identifies individual cells within a binary convection mask by seeding - each local intensity maximum (that rises at least `h` dBZ above its - surroundings) and growing watershed regions from those seeds. +_SEED_FIELDS = tuple(f.name for f in fields(SegmentationSeed)) - Parameters - ---------- - binary : np.ndarray (bool) - Closed binary convection mask (output of morphological closing). - field : np.ndarray (float) - Reflectivity values aligned with `binary`. - h : float - Minimum intensity rise above surroundings for a peak to seed a cell. - Returns - ------- - np.ndarray (int32) - 0 = background, 1..N = individual cell IDs. +def _seed_row(**values) -> dict: + """A SegmentationSeed record under construction: every field, unset ones None.""" + row = dict.fromkeys(_SEED_FIELDS) + row.update(values) + return row + + +def _independent_seed(marker: int, plateau: np.ndarray, fp: np.ndarray, components) -> dict: + """Decision row for an h-maxima marker: its peak pixel, plateau size and component.""" + rows, cols = np.nonzero(plateau) + k = int(np.argmax(fp[rows, cols])) + row, col = int(rows[k]), int(cols[k]) + return _seed_row( + marker_id=marker, + origin="HMAXIMA", + row=row, + col=col, + peak_value=float(fp[row, col]), + plateau_px=int(len(rows)), + component_id=int(components[row, col]), + decision="SEEDED", + ) + + +@dataclass(frozen=True) +class CarriedLabels: + """The previous frame's cell labels advected to this frame, with carry ages. + + ``labels`` is an int raster (y, x), 0 = none; ``ages[i]`` is the carry age + of label ``i + 1`` (0 = detected independently in the previous frame). """ - fp = np.where(binary, field, 0.0) - peaks = h_maxima(fp, h=h) - seeds, n_seeds = label(peaks) # label from scipy.ndimage - if n_seeds == 0: - return np.zeros(binary.shape, dtype=np.int32) - ws = watershed(-fp, seeds, mask=binary) - return np.where(binary, ws, 0).astype(np.int32) + + labels: np.ndarray + ages: tuple[int, ...] + + +def _footprint_seed(footprint: np.ndarray) -> tuple[int, int]: + """Centroid of a footprint, snapped to its nearest pixel when it falls outside.""" + row, col = (int(round(v)) for v in center_of_mass(footprint)) + if footprint[row, col]: + return row, col + rows, cols = np.nonzero(footprint) + nearest = int(np.argmin((rows - row) ** 2 + (cols - col) ** 2)) + return int(rows[nearest]), int(cols[nearest]) # --------------------------------------------------------------------------- @@ -162,7 +194,8 @@ class RadarCellSegmenter: reflectivity > threshold; the pyart methods use the convective class of a convective/stratiform classification) 2. **Morphological closing**: fill small holes within cells (tunable) - 3. **Cell seeding + labeling**: h-maxima seeds grown by watershed + 3. **Cell seeding + labeling**: h-maxima seeds — plus, with ``seed_carry``, + seeds carried from the previous frame — grown by watershed 4. **Size filtering**: remove cells smaller than min_gridpoints or larger than max_gridpoints (optional) 5. **Relabeling by size**: largest cell gets label 1, second-largest 2, etc. @@ -219,9 +252,13 @@ def __init__(self, config): self.min_gridpoints = config.min_cellsize_gridpoint self.max_gridpoints = config.max_cellsize_gridpoint self.h_maxima = config.h_maxima + self.seed_carry = config.seed_carry + self.seed_carry_max_frames = config.seed_carry_max_frames + self.seed_carry_min_separation = config.seed_carry_min_separation self.refl_name = config.reflectivity_var self.labels_name = config.labels_var self.z_level = config.z_level + self._decisions: SegmentationDecisions | None = None logger.info( "RadarCellSegmenter initialized: method=%s, params=%s", @@ -229,7 +266,12 @@ def __init__(self, config): self.method_params, ) - def segment(self, ds: xr.Dataset, grid_ds: xr.Dataset | None = None) -> xr.Dataset: + def segment( + self, + ds: xr.Dataset, + grid_ds: xr.Dataset | None = None, + carried: CarriedLabels | None = None, + ) -> xr.Dataset: """Segment 2D reflectivity and return dataset with cell labels. The configured ``method`` determines how the convective mask is built; @@ -247,14 +289,25 @@ def segment(self, ds: xr.Dataset, grid_ds: xr.Dataset | None = None) -> xr.Datas Full 3D gridded dataset (ingest's in-memory output). Required by the pyart convective/stratiform methods (they classify the 3D grid via ``pyart.xradar.Xgrid``); unused by ``threshold``. + carried : CarriedLabels | None + The previous frame's labels advected to this frame, with carry + ages. A connected component of the mask that holds fewer + independent seeds than prior cells has lost a core; only that + deficit is filled with extra watershed markers, so a cell's own + mispositioned footprint never splits it. Requires ``seed_carry``. Returns ------- xr.Dataset Copy of ``ds`` with an int32 ``cell_labels`` variable (0 = background, 1..N = cells by decreasing size). Label attrs - record method, threshold, z-level, and size-filter settings. + record method, threshold, z-level, and size-filter settings and, + with ``seed_carry``, ``carry_age`` per label. """ + require( + self.seed_carry or carried is None, + "Detection: carried labels offered while segmenter.seed_carry is off", + ) if self.refl_name not in ds.data_vars: raise ContractViolation( f"Detection: configured tracking field {self.refl_name!r} is " @@ -265,14 +318,7 @@ def segment(self, ds: xr.Dataset, grid_ds: xr.Dataset | None = None) -> xr.Datas refl = ds[self.refl_name].values binary_mask = self._convective_mask(refl, grid_ds) - labels = self._binary_to_labels( - binary_mask, - refl, - self.kernel_size, - self.filter_by_size, - self.min_gridpoints, - self.max_gridpoints, - ) + labels, ages = self._binary_to_labels(binary_mask, refl, carried) # Build attrs dict, excluding None values (NetCDF can't serialize None) attrs = { @@ -284,6 +330,11 @@ def segment(self, ds: xr.Dataset, grid_ds: xr.Dataset | None = None) -> xr.Datas } if self.max_gridpoints is not None: attrs["max_cellsize_gridpoint"] = self.max_gridpoints + # Provenance of the carry: which cells this scan owes to a carried seed. + # Only with the flag on (off must stay byte-identical) and only when + # there are cells (NetCDF cannot hold a zero-length attribute). + if self.seed_carry and ages: + attrs["carry_age"] = np.asarray(ages, dtype=np.int32) # Record the parameters actually passed to the selected method, so the # output states exactly what drove the classification. Skip None and cast # bool -> int for NetCDF attribute serialization. @@ -303,6 +354,12 @@ def segment(self, ds: xr.Dataset, grid_ds: xr.Dataset | None = None) -> xr.Datas ) return ds_out + def decisions(self) -> SegmentationDecisions: + """The decision record of the most recent ``segment`` call (analysis only).""" + if self._decisions is None: + raise ValueError("RadarCellSegmenter.decisions: nothing has been segmented yet") + return self._decisions + def _convective_mask(self, refl: np.ndarray, grid_ds: xr.Dataset | None) -> np.ndarray: """Build the 2D boolean convective mask for the configured method. @@ -335,67 +392,271 @@ def _binary_to_labels( self, binary_mask: np.ndarray, field: np.ndarray, - kernel_size: tuple, - filter_by_size: bool, - min_gridpoints: int, - max_gridpoints: int, - ) -> np.ndarray: - """Morphology, detect cells, filter.""" + carried: CarriedLabels | None, + ) -> tuple[np.ndarray, tuple[int, ...]]: + """Morphology, detect cells, filter; returns labels and carry age per label.""" from skimage.morphology import closing, footprint_rectangle - closed_mask = closing(binary_mask, footprint_rectangle(kernel_size)) + closed_mask = closing(binary_mask, footprint_rectangle(self.kernel_size)) + components, _ = label(closed_mask, structure=_COMPONENT_CONNECTIVITY) - labels = _label_maxtree(closed_mask, field, h=self.h_maxima) + basins, admitted, seed_rows = self._label_maxtree(closed_mask, field, carried, components) # if there are any cells, filter and/or renumber - if labels.max() > 0: - labels = self._filter_and_relabel( - labels, filter_by_size, min_gridpoints, max_gridpoints + labels = self._filter_and_relabel(basins) if basins.max() > 0 else basins + labels = labels.astype(np.int32) + + # A marker pixel keeps its own label through watershed and relabeling, + # so an admitted candidate's final label is read at its pixel; 0 means + # the size filter dropped that basin. Everything else is age 0. + ages = [0] * int(labels.max()) + for row, col, age in admitted: + final = int(labels[row, col]) + if final: + ages[final - 1] = age + 1 + + arrays = { + "mask": binary_mask, + "closed": closed_mask, + "components": components, + "field": field, + } + self._decisions = self._decide(arrays, basins, labels, seed_rows) + return labels, tuple(ages) + + def _decide( + self, arrays: dict, basins: np.ndarray, labels: np.ndarray, seed_rows: list[dict] + ) -> SegmentationDecisions: + """Assemble this scan's decision record from the stage arrays and seed rows.""" + components, field = arrays["components"], arrays["field"] + for row in seed_rows: + if row["marker_id"] is None: + continue + basin = basins == row["marker_id"] + row["basin_px"] = int(basin.sum()) + row["final_label"] = int(labels[basin].max()) if row["basin_px"] else 0 + markers = [r for r in seed_rows if r["marker_id"] is not None] + dropped = [r for r in markers if r["basin_px"] and r["final_label"] == 0] + n_dropped_small = sum(r["basin_px"] < self.min_gridpoints for r in dropped) + + component_rows = [] + for cid in range(1, int(components.max()) + 1): + inside = components == cid + here = [r for r in seed_rows if r["component_id"] == cid] + n_seeds = sum(r["origin"] == "HMAXIMA" for r in here) + n_claims = sum(r["origin"] == "CARRIED" for r in here) + component_rows.append( + SegmentationComponent( + component_id=cid, + area_px=int(inside.sum()), + field_max=float(np.nanmax(field[inside])), + field_min=float(np.nanmin(field[inside])), + n_seeds=n_seeds, + n_claims=n_claims, + deficit=n_claims - n_seeds, + n_admitted=sum( + r["origin"] == "CARRIED" and r["decision"] == "SEEDED" for r in here + ), + n_final_labels=len(set(labels[inside].tolist()) - {0}), + ) ) + carried = [r for r in seed_rows if r["origin"] == "CARRIED"] + frame = SegmentationFrame( + mask_px=int(arrays["mask"].sum()), + closed_mask_px=int(arrays["closed"].sum()), + n_components=len(component_rows), + n_independent_seeds=sum(r["origin"] == "HMAXIMA" for r in seed_rows), + n_carried_claims=sum(r["component_id"] is not None for r in carried), + n_carried_admitted=sum(r["decision"] == "SEEDED" for r in carried), + n_basins=sum(bool(r["basin_px"]) for r in markers), + n_dropped_small=n_dropped_small, + n_dropped_large=len(dropped) - n_dropped_small, + n_cells=int(labels.max()), + ) + seeds = tuple(SegmentationSeed(**row) for row in seed_rows) + return SegmentationDecisions(frame, seeds, tuple(component_rows)) + + def _label_maxtree( + self, + binary: np.ndarray, + field: np.ndarray, + carried: CarriedLabels | None, + components: np.ndarray, + ) -> tuple[np.ndarray, tuple[SeedCandidate, ...], list[dict]]: + """h-maxima seeding + watershed, with admitted carried seeds as extra markers. + + Identifies individual cells within a binary convection mask by seeding + each local intensity maximum (that rises at least ``h_maxima`` dBZ + above its surroundings) and growing watershed regions from those + seeds. Admitted carried seeds are appended as markers after the + h-maxima seeds — a union, never a replacement. - return labels.astype(np.int32) + Parameters + ---------- + binary : np.ndarray (bool) + Closed binary convection mask (output of morphological closing). + field : np.ndarray (float) + Reflectivity values aligned with `binary`. + carried : CarriedLabels | None + The previous frame's labels advected to this frame. + components : np.ndarray (int) + 8-connected components of ``binary``. - def _filter_and_relabel( + Returns + ------- + basins : np.ndarray (int32) + 0 = background, 1..M = one basin per marker (before size filtering). + admitted : tuple of (row, col, carry_age) + The carried seeds used as markers. + seed_rows : list of dict + One decision row per marker or carried claim. + """ + fp = np.where(binary, field, 0.0) + peaks = h_maxima(fp, h=self.h_maxima) + seeds, n_seeds = label(peaks) # label from scipy.ndimage + seed_rows = [ + _independent_seed(m, seeds == m, fp, components) for m in range(1, n_seeds + 1) + ] + admitted, carried_rows = self._admit(carried, binary, seeds, components) + for marker, (row, col, _) in enumerate(admitted, start=n_seeds + 1): + seeds[row, col] = marker + for entry in carried_rows: + if entry["decision"] == "SEEDED": + row, col = entry["row"], entry["col"] + entry.update( + marker_id=int(seeds[row, col]), peak_value=float(fp[row, col]), plateau_px=1 + ) + seed_rows += carried_rows + if n_seeds + len(admitted) == 0: + return np.zeros(binary.shape, dtype=np.int32), (), seed_rows + ws = watershed(-fp, seeds, mask=binary) + return np.where(binary, ws, 0).astype(np.int32), admitted, seed_rows + + def _admit( self, - labels: np.ndarray, - filter_by_size: bool, - min_gridpoints: int, - max_gridpoints: int, - ) -> np.ndarray: + carried: CarriedLabels | None, + binary: np.ndarray, + seeds: np.ndarray, + components: np.ndarray, + ) -> tuple[tuple[SeedCandidate, ...], list[dict]]: + """Fill each mask component's lost cores with carried seeds — nothing more. + + Per 8-connected component of the mask: every prior cell whose + advected footprint lands in it counts as a claim; a footprint that + already holds an independent seed is that component's survivor, not + a candidate. With ``k`` seeds and ``n`` claims, at most ``n - k`` + markers are added, farthest from a seed first, each at least + ``seed_carry_min_separation`` from any seed or marker. A cell's own + mispositioned footprint (one claim, one seed) therefore never splits + it; a second prior cell whose core dropped below ``h_maxima`` does + get rescued. Prior cells past the carry bound make no claim. + + Returns the admitted markers and one decision row per prior cell. + """ + if carried is None: + return (), [] + min_sep = self.seed_carry_min_separation + clearance = ( + distance_transform_edt(seeds == 0) if seeds.any() else np.full(seeds.shape, np.inf) + ) + + rows: list[dict] = [] + claims: dict[int, int] = {} + candidates: dict[int, list[dict]] = {} + for lab in np.unique(carried.labels[carried.labels > 0]): + age = carried.ages[int(lab) - 1] + footprint = carried.labels == lab + row, col = _footprint_seed(footprint) + entry = _seed_row( + origin="CARRIED", + row=row, + col=col, + prior_label=int(lab), + prior_age=int(age), + footprint_px=int(footprint.sum()), + ) + rows.append(entry) + if age + 1 > self.seed_carry_max_frames: + entry["decision"] = "EXPIRED" + continue + if not binary[row, col]: + entry["decision"] = "OUTSIDE_MASK" + continue + component = int(components[row, col]) + claims[component] = claims.get(component, 0) + 1 + entry.update( + component_id=component, + clearance_px=float(clearance[row, col]), + footprint_holds_seed=bool((seeds[footprint] > 0).any()), + ) + if entry["footprint_holds_seed"]: + entry["decision"] = "SURVIVOR" + continue + candidates.setdefault(component, []).append(entry) + + seeds_in = {c: len(np.unique(seeds[(components == c) & (seeds > 0)])) for c in claims} + admitted: list[SeedCandidate] = [] + for component, ranked in candidates.items(): + deficit = claims[component] - seeds_in[component] + placed = 0 + for entry in sorted(ranked, key=lambda e: (-e["clearance_px"], e["row"], e["col"])): + row, col = entry["row"], entry["col"] + if placed >= deficit: + entry["decision"] = "NO_DEFICIT" + elif entry["clearance_px"] < min_sep: + entry["decision"] = "TOO_CLOSE_TO_SEED" + elif any(np.hypot(row - r, col - c) < min_sep for r, c, _ in admitted): + entry["decision"] = "TOO_CLOSE_TO_MARKER" + else: + entry["decision"] = "SEEDED" + admitted.append((row, col, entry["prior_age"])) + placed += 1 + for entry in rows: + component = entry["component_id"] + if component is not None: + entry.update( + component_claims=claims[component], + component_seeds=seeds_in[component], + component_deficit=claims[component] - seeds_in[component], + ) + return tuple(admitted), rows + + def _filter_and_relabel(self, labels: np.ndarray) -> np.ndarray: """Filter, renumber by size.""" labels_unique, counts = np.unique(labels, return_counts=True) keep_mask = labels_unique > 0 - if filter_by_size: - if min_gridpoints > 1: - keep_mask &= counts >= min_gridpoints - num_small = np.sum((labels_unique > 0) & (counts < min_gridpoints)) + if self.filter_by_size: + if self.min_gridpoints > 1: + keep_mask &= counts >= self.min_gridpoints + num_small = np.sum((labels_unique > 0) & (counts < self.min_gridpoints)) if num_small > 0: - logger.debug(f"Removed {num_small} small (< {min_gridpoints})") + logger.debug(f"Removed {num_small} small (< {self.min_gridpoints})") - if max_gridpoints is not None: - keep_mask &= counts <= max_gridpoints - num_large = np.sum((labels_unique > 0) & (counts > max_gridpoints)) + if self.max_gridpoints is not None: + keep_mask &= counts <= self.max_gridpoints + num_large = np.sum((labels_unique > 0) & (counts > self.max_gridpoints)) if num_large > 0: - logger.debug(f"Removed {num_large} large (> {max_gridpoints})") + logger.debug(f"Removed {num_large} large (> {self.max_gridpoints})") labels_to_keep = labels_unique[keep_mask] - labels_renumbered = self._relabel_by_size(labels, labels_to_keep, counts) + labels_renumbered = self._relabel_by_size(labels, labels_to_keep, counts[keep_mask]) num_kept = len(labels_to_keep) num_removed = len(labels_unique) - 1 - num_kept - if filter_by_size and num_removed > 0: + if self.filter_by_size and num_removed > 0: logger.debug(f"Kept {num_kept}, removed {num_removed}") return labels_renumbered def _relabel_by_size( - self, labels: np.ndarray, labels_to_keep: np.ndarray, counts: np.ndarray + self, labels: np.ndarray, labels_to_keep: np.ndarray, keep_counts: np.ndarray ) -> np.ndarray: - """Renumber: largest=1.""" - keep_indices = np.isin(np.arange(len(counts)), labels_to_keep) - keep_counts = counts[keep_indices] + """Renumber: largest=1. + ``keep_counts[i]`` is the pixel count of ``labels_to_keep[i]``; the caller + slices both with the same mask so position is never read as label value. + """ sort_indices = np.argsort(-keep_counts) labels_sorted = labels_to_keep[sort_indices] diff --git a/src/adapt/modules/tracking/README.md b/src/adapt/modules/tracking/README.md index ed50468..c5dfd2c 100644 --- a/src/adapt/modules/tracking/README.md +++ b/src/adapt/modules/tracking/README.md @@ -31,7 +31,6 @@ constraints with optimisation as the last resort. |------|----------------| | `module.py` | `CellTracker` — per-scan flow, state, delegation | | `graph.py` | `TrackingGraph` (the only `networkx` home) | -| `projection.py` | `select_registration_labels` — minute-resolution registration hull | | `matching/geometry.py` | overlap (Opc/Ocp), hull dilation, length-scale strategies, `pair_cost`, centroids | | `matching/candidate.py` | `CandidateGenerator` — buffered high-recall candidate pairs | | `matching/validation.py` | `GeometricValidator` — bidirectional-overlap hard gate | @@ -54,7 +53,7 @@ Each frame pair is resolved in this order (registration-driven, optimisation las ``` scan-gap classification (physical time; hard reset on excess gap / non-monotonic time) ↓ -registration projected hulls (minute nearest the real gap) +registration projected hulls (previous labels advected by the full flow step) ↓ dilate hulls by projected_hull_buffer_km → liberal candidate pairs (high recall) ↓ @@ -212,10 +211,11 @@ optimisation → `PROPAGATED`). Only genuinely ambiguous connected components re ### Search Region -Candidates come from the registration projected hull — the minute-resolution -`registration_minutes` frame nearest the real scan gap (falling back to -`cell_projections[0]`) — dilated by `projected_hull_buffer_km` for recall -(`projection.select_registration_labels`, `matching/candidate.py`). +Candidates come from the registration projected hull — `cell_projections[0]`, +the previous scan's labels advected by the full flow step to the current scan +time (the minute-resolution `registration_minutes` frames stop short of it by +up to a minute) — dilated by `projected_hull_buffer_km` for recall +(`matching/candidate.py`). ### Diagnostics @@ -277,7 +277,7 @@ errors. This is a **designed extension point**, intentionally left unimplemented projected hulls* and *deterministic unique-overlap matching* in the hierarchy above — i.e. it adjusts the current-frame label field *before* matching, so all downstream stages operate on the corrected segmentation. It would consume the -same `select_registration_labels` hull plus the per-parent overlap structure +same `cell_projections[0]` registration hull plus the per-parent overlap structure already computed by `OverlapMatcher`. **Proposed shape when built.** A registered, swappable diff --git a/src/adapt/modules/tracking/config.py b/src/adapt/modules/tracking/config.py index 4221ff5..1df29b1 100644 --- a/src/adapt/modules/tracking/config.py +++ b/src/adapt/modules/tracking/config.py @@ -22,6 +22,7 @@ class TrackingConfig(BaseModel): max_tracking_gap_minutes: float max_speed_ms: float max_speed_multiplier: float + acceleration_floor_ms: float heading_change_penalty_weight: float projected_hull_buffer_km: float minimum_candidate_overlap: float diff --git a/src/adapt/modules/tracking/decision_log.py b/src/adapt/modules/tracking/decision_log.py new file mode 100644 index 0000000..643ea55 --- /dev/null +++ b/src/adapt/modules/tracking/decision_log.py @@ -0,0 +1,190 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Per-frame log of the tracker's matching decisions. + +``FrameLog`` accumulates, stage by stage, what happened to every candidate +pair of one scan pair and freezes it into ``TrackingDecisions`` — the +analysis-only record persisted to ``decisions.db``. It is a builder: mutable +for the duration of one ``CellTracker.track`` call, immutable afterwards. +""" + +from dataclasses import dataclass, field + +from adapt.contracts import ( + TrackingCandidate, + TrackingDecisions, + TrackingFrame, + TrackingSplitMergeTest, + TrackingUnmatched, +) + +__all__ = ["FrameLog", "LogCell", "STAGE_ASSIGNMENT", "STAGE_KINEMATIC", "STAGE_OVERLAP"] + +STAGE_OVERLAP = "OVERLAP" +STAGE_KINEMATIC = "KINEMATIC" +STAGE_ASSIGNMENT = "ASSIGNMENT" +_STAGE_ORDER = {STAGE_OVERLAP: 0, STAGE_KINEMATIC: 1, STAGE_ASSIGNMENT: 2} + + +@dataclass(frozen=True) +class LogCell: + """Identity of one previous cell as the log sees it.""" + + uid: str + label: int + steps_before: int = 0 + + +@dataclass +class FrameLog: + """Builder for one frame pair's ``TrackingDecisions``.""" + + dt_s: float | None + prev: list[LogCell] + curr_labels: list[int] + reset_code: str | None = None + _pairs: dict[tuple[int, int], dict] = field(default_factory=dict) + _tests: list[TrackingSplitMergeTest] = field(default_factory=list) + _matched_prev: dict[int, str] = field(default_factory=dict) # prev idx -> method + _split_children: set[int] = field(default_factory=set) + _merge_sources: set[str] = field(default_factory=set) # prev cell uids + _born_uids: dict[int, str] = field(default_factory=dict) + + # ── stage recorders ──────────────────────────────────────────────────── + + def candidate(self, i: int, j: int, **geometry) -> None: + """Stage 2/3: a generated pair with its overlap-gate result.""" + self._pairs[(i, j)] = { + **geometry, + "speed_ms": None, + "previous_speed_ms": None, + "accel_cap_ms": None, + "kinematic_code": None, + "displacement_m": None, + "length_scale_m": None, + "heading_change_deg": None, + "cost": None, + "component_n_prev": None, + "component_n_curr": None, + "cost_rank": None, + "match_method": None, + "last_stage": STAGE_OVERLAP, + "outcome": "CONTINUE" if geometry["overlap_passed"] else "REJECTED_OVERLAP", + } + + def kinematic(self, i: int, j: int, **motion) -> None: + """Stage 4: the kinematic gate's numbers; a code means rejection.""" + row = self._pairs[(i, j)] + row.update(motion, last_stage=STAGE_KINEMATIC) + if motion["kinematic_code"] is not None: + row["outcome"] = "REJECTED_KINEMATIC" + + def costed(self, i: int, j: int, **cost) -> None: + """Stage 5: the pair survived every gate and carries a cost.""" + self._pairs[(i, j)].update(cost, last_stage=STAGE_ASSIGNMENT, outcome="LOST_ASSIGNMENT") + + def matched(self, i: int, j: int, method: str) -> None: + """Stage 6: the pair won its component.""" + self._pairs[(i, j)].update(match_method=method, outcome="CONTINUE") + self._matched_prev[i] = method + + def component(self, prevs: list[int], currs: list[int]) -> None: + """Stage 6: the component (prev x curr) every costed pair was resolved in.""" + for i, j in self._pairs: + if i in prevs and j in currs and self._pairs[(i, j)]["cost"] is not None: + self._pairs[(i, j)].update(component_n_prev=len(prevs), component_n_curr=len(currs)) + + def split_merge_test(self, test: TrackingSplitMergeTest) -> None: + self._tests.append(test) + + def split_child(self, j: int) -> None: + self._split_children.add(j) + + def merge_source(self, prev_uid: str) -> None: + self._merge_sources.add(prev_uid) + + def born(self, j: int, uid: str) -> None: + self._born_uids[j] = uid + + # ── freeze ───────────────────────────────────────────────────────────── + + def freeze(self) -> TrackingDecisions: + self._rank_costs() + candidates = tuple( + TrackingCandidate( + prev_cell_uid=self.prev[i].uid, + prev_cell_label=self.prev[i].label, + curr_cell_label=self.curr_labels[j], + track_steps_before=self.prev[i].steps_before, + **row, + ) + for (i, j), row in sorted(self._pairs.items()) + ) + unmatched = tuple(self._unmatched_prev()) + tuple(self._unmatched_curr()) + rows = list(self._pairs.values()) + methods = list(self._matched_prev.values()) + frame = TrackingFrame( + dt_s=self.dt_s, + reset_code=self.reset_code, + n_prev=len(self.prev), + n_curr=len(self.curr_labels), + n_pairs=len(rows), + n_pass_overlap=sum(r["overlap_passed"] for r in rows), + n_pass_kinematic=sum(r["cost"] is not None for r in rows), + n_propagated=methods.count("PROPAGATED"), + n_hungarian=methods.count("HUNGARIAN"), + n_split=len(self._split_children), + n_merge=len(self._merge_sources), + n_initiation=len(self._born_uids) - len(self._split_children), + n_termination=len(self.prev) - len(self._matched_prev), + ) + return TrackingDecisions(frame, candidates, unmatched, tuple(self._tests)) + + def _rank_costs(self) -> None: + """Rank each previous cell's costed pairs, cheapest first (1-based).""" + by_prev: dict[int, list[tuple[float, int]]] = {} + for (i, j), row in self._pairs.items(): + if row["cost"] is not None: + by_prev.setdefault(i, []).append((row["cost"], j)) + for i, costs in by_prev.items(): + for rank, (_, j) in enumerate(sorted(costs), start=1): + self._pairs[(i, j)]["cost_rank"] = rank + + def _best(self, rows: list[tuple[int, dict]]) -> tuple[int | None, str | None]: + if not rows: + return None, None + label, row = max(rows, key=lambda item: _STAGE_ORDER[item[1]["last_stage"]]) + return label, row["last_stage"] + + def _reason(self, rows: list[tuple[int, dict]]) -> str: + if self.reset_code is not None: + return self.reset_code if self.reset_code == "FIRST_SCAN" else "RESET" + if not rows: + return "NO_CANDIDATE" + stages = {row["last_stage"] for _, row in rows} + if STAGE_ASSIGNMENT in stages: + return "LOST_ASSIGNMENT" + if STAGE_KINEMATIC in stages: + return "ALL_REJECTED_KINEMATIC" + return "ALL_REJECTED_OVERLAP" + + def _unmatched_prev(self): + for i, cell in enumerate(self.prev): + if i in self._matched_prev: + continue + rows = [(self.curr_labels[j], r) for (p, j), r in self._pairs.items() if p == i] + best_label, best_stage = self._best(rows) + reason = "MERGE_SOURCE" if cell.uid in self._merge_sources else self._reason(rows) + yield TrackingUnmatched( + "prev", cell.uid, cell.label, len(rows), best_label, best_stage, reason + ) + + def _unmatched_curr(self): + for j, uid in sorted(self._born_uids.items()): + rows = [(self.prev[i].label, r) for (i, c), r in self._pairs.items() if c == j] + best_label, best_stage = self._best(rows) + reason = "SPLIT_CHILD" if j in self._split_children else self._reason(rows) + yield TrackingUnmatched( + "curr", uid, self.curr_labels[j], len(rows), best_label, best_stage, reason + ) diff --git a/src/adapt/modules/tracking/models.py b/src/adapt/modules/tracking/models.py index f17eb0d..51305c0 100644 --- a/src/adapt/modules/tracking/models.py +++ b/src/adapt/modules/tracking/models.py @@ -34,7 +34,8 @@ class TrackingError(StrEnum): NON_MONOTONIC_TIME = "NON_MONOTONIC_TIME" TRACK_GAP_EXCEEDED = "TRACK_GAP_EXCEEDED" IRREGULAR_SCAN_CADENCE = "IRREGULAR_SCAN_CADENCE" - # Physical-constraint rejections + # Candidate-pair rejections + OVERLAP_REJECTED = "OVERLAP_REJECTED" VELOCITY_EXCEEDED = "VELOCITY_EXCEEDED" ACCELERATION_EXCEEDED = "ACCELERATION_EXCEEDED" @@ -61,10 +62,14 @@ class MatchDiagnostics: class TrackMotionState: """Per-track velocity carried forward for acceleration and heading checks. - ``speed`` is in m/s; ``heading`` is in radians measured as ``atan2(vy, vx)``. - ``has_velocity`` is False until a track has been observed across two scans. + ``speed`` is the running mean of the track's step speeds (m/s) over + ``n_steps`` — the acceleration cap references the track's established + motion, not one jittery step; ``heading`` is the last step's direction in + radians measured as ``atan2(vy, vx)``. ``has_velocity`` is False until a + track has been observed across two scans. """ speed: float = 0.0 heading: float = 0.0 has_velocity: bool = False + n_steps: int = 0 diff --git a/src/adapt/modules/tracking/module.py b/src/adapt/modules/tracking/module.py index 3a321e9..be1587c 100644 --- a/src/adapt/modules/tracking/module.py +++ b/src/adapt/modules/tracking/module.py @@ -40,7 +40,8 @@ import pandas as pd import xarray as xr -from adapt.contracts import stat_column +from adapt.contracts import TrackingDecisions, TrackingSplitMergeTest, stat_column +from adapt.modules.tracking.decision_log import FrameLog, LogCell from adapt.modules.tracking.events import ( build_cell_events_dataframe, event_continue, @@ -76,7 +77,6 @@ heading_change_degrees, heading_change_radians, ) -from adapt.modules.tracking.projection import select_registration_labels from adapt.utils.time import normalize_time_scalar # Beyond this ratio between consecutive scan intervals the cadence is flagged @@ -142,12 +142,17 @@ def __init__(self, config): self.validator = GeometricValidator( config.minimum_candidate_overlap, config.minimum_projected_overlap ) - self.motion = MotionValidator(config.max_speed_ms, config.max_speed_multiplier) + self.motion = MotionValidator( + config.max_speed_ms, config.max_speed_multiplier, config.acceleration_floor_ms + ) self.heading_penalty_weight = config.heading_change_penalty_weight self._previous_scan: tuple | None = None # (time, node_ids) self._cell_identity: dict[int, tuple[str, str]] = {} self._track_motion: dict[int, TrackMotionState] = {} # track_index → kinematics self._prev_dt_s: float | None = None # last good scan interval (cadence check) + # Decision log of the scan being tracked; frozen into _decisions at the end. + self._log = FrameLog(dt_s=None, prev=[], curr_labels=[], reset_code="FIRST_SCAN") + self._decisions: TrackingDecisions | None = None logger.info( "CellTracker initialized: buffer=%.1fkm min_opc=%.2f min_ocp=%.2f" @@ -187,17 +192,30 @@ def track( cells_current = self._extract_cells_from_analyzer(ds_projected, cell_stats_df) events: list[dict] = [] + curr_labels = [int(cell["cell_id"]) for cell in cells_current] if self._previous_scan is None: + self._log = FrameLog( + dt_s=None, prev=[], curr_labels=curr_labels, reset_code="FIRST_SCAN" + ) node_ids = self._initialize_tracks(current_time, cells_current) self._previous_scan = (current_time, node_ids) - for node_id in node_ids: + for j, node_id in enumerate(node_ids): events.append( event_initiation(self.graph, self._cell_identity, current_time, node_id) ) + self._log.born(j, str(self.graph.get_node_attr(node_id, "cell_uid"))) + self._decisions = self._log.freeze() else: prev_time, prev_node_ids = self._previous_scan dt_s = self._to_epoch_seconds(current_time) - self._to_epoch_seconds(prev_time) - if self._gap_forces_reset(dt_s, prev_time, current_time): + reset_code = self._gap_reset_code(dt_s, prev_time, current_time) + self._log = FrameLog( + dt_s=dt_s, + prev=self._log_cells(prev_node_ids), + curr_labels=curr_labels, + reset_code=reset_code, + ) + if reset_code is not None: events = self._reset_tracks(prev_node_ids, current_time, cells_current) else: self._prev_dt_s = dt_s @@ -217,16 +235,38 @@ def get_cell_identity(self, track_index: int) -> tuple[str, str]: raise ValueError(f"Missing cell identity for track_index={track_index}") return self._cell_identity[track_index] + def decisions(self) -> TrackingDecisions: + """The decision record of the most recent ``track`` call (analysis only).""" + if self._decisions is None: + raise ValueError("CellTracker.decisions: no scan has been tracked yet") + return self._decisions + + def _log_cells(self, node_ids: list[int]) -> list[LogCell]: + """Previous cells as the decision log identifies them.""" + cells = [] + for node_id in node_ids: + track_index = int(self.graph.get_node_attr(node_id, "track_index") or 0) + state = self._track_motion.get(track_index) + cells.append( + LogCell( + uid=str(self.graph.get_node_attr(node_id, "cell_uid")), + label=int(self.graph.get_node_attr(node_id, "cell_id")), + steps_before=state.n_steps if state is not None else 0, + ) + ) + return cells + # ------------------------------------------------------------------ # Scan-gap classification (physical time) # ------------------------------------------------------------------ - def _gap_forces_reset(self, dt_s: float, prev_time, curr_time) -> bool: + def _gap_reset_code(self, dt_s: float, prev_time, curr_time) -> str | None: """Classify the inter-scan interval; log a structured code. - Returns True when tracks must be terminated and restarted (non-monotonic - time or a gap above the hard limit) — never raises, never matches across - the gap. An irregular-but-monotonic cadence is a diagnostic warning only. + Returns the reset code when tracks must be terminated and restarted + (non-monotonic time or a gap above the hard limit) — never raises, never + matches across the gap. An irregular-but-monotonic cadence is a + diagnostic warning only and returns None. """ if dt_s <= 0: logger.error( @@ -236,7 +276,7 @@ def _gap_forces_reset(self, dt_s: float, prev_time, curr_time) -> bool: curr_time, dt_s, ) - return True + return TrackingError.NON_MONOTONIC_TIME.value if dt_s > self.max_tracking_gap_s: logger.error( "tracking_error code=%s dt_minutes=%.1f limit_minutes=%.1f", @@ -244,7 +284,7 @@ def _gap_forces_reset(self, dt_s: float, prev_time, curr_time) -> bool: dt_s / 60.0, self.max_tracking_gap_minutes, ) - return True + return TrackingError.TRACK_GAP_EXCEEDED.value if self._prev_dt_s is not None and ( dt_s > _CADENCE_IRREGULAR_RATIO * self._prev_dt_s or dt_s * _CADENCE_IRREGULAR_RATIO < self._prev_dt_s @@ -255,7 +295,7 @@ def _gap_forces_reset(self, dt_s: float, prev_time, curr_time) -> bool: dt_s, self._prev_dt_s, ) - return False + return None def _reset_tracks( self, prev_node_ids: list[int], curr_time, cells_current: list[dict] @@ -266,8 +306,10 @@ def _reset_tracks( for node_id in prev_node_ids ] self._initialize_tracks(curr_time, cells_current) - for node_id in self.graph.get_nodes_at_time(curr_time): + for j, node_id in enumerate(self.graph.get_nodes_at_time(curr_time)): events.append(event_initiation(self.graph, self._cell_identity, curr_time, node_id)) + self._log.born(j, str(self.graph.get_node_attr(node_id, "cell_uid"))) + self._decisions = self._log.freeze() return events # ------------------------------------------------------------------ @@ -411,10 +453,17 @@ def _update_motion( prev_cy = float(self.graph.get_node_attr(prev_node, "centroid_y")) vx = (float(curr_cell["centroid_x"]) - prev_cx) / dt_s vy = (float(curr_cell["centroid_y"]) - prev_cy) / dt_s + step_speed = math.hypot(vx, vy) + prior = self._track_motion.get(track_index) + n_steps = (prior.n_steps if prior is not None else 0) + 1 + mean_speed = ( + step_speed if prior is None else prior.speed + (step_speed - prior.speed) / n_steps + ) self._track_motion[track_index] = TrackMotionState( - speed=math.hypot(vx, vy), + speed=mean_speed, heading=math.atan2(vy, vx), has_velocity=True, + n_steps=n_steps, ) def _record_continue( @@ -525,31 +574,68 @@ def _build_edges( edges: dict[tuple[int, int], _EdgeCost] = {} for i, j in pairs: + prev_node = all_prev_ids[i] + track_index = int(self.graph.get_node_attr(prev_node, "track_index") or 0) result = self.validator.validate(prev_hulls[i], curr_masks[j]) + hull_c = hull_centroid_m[i] or (float("nan"), float("nan")) + self._log.candidate( + i, + j, + hull_area_px=int(hull_area_px[i]), + hull_centroid_x=hull_c[0], + hull_centroid_y=hull_c[1], + curr_area_px=int(curr_cells[j]["area_px"]), + curr_centroid_x=float(curr_cells[j]["centroid_x"]), + curr_centroid_y=float(curr_cells[j]["centroid_y"]), + intersection_px=int(np.count_nonzero(prev_hulls[i] & curr_masks[j])), + opc=result.opc, + ocp=result.ocp, + overlap_passed=result.passed, + ) if not result.passed: + # Same "tracking_error code=" grammar as the gap and kinematic + # rejections, so one grep explains why a track ended. + logger.debug( + "tracking_error code=%s track=%d curr=%d opc=%.2f ocp=%.2f", + TrackingError.OVERLAP_REJECTED.value, + track_index, + j, + result.opc, + result.ocp, + ) continue - prev_node = all_prev_ids[i] prev_cx = float(self.graph.get_node_attr(prev_node, "centroid_x")) prev_cy = float(self.graph.get_node_attr(prev_node, "centroid_y")) curr_cx = float(curr_cells[j]["centroid_x"]) curr_cy = float(curr_cells[j]["centroid_y"]) - track_index = int(self.graph.get_node_attr(prev_node, "track_index") or 0) state = self._track_motion.get(track_index) previous_speed = state.speed if state and state.has_velocity else None decision = self.motion.check(prev_cx, prev_cy, curr_cx, curr_cy, dt_s, previous_speed) + self._log.kinematic( + i, + j, + speed_ms=decision.speed_ms, + previous_speed_ms=previous_speed, + accel_cap_ms=decision.cap_ms, + kinematic_code=decision.code.value if decision.code is not None else None, + ) if not decision.ok: # B3 kinematic gate logger.debug( - "tracking_error code=%s track=%d speed=%.1fm/s", + "tracking_error code=%s track=%d curr=%d speed=%.1fm/s prev_speed=%s", decision.code.value, track_index, + j, decision.speed_ms, + f"{previous_speed:.1f}" if previous_speed is not None else "-", ) continue - hull_c = hull_centroid_m[i] - displacement = math.hypot(curr_cx - hull_c[0], curr_cy - hull_c[1]) if hull_c else 0.0 + hull_xy = hull_centroid_m[i] + displacement = ( + math.hypot(curr_cx - hull_xy[0], curr_cy - hull_xy[1]) if hull_xy else 0.0 + ) length_m = length_scale( self.length_scale_name, hull_area_px[i], @@ -558,11 +644,22 @@ def _build_edges( self.geometry_length_scale_km, ) cost = pair_cost(result.opc, result.ocp, displacement, length_m) - if self.heading_penalty_weight > 0.0 and state is not None and state.has_velocity: + heading_change_deg = None + if state is not None and state.has_velocity: cand_heading = math.atan2(curr_cy - prev_cy, curr_cx - prev_cx) # B5 - cost += self.heading_penalty_weight * heading_change_radians( - state.heading, cand_heading - ) + heading_change_deg = heading_change_degrees(state.heading, cand_heading) + if self.heading_penalty_weight > 0.0: + cost += self.heading_penalty_weight * heading_change_radians( + state.heading, cand_heading + ) + self._log.costed( + i, + j, + displacement_m=displacement, + length_scale_m=length_m, + heading_change_deg=heading_change_deg, + cost=cost, + ) edges[(i, j)] = _EdgeCost(result.opc, result.ocp, displacement, cost) return edges @@ -581,11 +678,13 @@ def _track_frame_pair( ) if projections_missing: logger.warning("No cell_projections — resetting %d tracks", len(prev_node_ids)) + self._log.reset_code = "NO_PROJECTIONS" return self._reset_tracks(prev_node_ids, curr_time, curr_cells) - # Registration hull at the minute nearest the real gap (falls back to - # cell_projections[0] when minute frames are absent). - proj_labels = select_registration_labels(ds_curr, dt_s) + # Registration hull: the previous labels advected by the full flow step + # (fraction 1.0 of the scan gap). The minute frames are registered to + # whole minutes short of the current scan, so they under-advect. + proj_labels = np.asarray(ds_curr["cell_projections"].values[0]) matched_prev: dict[int, int] = {} # prev_idx → new curr node_id matched_curr: dict[int, int] = {} # curr_idx → new curr node_id @@ -623,6 +722,7 @@ def _track_frame_pair( len(born) - len(split_born), len(split_born), ) + self._decisions = self._log.freeze() return events # ------------------------------------------------------------------ @@ -656,11 +756,17 @@ def record(i: int, c: int, method: MatchMethod) -> dict: ) forced, remaining = ConstraintPropagator.resolve(list(edge_costs.keys())) - events = [record(i, c, MatchMethod.PROPAGATED) for i, c in forced] + events = [] + for i, c in forced: + self._log.component([i], [c]) + self._log.matched(i, c, MatchMethod.PROPAGATED.value) + events.append(record(i, c, MatchMethod.PROPAGATED)) component_costs = {edge: edge_costs[edge].cost for edge in remaining} for prevs, currs in AssignmentGraph(remaining).components(): + self._log.component(prevs, currs) for i, c in HungarianMatcher.match(prevs, currs, component_costs): + self._log.matched(i, c, MatchMethod.HUNGARIAN.value) events.append(record(i, c, MatchMethod.HUNGARIAN)) return events @@ -696,12 +802,24 @@ def _detect_splits( for prev_idx, curr_node in matched_prev.items(): cell_id = self.graph.get_node_attr(prev_node_ids[prev_idx], "cell_id") overlap = self._hull_overlap_fraction(proj_labels, cell_id, b_mask) + self._log.split_merge_test( + TrackingSplitMergeTest( + "SPLIT", + str(self.graph.get_node_attr(curr_node, "cell_uid")), + int(curr_cells[b_idx]["cell_id"]), + float(overlap), + self.split_overlap, + overlap >= self.split_overlap, + ) + ) if overlap >= self.split_overlap and overlap > best_overlap: best_parent, best_overlap = curr_node, overlap if best_parent is not None: child_node = self._new_track_node(curr_cells[b_idx], curr_time) self.graph.add_edge(best_parent, child_node, edge_type="SPLIT", cost=0.0) split_born.add(b_idx) + self._log.born(b_idx, str(self.graph.get_node_attr(child_node, "cell_uid"))) + self._log.split_child(b_idx) events.append( event_split(self.graph, self._cell_identity, curr_time, best_parent, child_node) ) @@ -725,11 +843,22 @@ def _detect_merges( overlap = self._hull_overlap_fraction( proj_labels, cell_id, curr_cells[c_idx]["mask"] ) + self._log.split_merge_test( + TrackingSplitMergeTest( + "MERGE", + str(self.graph.get_node_attr(curr_node, "cell_uid")), + int(cell_id), + float(overlap), + self.split_overlap, + overlap >= self.split_overlap, + ) + ) if overlap >= self.split_overlap and overlap > best_overlap: best_target, best_overlap = curr_node, overlap if best_target is not None: self.graph.add_edge(d_node, best_target, edge_type="MERGE", cost=0.0) merged[d_node] = best_target + self._log.merge_source(str(self.graph.get_node_attr(d_node, "cell_uid"))) events.append( event_merge(self.graph, self._cell_identity, curr_time, d_node, best_target) ) @@ -745,6 +874,7 @@ def _emit_births( continue node_id = self._new_track_node(curr_cells[b_idx], curr_time) events.append(event_initiation(self.graph, self._cell_identity, curr_time, node_id)) + self._log.born(b_idx, str(self.graph.get_node_attr(node_id, "cell_uid"))) return events def _emit_terminations( diff --git a/src/adapt/modules/tracking/motion.py b/src/adapt/modules/tracking/motion.py index 816b24e..8032b3b 100644 --- a/src/adapt/modules/tracking/motion.py +++ b/src/adapt/modules/tracking/motion.py @@ -39,19 +39,25 @@ class MotionDecision: ok: bool speed_ms: float code: TrackingError | None = None + cap_ms: float | None = None # the acceleration cap applied, None when no prior speed class MotionValidator: """Reject candidate pairs that violate hard kinematic limits. A pair is rejected when its implied speed exceeds ``max_speed_ms`` (absolute - cap) or ``max_speed_multiplier * previous_speed`` (acceleration cap). Rejected - pairs never reach overlap or Hungarian matching. + cap) or ``max(max_speed_multiplier * previous_speed, acceleration_floor_ms)`` + (acceleration cap). The floor keeps a slow prior step — centroid jitter of a + px or two — from vetoing ordinary storm motion. Rejected pairs never reach + overlap or Hungarian matching. """ - def __init__(self, max_speed_ms: float, max_speed_multiplier: float): + def __init__( + self, max_speed_ms: float, max_speed_multiplier: float, acceleration_floor_ms: float + ): self.max_speed_ms = max_speed_ms self.max_speed_multiplier = max_speed_multiplier + self.acceleration_floor_ms = acceleration_floor_ms @staticmethod def speed_ms(prev_x: float, prev_y: float, curr_x: float, curr_y: float, dt_s: float) -> float: @@ -71,10 +77,9 @@ def check( speed = self.speed_ms(prev_x, prev_y, curr_x, curr_y, dt_s) if speed > self.max_speed_ms: return MotionDecision(False, speed, TrackingError.VELOCITY_EXCEEDED) - if ( - previous_speed is not None - and previous_speed > 0.0 - and speed > self.max_speed_multiplier * previous_speed - ): - return MotionDecision(False, speed, TrackingError.ACCELERATION_EXCEEDED) - return MotionDecision(True, speed, None) + cap = None + if previous_speed is not None and previous_speed > 0.0: + cap = max(self.max_speed_multiplier * previous_speed, self.acceleration_floor_ms) + if speed > cap: + return MotionDecision(False, speed, TrackingError.ACCELERATION_EXCEEDED, cap) + return MotionDecision(True, speed, None, cap) diff --git a/src/adapt/modules/tracking/projection.py b/src/adapt/modules/tracking/projection.py deleted file mode 100644 index 6a09677..0000000 --- a/src/adapt/modules/tracking/projection.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright © 2026, UChicago Argonne, LLC -# See LICENSE for terms and disclaimer. - -"""Registration-based projected hulls. - -The projection module already advects the previous scan's cell labels forward at -minute resolution (``registration_minutes`` + ``interpolation_fraction``). The -tracker consumes the minute frame closest to the actual scan gap so matching uses -a hull registered to the real elapsed time, not a fixed single-step projection. -Falls back to ``cell_projections[0]`` (the whole-step registration) when the -minute frames are absent. Pure xarray access — no state. -""" - -import numpy as np -import xarray as xr - -__all__ = ["select_registration_labels"] - - -def select_registration_labels(ds: xr.Dataset, dt_s: float) -> np.ndarray: - """Previous-scan cell labels registered to the current frame. - - Picks the ``registration_minutes`` frame whose minute is closest to ``dt_s``; - falls back to ``cell_projections[0]`` if no minute frames are present. - """ - if "registration_minutes" in ds.data_vars and ds["registration_minutes"].sizes.get("minute", 0): - minutes = np.asarray(ds["minute"].values, dtype=float) - nearest = int(np.argmin(np.abs(minutes - dt_s / 60.0))) - return np.asarray(ds["registration_minutes"].values[nearest]) - return np.asarray(ds["cell_projections"].values[0]) diff --git a/src/adapt/persistence/decision_store.py b/src/adapt/persistence/decision_store.py new file mode 100644 index 0000000..c42b77e --- /dev/null +++ b/src/adapt/persistence/decision_store.py @@ -0,0 +1,82 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Writer for the analysis-only decision log (``decisions.db``). + +Each decision bundle (``adapt.contracts.decisions``) maps to fixed tables whose +static DDL lives in ``configuration/schemas/collection_decisions.sql``; a row is +the record's fields plus the run/scan stamps from ``PersistenceMeta``. Writes +replace on the primary key, so a post-hoc re-run under the same run id +replaces its own rows. One connection per scan: use as a context manager. +""" + +import sqlite3 +from dataclasses import asdict + +from adapt.contracts import PersistenceMeta, SegmentationDecisions, TrackingDecisions +from adapt.persistence.sqlite_store import SqliteStore +from adapt.utils.time import to_scan_iso + +__all__ = ["DecisionStore"] + + +class DecisionStore(SqliteStore): + """One-scan writer for ``decisions.db``.""" + + def __init__(self, db_path) -> None: + super().__init__(db_path, "collection_decisions.sql") + + def __enter__(self) -> "DecisionStore": + return self + + def __exit__(self, *exc) -> None: + self.close() + + def write( + self, decisions: TrackingDecisions | SegmentationDecisions, meta: PersistenceMeta + ) -> None: + """Write one scan's bundle; every table of the bundle is written, even if empty.""" + if meta.scan_id is None or meta.scan_time is None: + raise ValueError("DecisionStore.write needs scan_id and scan_time") + stamp = { + "run_id": meta.run_id, + "scan_id": meta.scan_id, + "scan_time": to_scan_iso(meta.scan_time), + } + tables: tuple[tuple[str, tuple], ...] + match decisions: + case TrackingDecisions(): + tables = ( + ("tracking_frames", (decisions.frame,)), + ("tracking_candidates", decisions.candidates), + ("tracking_unmatched", decisions.unmatched), + ("tracking_split_merge_tests", decisions.split_merge_tests), + ) + case SegmentationDecisions(): + tables = ( + ("segmentation_frames", (decisions.frame,)), + ("segmentation_seeds", decisions.seeds), + ("segmentation_components", decisions.components), + ) + case _: + raise TypeError(f"DecisionStore cannot write {type(decisions).__name__}") + + conn = self._get_connection() + for table, records in tables: + rows = [{**stamp, **asdict(record)} for record in records] + if table == "segmentation_seeds": + for seq, row in enumerate(rows): + row["seq"] = seq + self._replace(conn, table, rows) + conn.commit() + + @staticmethod + def _replace(conn: sqlite3.Connection, table: str, rows: list[dict]) -> None: + if not rows: + return + columns = list(rows[0]) + placeholders = ", ".join("?" for _ in columns) + conn.executemany( + f"INSERT OR REPLACE INTO {table} ({', '.join(columns)}) VALUES ({placeholders})", + [tuple(row[c] for c in columns) for row in rows], + ) diff --git a/src/adapt/persistence/output_router.py b/src/adapt/persistence/output_router.py index 8e7c4b0..e2c7790 100644 --- a/src/adapt/persistence/output_router.py +++ b/src/adapt/persistence/output_router.py @@ -16,12 +16,14 @@ from datetime import datetime from adapt.contracts import ( + DecisionTableWrite, NetcdfArtifact, PersistenceMeta, PersistenceSpec, ProductTableWrite, TrackTablesWrite, ) +from adapt.persistence.decision_store import DecisionStore from adapt.persistence.errors import StoreError from adapt.persistence.objects import ArtifactMeta, ObjectStore from adapt.persistence.products import SchemaLedger, TableWriter @@ -80,9 +82,23 @@ def _dispatch( self._write_table(module_name, spec, result, meta) case TrackTablesWrite(): self._write_tracking(module_name, spec, result, meta) + case DecisionTableWrite(): + self._write_decisions(spec, result, meta) case _: raise TypeError(f"{module_name}: unknown persistence spec {type(spec).__name__}") + def _write_decisions( + self, spec: DecisionTableWrite, result: dict, meta: PersistenceMeta + ) -> None: + """Analysis-only decision log: never a scan product, so no catalog link. + + One connection per scan, as for the tracking tables. + """ + if spec.key not in result: + return + with DecisionStore(self._collection.decisions_path) as store: + store.write(result[spec.key], meta) + def _write_netcdf( self, module_name: str, spec: NetcdfArtifact, result: dict, meta: PersistenceMeta ) -> None: diff --git a/src/adapt/persistence/store.py b/src/adapt/persistence/store.py index ab7d5de..5645720 100644 --- a/src/adapt/persistence/store.py +++ b/src/adapt/persistence/store.py @@ -43,7 +43,11 @@ def _reject_legacy_root(root_path: Path) -> None: class Collection: - """One collection's on-disk domain: catalog.db + products.db + objects/.""" + """One collection's on-disk domain: catalog.db + products.db + decisions.db + objects/. + + ``decisions.db`` is the analysis-only decision log (segmentation and + tracking decisions); the pipeline never reads it. + """ def __init__(self, collection_id: str, path: Path, catalog: Catalog) -> None: self.collection_id = collection_id @@ -51,6 +55,7 @@ def __init__(self, collection_id: str, path: Path, catalog: Catalog) -> None: self.catalog = catalog self.objects_dir = path / "objects" self.products_path = path / "products.db" + self.decisions_path = path / "decisions.db" def close(self) -> None: self.catalog.close() @@ -85,6 +90,8 @@ def collection(self, collection_id: str) -> Collection: (collection_dir / "objects").mkdir(exist_ok=True) products = SqliteStore(collection_dir / "products.db", "collection_products.sql") products.close() + decisions = SqliteStore(collection_dir / "decisions.db", "collection_decisions.sql") + decisions.close() self.validate_collection(collection_id) coll = Collection(collection_id, collection_dir, Catalog(collection_dir)) diff --git a/src/adapt/runtime/processor.py b/src/adapt/runtime/processor.py index 2fe1c94..1213db9 100644 --- a/src/adapt/runtime/processor.py +++ b/src/adapt/runtime/processor.py @@ -50,6 +50,11 @@ logger = logging.getLogger(__name__) +# Context keys the processor injects per group; they must never enter a scan's +# result (and hence the rolling history), or each entry would retain its +# predecessor and every scan's grids would stay alive for the whole run. +_INJECTED_CONTEXT_KEYS = frozenset({"scan_history", "prior_scan"}) + class RadarProcessor(threading.Thread): """Worker thread that processes NEXRAD files through two execution graphs. @@ -336,6 +341,13 @@ def _run_scan( # ── Rolling window: run all executor groups in history-size order # required_history=N means N scans total (N-1 prior + current). # Skip if fewer than N-1 prior scans are available. + # The previous completed scan is a usable predecessor only when the + # time gap to it is within the multi-scan bound (False on the first + # scan). Single-scan modules that want it (detection's seed carry) + # read ``prior_scan``; multi-scan groups gate on the same check. + time_gap_valid, time_gap_minutes = self._validate_time_gap(base_ctx["scan_time"]) + prior_scan = self._scan_history[-1] if time_gap_valid else None + result: dict = {} for req_hist, executor in sorted(self._executors.items()): prior_needed = req_hist - 1 @@ -348,19 +360,15 @@ def _run_scan( ) continue - if req_hist > 1: - # Validate time gap before running multi-scan modules - current_scan_time = base_ctx["scan_time"] - time_gap_valid, time_gap_minutes = self._validate_time_gap(current_scan_time) - if not time_gap_valid: - logger.warning( - "Time gap %.1f min > %.1f min, skipping multi-scan modules.", - time_gap_minutes, - self._max_time_gap_minutes, - ) - continue - - ctx = {**base_ctx, **result} + if req_hist > 1 and not time_gap_valid: + logger.warning( + "Time gap %.1f min > %.1f min, skipping multi-scan modules.", + time_gap_minutes, + self._max_time_gap_minutes, + ) + continue + + ctx = {**base_ctx, **result, "prior_scan": prior_scan} if req_hist > 1: # Build scan_history: (N-1) prior entries + current partial context prior = self._scan_history[-prior_needed:] if prior_needed else [] @@ -368,7 +376,12 @@ def _run_scan( # authoritative — a module result must never override it. ctx["scan_history"] = list(prior) + [{**result, **base_ctx}] group_result = executor.run(ctx) - result.update(group_result) + # The executor returns the whole context. Injected keys must not + # enter ``result``: it becomes the next history entry, and an + # entry holding its predecessor would retain every scan's grids. + result.update( + {k: v for k, v in group_result.items() if k not in _INJECTED_CONTEXT_KEYS} + ) scan_time = base_ctx["scan_time"] elapsed_s = time.perf_counter() - t0 diff --git a/tests/configuration/test_seed_carry_config.py b/tests/configuration/test_seed_carry_config.py new file mode 100644 index 0000000..a6fb479 --- /dev/null +++ b/tests/configuration/test_seed_carry_config.py @@ -0,0 +1,57 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Seed-carry configuration: off by default, bounded, and needs projection.""" + +import pytest +from pydantic import ValidationError + +from adapt.configuration.schemas.param import SegmenterConfig +from adapt.configuration.schemas.user import UserSegmenterConfig +from adapt.execution.nodes.detection import DetectModule + +pytestmark = pytest.mark.unit + + +def test_seed_carry_is_off_by_default(): + seg = SegmenterConfig() + assert seg.seed_carry is False + assert seg.seed_carry_max_frames == 2 + assert seg.seed_carry_min_separation == 3 + + +@pytest.mark.parametrize("field", ["seed_carry_max_frames", "seed_carry_min_separation"]) +def test_seed_carry_bounds_below_one_are_rejected(field): + with pytest.raises(ValidationError): + SegmenterConfig(**{field: 0}) + + +def test_build_config_passes_seed_carry_fields(make_config): + cfg = make_config( + segmenter=UserSegmenterConfig( + seed_carry=True, seed_carry_max_frames=4, seed_carry_min_separation=5 + ) + ) + det = DetectModule.build_config(cfg) + assert det.seed_carry is True + assert det.seed_carry_max_frames == 4 + assert det.seed_carry_min_separation == 5 + + +@pytest.mark.parametrize( + "selection", + [ + {"exclude_modules": ["projection"]}, + {"only_modules": ["ingest", "detection"]}, + {"modules": ["ingest", "detection", "analysis"]}, + ], +) +def test_build_config_raises_when_seed_carry_is_on_without_projection(make_config, selection): + cfg = make_config(segmenter=UserSegmenterConfig(seed_carry=True)).model_copy(update=selection) + with pytest.raises(ValueError, match="projection"): + DetectModule.build_config(cfg) + + +def test_build_config_accepts_seed_carry_when_projection_is_enabled(make_config): + cfg = make_config(segmenter=UserSegmenterConfig(seed_carry=True)) + assert DetectModule.build_config(cfg).seed_carry is True diff --git a/tests/contracts/test_decisions_contract.py b/tests/contracts/test_decisions_contract.py new file mode 100644 index 0000000..a5db663 --- /dev/null +++ b/tests/contracts/test_decisions_contract.py @@ -0,0 +1,61 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Decision-record contracts: bundles are internally consistent and frozen.""" + +import dataclasses + +import pytest + +from adapt.contracts import ( + ContractViolation, + SegmentationDecisions, + SegmentationFrame, + TrackingDecisions, + TrackingFrame, + check_segmentation_decisions, + check_tracking_decisions, +) + +pytestmark = pytest.mark.unit + + +def _tracking_frame(**kw) -> TrackingFrame: + fields = dict.fromkeys( + ( + "n_prev", + "n_curr", + "n_pairs", + "n_pass_overlap", + "n_pass_kinematic", + "n_propagated", + "n_hungarian", + "n_split", + "n_merge", + "n_initiation", + "n_termination", + ), + 0, + ) + return TrackingFrame(dt_s=None, reset_code="FIRST_SCAN", **{**fields, **kw}) + + +def test_tracking_bundle_with_no_pairs_is_valid(): + check_tracking_decisions(TrackingDecisions(_tracking_frame(), (), (), ())) + + +def test_tracking_candidate_rows_must_match_generated_pair_count(): + with pytest.raises(ContractViolation): + check_tracking_decisions(TrackingDecisions(_tracking_frame(n_pairs=1), (), (), ())) + + +def test_segmentation_component_rows_must_match_component_count(): + frame = SegmentationFrame(0, 0, 1, 0, 0, 0, 0, 0, 0, 0) + with pytest.raises(ContractViolation): + check_segmentation_decisions(SegmentationDecisions(frame, (), ())) + + +def test_records_are_frozen(): + frame = _tracking_frame() + with pytest.raises(dataclasses.FrozenInstanceError): + frame.n_prev = 3 # type: ignore[misc] diff --git a/tests/contracts/test_persistence_specs.py b/tests/contracts/test_persistence_specs.py index d6e7983..130d6d9 100644 --- a/tests/contracts/test_persistence_specs.py +++ b/tests/contracts/test_persistence_specs.py @@ -29,6 +29,7 @@ def test_all_spec_types_exported(): "ProductTableWrite", "NetcdfArtifact", "TrackTablesWrite", + "DecisionTableWrite", "PersistenceSpec", "PersistenceMeta", ): diff --git a/tests/contracts/test_seed_carry_contract.py b/tests/contracts/test_seed_carry_contract.py new file mode 100644 index 0000000..8aaf0d8 --- /dev/null +++ b/tests/contracts/test_seed_carry_contract.py @@ -0,0 +1,36 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Contracts for the detection seed-carry state and the prior-scan context.""" + +from datetime import UTC, datetime + +import pytest + +from adapt.contracts import ContractViolation, check_prior_scan, check_seed_carry + +pytestmark = pytest.mark.unit + + +@pytest.mark.parametrize("ages", [(), (0,), (0, 1, 2)]) +def test_seed_carry_accepts_tuples_of_non_negative_ints(ages): + check_seed_carry(ages) + + +@pytest.mark.parametrize("bad", [[0, 1], (0.5,), (-1,), (True,), None]) +def test_seed_carry_rejects_non_tuple_or_non_int_or_negative(bad): + with pytest.raises(ContractViolation): + check_seed_carry(bad) + + +def test_prior_scan_accepts_none(): + check_prior_scan(None) + + +def test_prior_scan_accepts_a_scan_context(): + check_prior_scan({"scan_time": datetime(2024, 1, 1, tzinfo=UTC)}) + + +def test_prior_scan_rejects_a_context_without_scan_time(): + with pytest.raises(ContractViolation): + check_prior_scan({}) diff --git a/tests/execution/nodes/test_detection_node_seed_carry.py b/tests/execution/nodes/test_detection_node_seed_carry.py new file mode 100644 index 0000000..dc88c7c --- /dev/null +++ b/tests/execution/nodes/test_detection_node_seed_carry.py @@ -0,0 +1,134 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""DetectModule hands the previous scan's advected labels to the segmenter. + +Same synthetic field as the segmenter tests: peak A at (5, 3) is a real +h-maxima seed; bump B at (5, 8) is too shallow to seed on its own. The prior +scan had two cells (A = label 1, B = label 2), so B's core is the one lost. +""" + +from datetime import UTC, datetime + +import numpy as np +import pytest +import xarray as xr + +from adapt.configuration.schemas.user import UserSegmenterConfig +from adapt.contracts import ContractViolation, check_prior_scan, check_seed_carry +from adapt.execution.nodes.detection import DetectModule + +pytestmark = pytest.mark.unit + +PEAK_A = (5, 3) +BUMP_B = (5, 8) +T0 = datetime(2024, 1, 1, 0, 0, tzinfo=UTC) + + +def _plateau_ds() -> xr.Dataset: + field = np.full((12, 12), 10.0, dtype=np.float32) + field[3:9, 2:10] = 40.0 + field[PEAK_A] = 48.0 + field[BUMP_B] = 43.0 + return xr.Dataset( + {"reflectivity": (("y", "x"), field)}, + coords={"y": np.arange(12), "x": np.arange(12)}, + ) + + +def _advected(*blocks: tuple[int, int]) -> np.ndarray: + out = np.zeros((12, 12), dtype=np.int32) + for lab, (r, c) in enumerate(blocks, start=1): + out[r - 1 : r + 2, c - 1 : c + 2] = lab + return out + + +def _projected_ds(advected: np.ndarray) -> xr.Dataset: + """A projected_ds whose frame_offset=1 slice is ``advected``.""" + stack = np.stack([np.zeros_like(advected), advected], axis=0) + return xr.Dataset( + {"cell_projections": (("frame_offset", "y", "x"), stack)}, + coords={"frame_offset": [0, 1], "y": np.arange(12), "x": np.arange(12)}, + ) + + +def _prior(advected: np.ndarray | None, ages: tuple[int, ...]) -> dict: + prior = {"scan_time": T0, "seed_carry": ages, "num_cells": len(ages)} + if advected is not None: + prior["projected_ds"] = _projected_ds(advected) + return prior + + +def _run(make_detection_config, prior, **segmenter_kw) -> dict: + kw = {"filter_by_size": False, "h_maxima": 5.0, "seed_carry": True, **segmenter_kw} + ctx = { + "grid_ds_2d": _plateau_ds(), + "grid_ds": None, + "prior_scan": prior, + "detection_config": make_detection_config( + threshold=35.0, segmenter=UserSegmenterConfig(**kw) + ), + } + return DetectModule().run(ctx) + + +def test_node_declares_prior_scan_input_and_seed_carry_output(): + assert "prior_scan" in DetectModule.inputs + assert "seed_carry" in DetectModule.outputs + assert DetectModule.input_contracts["prior_scan"] is check_prior_scan + assert DetectModule.output_contracts["seed_carry"] is check_seed_carry + + +def test_seed_carry_is_empty_with_the_flag_off(make_detection_config): + out = _run(make_detection_config, _prior(_advected(PEAK_A, BUMP_B), (0, 0)), seed_carry=False) + assert out["seed_carry"] == () + assert out["num_cells"] == 1 + + +def test_no_carry_when_prior_scan_is_none(make_detection_config): + out = _run(make_detection_config, None) + assert out["num_cells"] == 1 + assert out["seed_carry"] == (0,) + + +def test_no_carry_when_prior_has_no_motion_field(make_detection_config): + out = _run(make_detection_config, _prior(None, (0, 0))) + assert out["num_cells"] == 1 + assert out["seed_carry"] == (0,) + + +def test_lost_core_of_a_second_prior_cell_is_rescued(make_detection_config): + out = _run(make_detection_config, _prior(_advected(PEAK_A, BUMP_B), (0, 0))) + labels = out["segmented_ds"]["cell_labels"].values + assert out["num_cells"] == 2 + assert out["seed_carry"][labels[BUMP_B] - 1] == 1 + assert out["seed_carry"][labels[PEAK_A] - 1] == 0 + + +def test_single_prior_cell_never_splits_its_own_core(make_detection_config): + own_footprint_4px_off = (PEAK_A[0], PEAK_A[1] + 4) + out = _run(make_detection_config, _prior(_advected(own_footprint_4px_off), (0,))) + assert out["num_cells"] == 1 + + +def test_label_at_the_carry_bound_is_not_offered(make_detection_config): + out = _run( + make_detection_config, + _prior(_advected(PEAK_A, BUMP_B), (0, 2)), + seed_carry_max_frames=2, + ) + assert out["num_cells"] == 1 + + +def test_seed_carry_length_must_match_prior_cell_count(make_detection_config): + prior = _prior(_advected(PEAK_A, BUMP_B), (0, 0)) + prior["num_cells"] = 1 + with pytest.raises(ContractViolation): + _run(make_detection_config, prior) + + +def test_centroid_outside_a_hollow_footprint_is_snapped_onto_it(make_detection_config): + advected = _advected(PEAK_A, BUMP_B) + advected[BUMP_B] = 0 # B's footprint is a ring; its centroid is the empty centre + out = _run(make_detection_config, _prior(advected, (0, 0))) + assert out["num_cells"] == 2 diff --git a/tests/execution/nodes/test_tracking_node.py b/tests/execution/nodes/test_tracking_node.py index 00a96fa..57a42be 100644 --- a/tests/execution/nodes/test_tracking_node.py +++ b/tests/execution/nodes/test_tracking_node.py @@ -48,6 +48,13 @@ def __init__(self, results): def track(self, ds_projected, cell_stats_df, *, scan_id): return self._results.pop(0) + @staticmethod + def decisions(): + from adapt.contracts import TrackingDecisions, TrackingFrame + + frame = TrackingFrame(None, "FIRST_SCAN", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + return TrackingDecisions(frame, (), (), ()) + def _run_module(module, tracked_per_scan): module._tracker = _FakeTracker([(t, pd.DataFrame()) for t in tracked_per_scan]) diff --git a/tests/modules/detection/test_detection_config.py b/tests/modules/detection/test_detection_config.py index d0da62a..8d68e87 100644 --- a/tests/modules/detection/test_detection_config.py +++ b/tests/modules/detection/test_detection_config.py @@ -22,6 +22,9 @@ def _make() -> DetectionConfig: min_cellsize_gridpoint=5, max_cellsize_gridpoint=None, h_maxima=3.0, + seed_carry=False, + seed_carry_max_frames=2, + seed_carry_min_separation=3, reflectivity_var="reflectivity", labels_var="cell_labels", z_level=2000.0, diff --git a/tests/modules/detection/test_segmentation_decisions.py b/tests/modules/detection/test_segmentation_decisions.py new file mode 100644 index 0000000..9e3c8f1 --- /dev/null +++ b/tests/modules/detection/test_segmentation_decisions.py @@ -0,0 +1,115 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""The segmenter records every seed and carried claim and what became of it. + +Same field as the carried-seeds tests: one 48-px blob, peak A at (5, 3) and a +shallow bump B at (5, 8). +""" + +import numpy as np +import pytest +import xarray as xr + +from adapt.configuration.schemas.user import UserSegmenterConfig +from adapt.modules.detection.module import CarriedLabels, RadarCellSegmenter + +pytestmark = pytest.mark.unit + +PEAK_A = (5, 3) +BUMP_B = (5, 8) + + +def _plateau_ds() -> xr.Dataset: + field = np.full((12, 12), 10.0, dtype=np.float32) + field[3:9, 2:10] = 40.0 + field[PEAK_A] = 48.0 + field[BUMP_B] = 43.0 + return xr.Dataset( + {"reflectivity": (("y", "x"), field)}, + coords={"y": np.arange(12), "x": np.arange(12)}, + ) + + +def _carried(*blocks: tuple[int, int], ages=None) -> CarriedLabels: + raster = np.zeros((12, 12), dtype=np.int32) + for lab, (r, c) in enumerate(blocks, start=1): + raster[r - 1 : r + 2, c - 1 : c + 2] = lab + return CarriedLabels(labels=raster, ages=ages or (0,) * len(blocks)) + + +def _segmenter(make_detection_config, **kw) -> RadarCellSegmenter: + seg_kw = {"filter_by_size": False, "h_maxima": 5.0, "seed_carry": True, **kw} + return RadarCellSegmenter( + make_detection_config(threshold=35.0, segmenter=UserSegmenterConfig(**seg_kw)) + ) + + +def test_independent_seed_is_recorded_with_its_peak_and_basin(make_detection_config): + seg = _segmenter(make_detection_config) + seg.segment(_plateau_ds()) + d = seg.decisions() + (seed,) = d.seeds + assert (seed.origin, seed.marker_id, (seed.row, seed.col)) == ("HMAXIMA", 1, PEAK_A) + assert (seed.peak_value, seed.plateau_px, seed.component_id) == (48.0, 1, 1) + assert (seed.basin_px, seed.final_label) == (48, 1) + assert (d.frame.n_components, d.frame.n_independent_seeds, d.frame.n_cells) == (1, 1, 1) + assert d.frame.mask_px == 48 + + +def test_carried_claims_record_survivor_and_rescue(make_detection_config): + seg = _segmenter(make_detection_config) + seg.segment(_plateau_ds(), carried=_carried(PEAK_A, BUMP_B, ages=(0, 1))) + d = seg.decisions() + by_prior = {s.prior_label: s for s in d.seeds if s.origin == "CARRIED"} + assert by_prior[1].decision == "SURVIVOR" and by_prior[1].footprint_holds_seed is True + assert by_prior[2].decision == "SEEDED" and by_prior[2].marker_id == 2 + assert ( + by_prior[2].component_claims, + by_prior[2].component_seeds, + by_prior[2].component_deficit, + ) == (2, 1, 1) + assert by_prior[2].final_label == 2 and by_prior[2].basin_px > 0 + (component,) = d.components + assert ( + component.n_seeds, + component.n_claims, + component.n_admitted, + component.n_final_labels, + ) == (1, 2, 1, 2) + assert (d.frame.n_carried_claims, d.frame.n_carried_admitted) == (2, 1) + + +def test_expired_and_outside_claims_are_recorded_but_do_not_claim(make_detection_config): + seg = _segmenter(make_detection_config, seed_carry_max_frames=2) + seg.segment(_plateau_ds(), carried=_carried((1, 1), BUMP_B, ages=(0, 2))) + d = seg.decisions() + decisions = {s.prior_label: s.decision for s in d.seeds if s.origin == "CARRIED"} + assert decisions == {1: "OUTSIDE_MASK", 2: "EXPIRED"} + assert all(s.component_id is None for s in d.seeds if s.origin == "CARRIED") + assert d.frame.n_carried_claims == 0 + + +def test_own_footprint_without_deficit_is_recorded_as_no_deficit(make_detection_config): + seg = _segmenter(make_detection_config) + seg.segment(_plateau_ds(), carried=_carried((PEAK_A[0], PEAK_A[1] + 4))) + (claim,) = [s for s in seg.decisions().seeds if s.origin == "CARRIED"] + assert claim.decision == "NO_DEFICIT" and claim.marker_id is None + assert claim.clearance_px == pytest.approx(4.0) + + +def test_size_filter_drop_is_recorded_on_the_seed_and_the_frame(make_detection_config): + seg = _segmenter(make_detection_config, filter_by_size=True, min_cellsize_gridpoint=60) + seg.segment(_plateau_ds()) + d = seg.decisions() + (seed,) = d.seeds + assert (seed.basin_px, seed.final_label) == (48, 0) + assert (d.frame.n_basins, d.frame.n_dropped_small, d.frame.n_cells) == (1, 1, 0) + + +def test_empty_mask_yields_an_empty_record(make_detection_config): + seg = _segmenter(make_detection_config) + seg.segment(_plateau_ds().assign(reflectivity=lambda ds: ds.reflectivity * 0)) + d = seg.decisions() + assert d.seeds == () and d.components == () + assert (d.frame.mask_px, d.frame.n_cells) == (0, 0) diff --git a/tests/modules/detection/test_segmenter_carried_seeds.py b/tests/modules/detection/test_segmenter_carried_seeds.py new file mode 100644 index 0000000..9025454 --- /dev/null +++ b/tests/modules/detection/test_segmenter_carried_seeds.py @@ -0,0 +1,142 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Carried seeds: the previous frame's advected labels as extra watershed markers. + +The field below is one 35 dBZ blob with a strong peak A at (5, 3) and a +shallow bump B at (5, 8) that rises only 3 dBZ above the plateau — below the +h-maxima height of 5 — so on its own the operator finds one cell. + +A carried seed is admitted only when its connected component has *lost* a +core: with k independent seeds and n prior cells landing in the component, +at most n − k carried seeds are added, farthest from a seed first. So the +previous frame's own footprint, mispositioned by advection error, never +splits a healthy core; a second prior cell whose core dropped below h does +get rescued. +""" + +import numpy as np +import pytest +import xarray as xr + +from adapt.configuration.schemas.user import UserSegmenterConfig +from adapt.contracts import ContractViolation +from adapt.modules.detection.module import CarriedLabels, RadarCellSegmenter + +pytestmark = pytest.mark.unit + +PEAK_A = (5, 3) +BUMP_B = (5, 8) + + +def _plateau_ds() -> xr.Dataset: + field = np.full((12, 12), 10.0, dtype=np.float32) + field[3:9, 2:10] = 40.0 + field[PEAK_A] = 48.0 + field[BUMP_B] = 43.0 + return xr.Dataset( + {"reflectivity": (("y", "x"), field)}, + coords={"y": np.arange(12), "x": np.arange(12)}, + ) + + +def _raster(*blocks: tuple[int, int]) -> np.ndarray: + """Advected-label raster with one 3x3 footprint per block centre, labels 1..n.""" + out = np.zeros((12, 12), dtype=np.int32) + for lab, (r, c) in enumerate(blocks, start=1): + out[r - 1 : r + 2, c - 1 : c + 2] = lab + return out + + +def _carried(*blocks: tuple[int, int], ages: tuple[int, ...] | None = None) -> CarriedLabels: + return CarriedLabels(labels=_raster(*blocks), ages=ages or (0,) * len(blocks)) + + +def _segmenter(make_detection_config, **segmenter_kw) -> RadarCellSegmenter: + kw = {"filter_by_size": False, "h_maxima": 5.0, "seed_carry": True, **segmenter_kw} + return RadarCellSegmenter( + make_detection_config(threshold=35.0, segmenter=UserSegmenterConfig(**kw)) + ) + + +def test_labels_identical_with_no_carried_labels(make_detection_config): + seg = _segmenter(make_detection_config) + plain = seg.segment(_plateau_ds())["cell_labels"].values + none = seg.segment(_plateau_ds(), carried=None)["cell_labels"].values + assert np.array_equal(plain, none) + assert int(plain.max()) == 1 + + +def test_carry_age_is_all_zero_when_nothing_is_offered(make_detection_config): + out = _segmenter(make_detection_config).segment(_plateau_ds()) + assert tuple(out["cell_labels"].attrs["carry_age"]) == (0,) + + +def test_carry_age_is_absent_when_the_flag_is_off(make_detection_config): + out = _segmenter(make_detection_config, seed_carry=False).segment(_plateau_ds()) + assert "carry_age" not in out["cell_labels"].attrs + + +def test_carried_labels_with_the_flag_off_raise(make_detection_config): + seg = _segmenter(make_detection_config, seed_carry=False) + with pytest.raises(ContractViolation): + seg.segment(_plateau_ds(), carried=_carried(BUMP_B)) + + +def test_footprint_outside_the_mask_is_ignored(make_detection_config): + out = _segmenter(make_detection_config).segment(_plateau_ds(), carried=_carried((1, 1))) + assert int(out["cell_labels"].values.max()) == 1 + assert tuple(out["cell_labels"].attrs["carry_age"]) == (0,) + + +def test_own_footprint_mispositioned_by_advection_does_not_split_the_core(make_detection_config): + """One prior cell, one seed: nothing was lost, so nothing is added.""" + own_footprint_4px_off = (PEAK_A[0], PEAK_A[1] + 4) + out = _segmenter(make_detection_config).segment( + _plateau_ds(), carried=_carried(own_footprint_4px_off) + ) + assert int(out["cell_labels"].values.max()) == 1 + assert tuple(out["cell_labels"].attrs["carry_age"]) == (0,) + + +def test_second_prior_cell_in_a_one_seed_component_is_rescued(make_detection_config): + out = _segmenter(make_detection_config).segment( + _plateau_ds(), carried=_carried(PEAK_A, BUMP_B, ages=(0, 1)) + ) + labels = out["cell_labels"].values + ages = tuple(out["cell_labels"].attrs["carry_age"]) + + assert int(labels.max()) == 2 + assert labels[PEAK_A] != labels[BUMP_B] + assert ages[labels[BUMP_B] - 1] == 2 # carried one more frame + assert ages[labels[PEAK_A] - 1] == 0 # independently detected + + +def test_only_the_deficit_is_admitted_farthest_from_a_seed_first(make_detection_config): + """Two prior cells besides A's own, one seed → deficit 2, but the second + candidate sits within min_separation of the first admitted marker.""" + near_b = (BUMP_B[0] + 1, BUMP_B[1] - 1) + out = _segmenter(make_detection_config).segment( + _plateau_ds(), carried=_carried(PEAK_A, near_b, BUMP_B) + ) + labels = out["cell_labels"].values + assert int(labels.max()) == 2 + assert labels[BUMP_B] != labels[PEAK_A] # the farther candidate (B) was the one admitted + + +def test_prior_cell_at_the_carry_bound_is_not_offered(make_detection_config): + out = _segmenter(make_detection_config, seed_carry_max_frames=2).segment( + _plateau_ds(), carried=_carried(PEAK_A, BUMP_B, ages=(0, 2)) + ) + assert int(out["cell_labels"].values.max()) == 1 + + +def test_labels_stay_contiguous_and_size_ranked_with_carried_markers(make_detection_config): + labels = ( + _segmenter(make_detection_config) + .segment(_plateau_ds(), carried=_carried(PEAK_A, BUMP_B))["cell_labels"] + .values + ) + present = sorted(set(labels.flatten()) - {0}) + assert present == [1, 2] + assert (labels == 1).sum() >= (labels == 2).sum() diff --git a/tests/modules/detection/test_segmenter_pyart_conv_strat.py b/tests/modules/detection/test_segmenter_pyart_conv_strat.py index 0ffe83f..4daf61b 100644 --- a/tests/modules/detection/test_segmenter_pyart_conv_strat.py +++ b/tests/modules/detection/test_segmenter_pyart_conv_strat.py @@ -47,6 +47,9 @@ def _config(method: str) -> DetectionConfig: min_cellsize_gridpoint=3, max_cellsize_gridpoint=None, h_maxima=3.0, + seed_carry=False, + seed_carry_max_frames=2, + seed_carry_min_separation=3, reflectivity_var="reflectivity", labels_var="cell_labels", z_level=Z_LEVEL, diff --git a/tests/modules/detection/test_segmenter_size_ranking.py b/tests/modules/detection/test_segmenter_size_ranking.py new file mode 100644 index 0000000..35d0bbf --- /dev/null +++ b/tests/modules/detection/test_segmenter_size_ranking.py @@ -0,0 +1,45 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Size ranking must not depend on label values being contiguous from 0.""" + +import numpy as np +import pytest +import xarray as xr + +from adapt.configuration.schemas.user import UserSegmenterConfig +from adapt.modules.detection.module import RadarCellSegmenter + +pytestmark = pytest.mark.unit + + +def _all_convective_two_dome_ds() -> xr.Dataset: + """12x12 field entirely above threshold with two domes of different size. + + Base 40 dBZ; a tall narrow dome at (3, 3) and a broader one at (8, 8). + With threshold 35 the convective mask covers every pixel, so the label + image contains no background (no 0), which is the case a positional + label lookup gets wrong. + """ + yy, xx = np.mgrid[0:12, 0:12] + field = ( + 40.0 + + 18.0 * np.exp(-((yy - 3) ** 2 + (xx - 3) ** 2) / 4.0) + + 14.0 * np.exp(-((yy - 8) ** 2 + (xx - 8) ** 2) / 12.0) + ).astype(np.float32) + return xr.Dataset( + {"reflectivity": (("y", "x"), field)}, + coords={"y": np.arange(12), "x": np.arange(12)}, + ) + + +def test_size_ranking_when_every_pixel_is_convective(make_detection_config): + """Both domes are labelled, nothing is dropped, and the larger one is label 1.""" + config = make_detection_config( + threshold=35.0, segmenter=UserSegmenterConfig(filter_by_size=False) + ) + labels = RadarCellSegmenter(config).segment(_all_convective_two_dome_ds())["cell_labels"].values + + assert int(labels.max()) == 2 + assert int((labels == 0).sum()) == 0 + assert (labels == 1).sum() >= (labels == 2).sum() diff --git a/tests/modules/tracking/test_decision_log.py b/tests/modules/tracking/test_decision_log.py new file mode 100644 index 0000000..b5c5462 --- /dev/null +++ b/tests/modules/tracking/test_decision_log.py @@ -0,0 +1,154 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""The tracker records every candidate it considered and why each cell ended. + +Synthetic 8x8 scans (1000 m pixels) with analytically known gate outcomes. +""" + +import numpy as np +import pytest + +from adapt.modules.tracking.module import CellTracker +from tests.modules.tracking.test_tracker_quality import ( + _cell_stats, + _make_config, + _one_cell_scan, + _synthetic_ds, +) + +pytestmark = pytest.mark.unit + +T0 = np.datetime64("2024-01-01T12:00:00") +T1 = np.datetime64("2024-01-01T12:05:00") + + +def _cells_scan(time, spans: dict[int, tuple[int, int]], proj_labels=None): + """Cells as {label: (col_start, col_stop)} on rows 2-3 of an 8x8 grid.""" + labels = np.zeros((8, 8), dtype=np.int32) + rows = [] + for lab, (start, stop) in spans.items(): + labels[2:4, start:stop] = lab + rows.append( + { + "id": lab, + "area": float(2 * (stop - start)), + "cx": (start + stop - 1) / 2 * 1000.0, + "cy": 2500.0, + "mean_refl": 40.0, + "max_refl": 45.0, + } + ) + return _synthetic_ds(time, labels, proj_labels=proj_labels), _cell_stats(time, rows) + + +def test_first_scan_is_a_frame_with_no_pairs_and_born_cells(): + tracker = CellTracker(_make_config()) + tracker.track(*_one_cell_scan(T0, 2), scan_id="scan0") + d = tracker.decisions() + assert d.frame.reset_code == "FIRST_SCAN" + assert (d.frame.n_prev, d.frame.n_curr, d.frame.n_pairs) == (0, 1, 0) + assert [u.reason for u in d.unmatched] == ["FIRST_SCAN"] + + +def test_perfect_continuation_is_one_propagated_pair(): + tracker = CellTracker(_make_config()) + tracker.track(*_one_cell_scan(T0, 2), scan_id="scan0") + tracker.track(*_one_cell_scan(T1, 3), scan_id="scan1") + d = tracker.decisions() + (pair,) = d.candidates + assert pair.outcome == "CONTINUE" + assert pair.match_method == "PROPAGATED" + assert pair.last_stage == "ASSIGNMENT" + assert pair.opc == pair.ocp == 1.0 + assert pair.cost_rank == 1 + assert pair.speed_ms == pytest.approx(1000.0 / 300.0) + assert d.frame.n_propagated == 1 and d.frame.n_termination == 0 + assert d.unmatched == () + + +def test_overlap_rejection_is_recorded_with_its_numbers(): + """Hull left at x=2, cell at x=4: the 1 km buffer generates the pair, the + un-buffered hull has zero intersection, so the overlap gate rejects it.""" + tracker = CellTracker(_make_config()) + ds0, stats0 = _one_cell_scan(T0, 2) + tracker.track(ds0, stats0, scan_id="scan0") + tracker.track(*_one_cell_scan(T1, 4, proj_labels=ds0["cell_labels"].values), scan_id="scan1") + d = tracker.decisions() + (pair,) = d.candidates + assert (pair.outcome, pair.last_stage) == ("REJECTED_OVERLAP", "OVERLAP") + assert pair.intersection_px == 0 and pair.opc == 0.0 + assert pair.cost is None + prev, curr = d.unmatched + assert (prev.side, prev.reason, prev.best_candidate_label) == ( + "prev", + "ALL_REJECTED_OVERLAP", + 1, + ) + assert (curr.side, curr.reason) == ("curr", "ALL_REJECTED_OVERLAP") + assert d.frame.n_termination == 1 and d.frame.n_initiation == 1 + + +def test_kinematic_rejection_records_speed_and_code(): + tracker = CellTracker(_make_config(max_speed_ms=5.0)) + tracker.track(*_one_cell_scan(T0, 2), scan_id="scan0") + tracker.track(*_one_cell_scan(T1, 6), scan_id="scan1") # 13.3 m/s > 5 + (pair,) = tracker.decisions().candidates + assert pair.overlap_passed is True + assert (pair.outcome, pair.kinematic_code) == ("REJECTED_KINEMATIC", "VELOCITY_EXCEEDED") + assert pair.speed_ms == pytest.approx(4000.0 / 300.0) + assert tracker.decisions().unmatched[0].reason == "ALL_REJECTED_KINEMATIC" + + +def test_ambiguous_component_records_hungarian_ranks_and_the_loser(): + """A (cols 1-3) and B (cols 4-6) vs X (cols 2-4) and Y (cols 5-7): X overlaps + both, so nothing is forced and one 2x2 component goes to Hungarian.""" + tracker = CellTracker(_make_config()) + ds0, stats0 = _cells_scan(T0, {1: (1, 4), 2: (4, 7)}) + tracker.track(ds0, stats0, scan_id="scan0") + tracker.track( + *_cells_scan(T1, {1: (2, 5), 2: (5, 8)}, proj_labels=ds0["cell_labels"].values), + scan_id="scan1", + ) + d = tracker.decisions() + by_pair = {(c.prev_cell_label, c.curr_cell_label): c for c in d.candidates} + assert {(1, 1), (2, 1), (2, 2)} <= set(by_pair) + assert by_pair[(1, 1)].outcome == "CONTINUE" and by_pair[(2, 2)].outcome == "CONTINUE" + assert by_pair[(2, 1)].outcome == "LOST_ASSIGNMENT" + # A's buffered hull may graze Y: that pair exists only as an overlap rejection. + others = set(by_pair) - {(1, 1), (2, 1), (2, 2)} + assert all(by_pair[p].outcome == "REJECTED_OVERLAP" for p in others) + assert by_pair[(2, 1)].cost_rank == 2 and by_pair[(2, 2)].cost_rank == 1 + assert {c.match_method for c in d.candidates if c.outcome == "CONTINUE"} == {"HUNGARIAN"} + assert by_pair[(1, 1)].component_n_prev == 2 and by_pair[(1, 1)].component_n_curr == 2 + assert d.frame.n_hungarian == 2 + + +def test_split_test_is_recorded_pass_or_fail(): + """A 10-px parent splits into a 4-px and a 6-px piece. The parent continues + into the larger piece (cheaper); the born 4-px piece covers 0.4 of the + parent's hull, exactly split_overlap, and is recorded as a passing test.""" + tracker = CellTracker(_make_config()) + ds0, stats0 = _cells_scan(T0, {1: (1, 6)}) + tracker.track(ds0, stats0, scan_id="scan0") + tracker.track( + *_cells_scan(T1, {1: (1, 3), 2: (3, 6)}, proj_labels=ds0["cell_labels"].values), + scan_id="scan1", + ) + d = tracker.decisions() + tests = {(t.kind, t.tested_cell_label): t for t in d.split_merge_tests} + born = [u for u in d.unmatched if u.side == "curr"] + assert d.frame.n_split == 1 and born[0].reason == "SPLIT_CHILD" + assert tests[("SPLIT", born[0].cell_label)].passed is True + assert tests[("SPLIT", born[0].cell_label)].threshold == 0.4 + + +def test_gap_reset_marks_every_cell_reset(): + tracker = CellTracker(_make_config(max_tracking_gap_minutes=10.0)) + tracker.track(*_one_cell_scan(T0, 2), scan_id="scan0") + tracker.track(*_one_cell_scan(np.datetime64("2024-01-01T12:20:00"), 2), scan_id="scan1") + d = tracker.decisions() + assert d.frame.reset_code == "TRACK_GAP_EXCEEDED" + assert d.candidates == () + assert {u.reason for u in d.unmatched} == {"RESET"} + assert d.frame.n_termination == 1 and d.frame.n_initiation == 1 diff --git a/tests/modules/tracking/test_motion.py b/tests/modules/tracking/test_motion.py new file mode 100644 index 0000000..b1d7ba3 --- /dev/null +++ b/tests/modules/tracking/test_motion.py @@ -0,0 +1,38 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Kinematic gate: absolute speed cap and a floored acceleration cap.""" + +import pytest + +from adapt.modules.tracking.models import TrackingError +from adapt.modules.tracking.motion import MotionValidator + +pytestmark = pytest.mark.unit + +DT_S = 270.0 + + +def _check(validator: MotionValidator, speed_ms: float, previous_speed: float | None): + return validator.check(0.0, 0.0, speed_ms * DT_S, 0.0, DT_S, previous_speed) + + +def test_floor_admits_a_step_above_the_relative_cap(): + decision = _check(MotionValidator(40.0, 3.0, 20.0), 13.3, previous_speed=3.8) + assert decision.ok # 13.3 > 3 x 3.8 = 11.4 but <= floor 20 + + +def test_floor_does_not_admit_a_step_above_itself(): + decision = _check(MotionValidator(40.0, 3.0, 20.0), 25.0, previous_speed=3.8) + assert decision.code is TrackingError.ACCELERATION_EXCEEDED + + +def test_relative_cap_still_applies_above_the_floor(): + validator = MotionValidator(40.0, 3.0, 20.0) # previous 10 m/s -> cap 30 + assert _check(validator, 25.0, previous_speed=10.0).ok + assert _check(validator, 35.0, previous_speed=10.0).code is TrackingError.ACCELERATION_EXCEEDED + + +def test_absolute_cap_is_independent_of_the_floor(): + decision = _check(MotionValidator(40.0, 3.0, 100.0), 45.0, previous_speed=None) + assert decision.code is TrackingError.VELOCITY_EXCEEDED diff --git a/tests/modules/tracking/test_tracker_quality.py b/tests/modules/tracking/test_tracker_quality.py index 81642ee..76df2cf 100644 --- a/tests/modules/tracking/test_tracker_quality.py +++ b/tests/modules/tracking/test_tracker_quality.py @@ -21,7 +21,6 @@ from adapt.configuration.schemas.user import UserConfig from adapt.execution.nodes.tracking import TrackingModule from adapt.modules.tracking.module import CellTracker -from adapt.modules.tracking.projection import select_registration_labels pytestmark = pytest.mark.unit @@ -194,6 +193,7 @@ def test_acceleration_exceeded_rejects_match(): cfg = _make_config( max_speed_ms=40.0, max_speed_multiplier=2.0, + acceleration_floor_ms=5.0, max_tracking_gap_minutes=60.0, ) tracker = CellTracker(cfg) @@ -213,6 +213,61 @@ def test_acceleration_exceeded_rejects_match(): assert (events2["event_type"] == "CONTINUE").sum() == 0, "accelerating step must be rejected" +def test_acceleration_floor_admits_storm_motion_after_a_slow_step(): + """A slow prior step (centroid jitter) must not cap the next step below + plausible storm motion: the cap never falls below the floor.""" + cfg = _make_config(max_speed_multiplier=2.0, acceleration_floor_ms=12.0) + tracker = CellTracker(cfg) + t0, t1, t2 = (np.datetime64(f"2024-01-01T12:{m:02d}:00") for m in (0, 5, 10)) + + ds0, stats0 = _one_cell_scan(t0, 2) + tracker.track(ds0, stats0, scan_id="site032scan") + ds1, stats1 = _one_cell_scan(t1, 3) # 3.33 m/s → 2x cap 6.67, floor 12 wins + tracker.track(ds1, stats1, scan_id="site033scan") + ds2, stats2 = _one_cell_scan(t2, 6) # 10 m/s + _, events2 = tracker.track(ds2, stats2, scan_id="site034scan") + assert (events2["event_type"] == "CONTINUE").sum() == 1 + + +def _one_cell_scan_wide(time, x_pix): + """Like ``_one_cell_scan`` on a 16-column grid, for multi-step motion.""" + labels = np.zeros((8, 16), dtype=np.int32) + labels[2:4, x_pix : x_pix + 2] = 1 + stats = _cell_stats( + time, + [ + { + "id": 1, + "area": 4.0, + "cx": (x_pix + 0.5) * 1000.0, + "cy": 2500.0, + "mean_refl": 40.0, + "max_refl": 45.0, + } + ], + ) + return _synthetic_ds(time, labels), stats + + +def test_acceleration_reference_is_the_tracks_mean_speed_not_its_last_step(): + """Steps of 13.3, 13.3 then 3.3 m/s give a mean of 10 m/s; a 13.3 m/s step + is within 2x the mean although it is 4x the last step.""" + cfg = _make_config(max_speed_multiplier=2.0, acceleration_floor_ms=5.0) + tracker = CellTracker(cfg) + t = [ + np.datetime64("2024-01-01T12:00:00") + np.timedelta64(s, "s") + for s in (0, 150, 300, 600, 750) + ] + + for i, x in enumerate((1, 3, 5, 6)): # +2 px/150 s, +2 px/150 s, +1 px/300 s + ds, stats = _one_cell_scan_wide(t[i], x) + _, events = tracker.track(ds, stats, scan_id=f"site04{i}scan") + assert (events["event_type"] == "TERMINATION").sum() == 0 + ds, stats = _one_cell_scan_wide(t[4], 8) # +2 px/150 s = 13.3 m/s again + _, events = tracker.track(ds, stats, scan_id="site044scan") + assert (events["event_type"] == "CONTINUE").sum() == 1 + + # --------------------------------------------------------------------------- # B4 — deterministic constraint propagation (no optimisation) # --------------------------------------------------------------------------- @@ -373,26 +428,35 @@ def test_heading_penalty_breaks_ambiguous_match_toward_consistent_track(): # --------------------------------------------------------------------------- -def test_registration_selects_nearest_minute(): - """The minute frame closest to the real gap is chosen for the hull.""" - f1 = np.full((4, 4), 11, dtype=np.int32) - f2 = np.full((4, 4), 22, dtype=np.int32) - f3 = np.full((4, 4), 33, dtype=np.int32) - ds = xr.Dataset( - { - "registration_minutes": (["minute", "y", "x"], np.stack([f1, f2, f3])), - "cell_projections": (["frame_offset", "y", "x"], np.zeros((1, 4, 4), dtype=np.int32)), - }, - coords={"minute": [1, 2, 3]}, - ) - out = select_registration_labels(ds, dt_s=130.0) # 2.17 min → nearest minute 2 - assert int(out[0, 0]) == 22 +def test_matching_uses_the_hull_registered_to_the_current_scan(): + """The registration hull is the previous labels advected by the full flow + step (``cell_projections[0]``, fraction 1.0 of the gap) — never an + under-advected minute frame. + The minute frames carry a datetime64 ``minute`` coord and an + ``interpolation_fraction`` coord exactly as the projection module writes + them; the frame nearest t_prev sits 3 px short of the cell. + """ + cfg = _make_config(max_tracking_gap_minutes=60.0) + tracker = CellTracker(cfg) + t0, t1 = np.datetime64("2024-01-01T12:00:00"), np.datetime64("2024-01-01T12:04:30") -def test_registration_falls_back_to_cell_projections(): - """Without minute frames the whole-step cell_projections[0] is used.""" - ds = xr.Dataset( - {"cell_projections": (["frame_offset", "y", "x"], np.full((1, 4, 4), 7, dtype=np.int32))} + ds0, stats0 = _one_cell_scan(t0, 2) + tracker.track(ds0, stats0, scan_id="site030scan") + + ds1, stats1 = _one_cell_scan(t1, 6) # cell_projections[0]: the cell exactly at x=6 + frames = [] + for x in (3, 4, 5, 5): + frame = np.zeros((8, 8), dtype=np.int32) + frame[2:4, x : x + 2] = 1 + frames.append(frame) + minutes = np.arange(np.datetime64("2024-01-01T12:01:00"), t1, np.timedelta64(1, "m")).astype( + "datetime64[ns]" ) - out = select_registration_labels(ds, dt_s=300.0) - assert int(out[0, 0]) == 7 + fractions = ((minutes - t0) / (t1 - t0)).astype(np.float32) + ds1["registration_minutes"] = (("minute", "y", "x"), np.stack(frames)) + ds1 = ds1.assign_coords(minute=minutes, interpolation_fraction=("minute", fractions)) + + _, events1 = tracker.track(ds1, stats1, scan_id="site031scan") + assert (events1["event_type"] == "CONTINUE").sum() == 1 + assert (events1["event_type"] == "TERMINATION").sum() == 0 diff --git a/tests/persistence/test_collection_provisioning.py b/tests/persistence/test_collection_provisioning.py index 2362c3b..b09b24b 100644 --- a/tests/persistence/test_collection_provisioning.py +++ b/tests/persistence/test_collection_provisioning.py @@ -41,7 +41,12 @@ def test_creates_exact_collection_layout(self, store): store.close() cdir = store.root / "collections" / "KILX" - assert {p.name for p in cdir.iterdir()} == {"catalog.db", "products.db", "objects"} + assert {p.name for p in cdir.iterdir()} == { + "catalog.db", + "products.db", + "decisions.db", + "objects", + } assert (cdir / "objects").is_dir() def test_catalog_has_discovery_tables_only(self, store): @@ -65,6 +70,20 @@ def test_products_has_static_tables_only(self, store): "annotations", } + def test_decisions_has_the_decision_tables_only(self, store): + store.collection("KILX") + store.close() + + assert _tables(store.root / "collections" / "KILX" / "decisions.db") == { + "tracking_frames", + "tracking_candidates", + "tracking_unmatched", + "tracking_split_merge_tests", + "segmentation_frames", + "segmentation_seeds", + "segmentation_components", + } + def test_second_open_reuses_layout(self, store): first = store.collection("KILX") second = store.collection("KILX") @@ -72,7 +91,12 @@ def test_second_open_reuses_layout(self, store): assert second is first store.close() cdir = store.root / "collections" / "KILX" - assert {p.name for p in cdir.iterdir()} == {"catalog.db", "products.db", "objects"} + assert {p.name for p in cdir.iterdir()} == { + "catalog.db", + "products.db", + "decisions.db", + "objects", + } def test_collection_exposes_paths_and_catalog(self, store): coll = store.collection("KILX") @@ -80,6 +104,7 @@ def test_collection_exposes_paths_and_catalog(self, store): assert coll.collection_id == "KILX" assert coll.objects_dir == store.root / "collections" / "KILX" / "objects" assert coll.products_path == store.root / "collections" / "KILX" / "products.db" + assert coll.decisions_path == store.root / "collections" / "KILX" / "decisions.db" assert coll.catalog is not None diff --git a/tests/persistence/test_decision_store.py b/tests/persistence/test_decision_store.py new file mode 100644 index 0000000..505b2cc --- /dev/null +++ b/tests/persistence/test_decision_store.py @@ -0,0 +1,177 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""decisions.db: bundles land in their tables with run/scan stamps, replace on re-write, +and the router dispatches ``DecisionTableWrite`` there without touching products.""" + +import sqlite3 +from datetime import UTC, datetime + +import pytest + +from adapt.contracts import ( + DecisionTableWrite, + PersistenceMeta, + SegmentationComponent, + SegmentationDecisions, + SegmentationFrame, + SegmentationSeed, + TrackingCandidate, + TrackingDecisions, + TrackingFrame, + TrackingUnmatched, +) +from adapt.persistence.decision_store import DecisionStore +from adapt.persistence.output_router import StoreOutputRouter +from adapt.persistence.store import Store, init_store + +pytestmark = pytest.mark.unit + +SCAN_TIME = datetime(2026, 8, 13, 17, 31, 58, tzinfo=UTC) +META = PersistenceMeta( + scan_time=SCAN_TIME, scan_id="abc123", run_id="run-1", source_file="f", collection_id="KILX" +) + + +def _tracking() -> TrackingDecisions: + frame = TrackingFrame(270.0, None, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0) + candidate = TrackingCandidate( + prev_cell_uid="AAAA", + prev_cell_label=1, + curr_cell_label=1, + track_steps_before=0, + hull_area_px=4, + hull_centroid_x=2500.0, + hull_centroid_y=2500.0, + curr_area_px=4, + curr_centroid_x=3500.0, + curr_centroid_y=2500.0, + intersection_px=2, + opc=0.5, + ocp=0.5, + overlap_passed=True, + speed_ms=3.7, + previous_speed_ms=None, + accel_cap_ms=None, + kinematic_code=None, + displacement_m=1000.0, + length_scale_m=2256.0, + heading_change_deg=None, + cost=0.94, + component_n_prev=1, + component_n_curr=1, + cost_rank=1, + match_method="PROPAGATED", + last_stage="ASSIGNMENT", + outcome="CONTINUE", + ) + return TrackingDecisions(frame, (candidate,), (), ()) + + +def _segmentation() -> SegmentationDecisions: + frame = SegmentationFrame(10, 10, 1, 1, 1, 0, 1, 0, 0, 1) + seeds = ( + SegmentationSeed(1, "HMAXIMA", 5, 3, 48.0, 1, 1, *([None] * 8), "SEEDED", 10, 1), + SegmentationSeed( + None, + "CARRIED", + 5, + 7, + None, + None, + 1, + 1, + 0, + 9, + 4.0, + False, + 1, + 1, + 0, + "NO_DEFICIT", + None, + None, + ), + ) + components = (SegmentationComponent(1, 10, 48.0, 40.0, 1, 1, 0, 0, 1),) + return SegmentationDecisions(frame, seeds, components) + + +@pytest.fixture +def collection(tmp_path): + store = Store.open(init_store(tmp_path / "store")) + coll = store.collection("KILX") + yield coll + store.close() + + +def _rows(path, sql): + conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + try: + return [dict(r) for r in conn.execute(sql)] + finally: + conn.close() + + +def test_tracking_bundle_lands_in_its_tables_with_stamps(collection): + with DecisionStore(collection.decisions_path) as store: + store.write(_tracking(), META) + (frame,) = _rows(collection.decisions_path, "select * from tracking_frames") + (cand,) = _rows(collection.decisions_path, "select * from tracking_candidates") + assert (frame["run_id"], frame["scan_id"], frame["n_prev"]) == ("run-1", "abc123", 1) + assert frame["scan_time"].startswith("2026-08-13T17:31:58") + assert (cand["prev_cell_uid"], cand["outcome"], cand["overlap_passed"]) == ( + "AAAA", + "CONTINUE", + 1, + ) + assert cand["previous_speed_ms"] is None + + +def test_segmentation_bundle_numbers_seeds_in_order(collection): + with DecisionStore(collection.decisions_path) as store: + store.write(_segmentation(), META) + seeds = _rows(collection.decisions_path, "select * from segmentation_seeds order by seq") + assert [s["seq"] for s in seeds] == [0, 1] + assert [s["decision"] for s in seeds] == ["SEEDED", "NO_DEFICIT"] + assert seeds[1]["footprint_holds_seed"] == 0 and seeds[1]["final_label"] is None + + +def test_rewriting_a_scan_replaces_its_rows(collection): + with DecisionStore(collection.decisions_path) as store: + store.write(_tracking(), META) + store.write(_tracking(), META) + assert len(_rows(collection.decisions_path, "select * from tracking_candidates")) == 1 + + +def test_router_dispatches_decision_writes_without_a_scan_product(collection): + class _Module: + name = "tracking" + persistence = (DecisionTableWrite(key="tracking_decisions"),) + + StoreOutputRouter(collection).persist([_Module()], {"tracking_decisions": _tracking()}, META) + assert len(_rows(collection.decisions_path, "select * from tracking_frames")) == 1 + assert _rows(collection.products_path, "select * from table_schemas") == [] + + +def test_router_skips_when_the_key_is_absent(collection): + class _Module: + name = "tracking" + persistence = (DecisionTableWrite(key="tracking_decisions"),) + + StoreOutputRouter(collection).persist([_Module()], {}, META) + assert _rows(collection.decisions_path, "select * from tracking_frames") == [] + + +def test_unmatched_rows_keep_their_reason(collection): + unmatched = (TrackingUnmatched("prev", "AAAA", 1, 0, None, None, "NO_CANDIDATE"),) + bundle = TrackingDecisions(_tracking().frame, _tracking().candidates, unmatched, ()) + with DecisionStore(collection.decisions_path) as store: + store.write(bundle, META) + (row,) = _rows(collection.decisions_path, "select * from tracking_unmatched") + assert (row["side"], row["reason"], row["best_candidate_label"]) == ( + "prev", + "NO_CANDIDATE", + None, + ) diff --git a/tests/runtime/test_processor_prior_scan.py b/tests/runtime/test_processor_prior_scan.py new file mode 100644 index 0000000..2c92f61 --- /dev/null +++ b/tests/runtime/test_processor_prior_scan.py @@ -0,0 +1,98 @@ +"""The processor seeds ``prior_scan`` and keeps injected keys out of history. + +Executors are patched to return the full context plus their outputs, exactly +as ``GraphExecutor.run`` does, so the history-retention test exercises the +real merge path. +""" + +from datetime import UTC, datetime, timedelta + +import numpy as np +import pandas as pd +import pytest +import xarray as xr + +from tests.helpers.queue_msg import msg as _msg + +pytestmark = [pytest.mark.unit, pytest.mark.pipeline] + +T1 = datetime(2024, 5, 18, 12, 0, 0, tzinfo=UTC) +T2 = T1 + timedelta(minutes=5) + + +def _fake_ds(): + return xr.Dataset( + { + "reflectivity": (("y", "x"), np.ones((4, 4))), + "cell_labels": (("y", "x"), np.zeros((4, 4), dtype=int)), + }, + coords={"x": np.arange(4), "y": np.arange(4)}, + attrs={"z_level_m": 2000}, + ) + + +def _wire(proc, monkeypatch, seen: list): + def _single(ctx): + seen.append(ctx) + return { + **ctx, + "grid_ds": _fake_ds(), + "grid_ds_2d": _fake_ds(), + "segmented_ds": _fake_ds(), + "num_cells": 0, + "seed_carry": (), + } + + def _multi(ctx): + return { + **ctx, + "projected_ds": _fake_ds(), + "cell_stats": pd.DataFrame(), + "cell_adjacency": pd.DataFrame(), + } + + monkeypatch.setattr(proc._executors[1], "run", _single) + monkeypatch.setattr(proc._executors[2], "run", _multi) + monkeypatch.setattr(proc._router, "persist", lambda modules, result, meta: None) + + +def test_prior_scan_is_none_first_then_the_previous_entry( + tmp_path, monkeypatch, make_processor, store_env +): + proc = make_processor() + seen: list = [] + _wire(proc, monkeypatch, seen) + + assert proc.process_file(_msg(store_env, tmp_path, "f1", scan_time=T1)) is True + assert proc.process_file(_msg(store_env, tmp_path, "f2", scan_time=T2)) is True + + assert seen[0]["prior_scan"] is None + assert seen[1]["prior_scan"]["scan_time"] == T1 + + +def test_prior_scan_is_none_when_the_time_gap_is_too_large( + tmp_path, monkeypatch, make_processor, store_env +): + proc = make_processor() + seen: list = [] + _wire(proc, monkeypatch, seen) + too_late = T1 + timedelta(minutes=proc._max_time_gap_minutes + 1) + + proc.process_file(_msg(store_env, tmp_path, "f1", scan_time=T1)) + proc.process_file(_msg(store_env, tmp_path, "f2", scan_time=too_late)) + + assert seen[1]["prior_scan"] is None + + +def test_history_entries_do_not_retain_injected_context_keys( + tmp_path, monkeypatch, make_processor, store_env +): + proc = make_processor() + _wire(proc, monkeypatch, []) + + proc.process_file(_msg(store_env, tmp_path, "f1", scan_time=T1)) + proc.process_file(_msg(store_env, tmp_path, "f2", scan_time=T2)) + + latest = proc._scan_history[-1] + assert "prior_scan" not in latest + assert "scan_history" not in latest diff --git a/tests/stories/test_seed_carry_story.py b/tests/stories/test_seed_carry_story.py new file mode 100644 index 0000000..35c3666 --- /dev/null +++ b/tests/stories/test_seed_carry_story.py @@ -0,0 +1,81 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Story: a core that loses prominence for one frame survives with seed_carry. + +Given two convective cores, a strong one (A) fixed in place and a weaker one +(B) approaching it over three scans, until at scan 3 the field between them +rises so far that B is no longer an h-maximum of its own. + +When the pipeline segments scan 3 with the flag off, B is absorbed into A. +When the flag is on, B's labels from scan 2 — advected by the projection +module — seed the watershed at scan 3 and B is kept as its own cell. +""" + +from datetime import UTC, datetime, timedelta + +import numpy as np +import pytest +import xarray as xr + +from adapt.configuration.schemas.user import UserSegmenterConfig +from adapt.execution.nodes.detection import DetectModule +from adapt.execution.nodes.projection import ProjectionModule + +pytestmark = pytest.mark.unit + +T0 = datetime(2024, 6, 17, 0, 0, tzinfo=UTC) +A_COL = 6 +B_COLS = (22, 16, 12) # B walks toward A; at 12 it is no longer a separate h-maximum + + +def _frame(b_col: int, t: datetime) -> xr.Dataset: + yy, xx = np.mgrid[0:21, 0:30] + field = ( + 30.0 + + 25.0 * np.exp(-((yy - 10) ** 2 + (xx - A_COL) ** 2) / 8.0) + + 15.0 * np.exp(-((yy - 10) ** 2 + (xx - b_col) ** 2) / 18.0) + ).astype(np.float32) + time = np.datetime64(t.replace(tzinfo=None)) + return xr.Dataset( + {"reflectivity": (("y", "x"), field)}, + coords={"y": np.arange(21), "x": np.arange(30), "time": time}, + ) + + +def _cells_per_scan(make_detection_config, make_projection_config, seed_carry: bool) -> list[int]: + detect, project = DetectModule(), ProjectionModule() + detection_config = make_detection_config( + threshold=35.0, + segmenter=UserSegmenterConfig(seed_carry=seed_carry, filter_by_size=False, h_maxima=5.0), + ) + projection_config = make_projection_config() + + completed: list[dict] = [] + counts: list[int] = [] + for i, b_col in enumerate(B_COLS): + t = T0 + timedelta(minutes=5 * i) + ctx = { + "grid_ds_2d": _frame(b_col, t), + "grid_ds": None, + "scan_time": t, + "prior_scan": completed[-1] if completed else None, + "detection_config": detection_config, + "projection_config": projection_config, + } + result = detect.run(ctx) + entry = {**result, "scan_time": t} + if completed: + history = [completed[-1], entry] + entry.update(project.run({**ctx, **result, "scan_history": history})) + completed.append(entry) + counts.append(result["num_cells"]) + return counts + + +def test_weak_core_is_absorbed_with_the_flag_off(make_detection_config, make_projection_config): + assert _cells_per_scan(make_detection_config, make_projection_config, False) == [2, 2, 1] + + +def test_weak_core_survives_with_the_flag_on(make_detection_config, make_projection_config): + assert _cells_per_scan(make_detection_config, make_projection_config, True) == [2, 2, 2] diff --git a/tests/test_architecture.py b/tests/test_architecture.py index d24fe1f..284935c 100644 --- a/tests/test_architecture.py +++ b/tests/test_architecture.py @@ -305,6 +305,7 @@ def test_heavy_dependency_stays_in_its_component(dep: str) -> None: { "nexrad_file", # queued file path from the source "scan_history", # rolling window of prior segmented scans + "prior_scan", # previous completed scan context; processor seeds it (None when unusable) "grid_ds_3d", # full 3D grid sliced in by the processor "run_id", # repository run identifier "scan_id", # sha256[:16] of raw bytes; processor seeds it every scan (uid-v2 birth input) diff --git a/tests/unit/test_base_module_persistence.py b/tests/unit/test_base_module_persistence.py index d914d38..ed1961e 100644 --- a/tests/unit/test_base_module_persistence.py +++ b/tests/unit/test_base_module_persistence.py @@ -26,14 +26,21 @@ def test_default_persistence_is_empty(self): assert _NoPersistenceModule.persistence == () def test_pure_compute_modules_declare_no_persistence(self): - from adapt.execution.nodes.detection import DetectModule from adapt.execution.nodes.projection import ProjectionModule - assert DetectModule.persistence == () assert ProjectionModule.persistence == () + def test_decision_logging_modules_declare_only_decision_writes(self): + """Detection writes no product; its only persistence is the decision log.""" + from adapt.contracts import DecisionTableWrite + from adapt.execution.nodes.detection import DetectModule + + assert {type(s) for s in DetectModule.persistence} == {DecisionTableWrite} + assert DetectModule.persistence[0].key == "detection_decisions" + def test_persisting_modules_declare_specs(self): from adapt.contracts import ( + DecisionTableWrite, NetcdfArtifact, ProductTableWrite, TrackTablesWrite, @@ -50,5 +57,9 @@ def test_persisting_modules_declare_specs(self): assert stats.primary_key == ("run_id", "scan_id", "cell_label") assert adjacency.table == "cell_adjacency" assert adjacency.primary_key == ("run_id", "scan_id", "cell_label_a", "cell_label_b") - assert {type(s) for s in TrackingModule.persistence} == {NetcdfArtifact, TrackTablesWrite} + assert {type(s) for s in TrackingModule.persistence} == { + NetcdfArtifact, + TrackTablesWrite, + DecisionTableWrite, + } assert {type(s) for s in CellVolumeStatsModule.persistence} == {ProductTableWrite} From a81e7e9cfdf8f8683b7e9aaac5ed1ce063b84221 Mon Sep 17 00:00:00 2001 From: Bhupendra Raut Date: Sun, 20 Sep 2026 17:55:51 -0500 Subject: [PATCH 02/12] FIX#71:Identifying OSM User-Agent and cache tiles --- src/adapt/consumers/live/_renderer.py | 5 ++++- src/adapt/visualization/plotter.py | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/adapt/consumers/live/_renderer.py b/src/adapt/consumers/live/_renderer.py index 6b8835e..75e9295 100644 --- a/src/adapt/consumers/live/_renderer.py +++ b/src/adapt/consumers/live/_renderer.py @@ -23,12 +23,14 @@ from matplotlib.figure import Figure from adapt.consumers.live._utils import _centroid_track_to_km, _visible_uids_in_scan, cells_for_scan +from adapt.utils.basemap import TILE_CACHE_DIR, tile_headers logger = logging.getLogger(__name__) try: import contextily as ctx + ctx.set_cache_dir(str(TILE_CACHE_DIR)) # persist tiles across sessions HAS_CTX = True except ImportError: ctx = None @@ -437,7 +439,7 @@ def _basemap_extent_key(x_km, y_km) -> tuple[float, float, float, float]: def add_basemap(ax, ds, x_km, y_km) -> None: - """Add an OpenStreetMap basemap to *ax* using the dataset's radar location. + """Add a OpenStreetMap basemap to *ax* using the dataset's radar location. Tiles are fetched at most once per extent and cached on the axes, then re-drawn from that cache on later frames: every ``render_scan`` clears the @@ -481,6 +483,7 @@ def add_basemap(ax, ds, x_km, y_km) -> None: ax, crs=crs_str, source=ctx.providers.OpenStreetMap.Mapnik, + headers=tile_headers(), alpha=0.6, attribution=False, zoom=8, diff --git a/src/adapt/visualization/plotter.py b/src/adapt/visualization/plotter.py index 14c00f4..4bbb2de 100644 --- a/src/adapt/visualization/plotter.py +++ b/src/adapt/visualization/plotter.py @@ -19,6 +19,10 @@ import xarray as xr from adapt.contracts import CELL_LABELS_VAR # noqa: E402 (layer-legal: contracts is a leaf) +from adapt.utils.basemap import ( # noqa: E402 (layer-legal: utils is a leaf) + TILE_CACHE_DIR, + tile_headers, +) matplotlib.use("Agg") import contextlib @@ -28,6 +32,7 @@ try: import contextily as ctx + ctx.set_cache_dir(str(TILE_CACHE_DIR)) # persist tiles across sessions CONTEXTILY_AVAILABLE = True except ImportError: CONTEXTILY_AVAILABLE = False @@ -266,9 +271,10 @@ def _add_basemap( ax, crs=crs_str, source=ctx.providers.OpenStreetMap.Mapnik, + headers=tile_headers(), alpha=self.basemap_alpha, attribution=False, - zoom="auto", + zoom=8, # fixed: "auto" can pick zoom 10 (16x the tiles, ~100 MB cached per site) ) except Exception as e: logger.warning(f"Could not add basemap: {e}") From baf28cb9b3fa0921451c8de36f2fe66fd746a033 Mon Sep 17 00:00:00 2001 From: Bhupendra Raut Date: Tue, 22 Sep 2026 17:35:32 -0500 Subject: [PATCH 03/12] FIX: split detection normalizes by born-cell area, not parent hull --- .../schemas/collection_decisions.sql | 21 +++++- src/adapt/configuration/schemas/internal.py | 3 +- src/adapt/configuration/schemas/param.py | 23 ++++-- src/adapt/contracts/decisions.py | 24 +++++- src/adapt/execution/nodes/tracking.py | 1 + src/adapt/modules/tracking/config.py | 3 +- src/adapt/modules/tracking/decision_log.py | 39 ++++++++-- src/adapt/modules/tracking/module.py | 75 ++++++++++++++++--- src/adapt/persistence/track_store.py | 43 +++++++++++ .../test_split_merge_normalisation.py | 64 ++++++++++++++++ .../test_tracker_scan_local_outputs.py | 4 + tests/persistence/test_merge_history.py | 75 +++++++++++++++++++ 12 files changed, 344 insertions(+), 31 deletions(-) create mode 100644 tests/modules/tracking/test_split_merge_normalisation.py create mode 100644 tests/persistence/test_merge_history.py diff --git a/src/adapt/configuration/schemas/collection_decisions.sql b/src/adapt/configuration/schemas/collection_decisions.sql index de0736f..a42a6f8 100644 --- a/src/adapt/configuration/schemas/collection_decisions.sql +++ b/src/adapt/configuration/schemas/collection_decisions.sql @@ -5,7 +5,13 @@ -- the public API. Rows mirror the frozen record fields plus run/scan stamps. -- Config thresholds are not repeated here: join run_id to the run registry. -PRAGMA user_version = 1; +-- user_version 2: split/merge tests carry both normalisations and their areas; +-- tracking_unmatched carries the best candidate's overlaps and cost; +-- segmentation_frames counts size-filter exemptions for carried cells. +-- CREATE TABLE IF NOT EXISTS does not migrate an existing file — a collection +-- written under version 1 keeps its old columns, so start a new base_dir when +-- the new fields are needed. +PRAGMA user_version = 2; CREATE TABLE IF NOT EXISTS tracking_frames ( run_id TEXT NOT NULL, @@ -74,6 +80,9 @@ CREATE TABLE IF NOT EXISTS tracking_unmatched ( n_candidates INTEGER NOT NULL, best_candidate_label INTEGER, best_stage TEXT, + best_opc REAL, + best_ocp REAL, + best_cost REAL, reason TEXT NOT NULL, PRIMARY KEY (run_id, scan_id, side, cell_uid) ); @@ -87,9 +96,16 @@ CREATE TABLE IF NOT EXISTS tracking_split_merge_tests ( kind TEXT NOT NULL, continuing_cell_uid TEXT NOT NULL, tested_cell_label INTEGER NOT NULL, - overlap_fraction REAL NOT NULL, + overlap_fraction REAL NOT NULL, -- the fraction compared against `threshold` threshold REAL NOT NULL, passed INTEGER NOT NULL, + -- Both normalisations and the raw areas: MERGE decides on hull_fraction, + -- SPLIT on cell_fraction. Kept side by side so the denominator is auditable. + intersection_px INTEGER, + hull_px INTEGER, + tested_px INTEGER, + hull_fraction REAL, + cell_fraction REAL, PRIMARY KEY (run_id, scan_id, kind, continuing_cell_uid, tested_cell_label) ); @@ -107,6 +123,7 @@ CREATE TABLE IF NOT EXISTS segmentation_frames ( n_dropped_small INTEGER NOT NULL, n_dropped_large INTEGER NOT NULL, n_cells INTEGER NOT NULL, + n_size_exempt INTEGER, PRIMARY KEY (run_id, scan_id) ); diff --git a/src/adapt/configuration/schemas/internal.py b/src/adapt/configuration/schemas/internal.py index ccb8554..af96a5e 100644 --- a/src/adapt/configuration/schemas/internal.py +++ b/src/adapt/configuration/schemas/internal.py @@ -168,7 +168,8 @@ class InternalCellUidConfig(AdaptBaseModel): width: int = Field(ge=1) alphabet: Literal["base36_upper"] - split_overlap_threshold: float = Field(default=0.8, ge=0.0, le=1.0) + split_overlap_threshold: float = Field(default=0.7, ge=0.0, le=1.0) + merge_overlap_threshold: float = Field(default=0.7, ge=0.0, le=1.0) core_field_threshold: float = Field(default=40.0, ge=0.0) max_tracking_gap_minutes: float = Field(default=20.0, gt=0.0) max_speed_ms: float = Field(default=40.0, gt=0.0) diff --git a/src/adapt/configuration/schemas/param.py b/src/adapt/configuration/schemas/param.py index 07faa83..6bdb26e 100644 --- a/src/adapt/configuration/schemas/param.py +++ b/src/adapt/configuration/schemas/param.py @@ -257,12 +257,23 @@ class CellUidConfig(AdaptBaseModel): alphabet: Literal["base36_upper"] = "base36_upper" split_overlap_threshold: float = Field( - 0.8, + 0.6, ge=0.0, le=1.0, description=( - "Min fraction of projected hull area overlapping born/surviving cell " - "to confirm SPLIT or MERGE" + "Min fraction of the BORN cell explained by the continuing parent's " + "projected hull to confirm a SPLIT (Opc = intersection / born cell area)" + ), + ) + merge_overlap_threshold: float = Field( + 0.7, + ge=0.0, + le=1.0, + description=( + "Min fraction of the DISSIPATING cell's projected hull covered by the " + "continuing cell to confirm a MERGE (intersection / projected hull area). " + "Separate from split_overlap_threshold because the two tests normalise by " + "different areas; a single value governed both before they were split apart." ), ) core_field_threshold: float = Field( @@ -292,7 +303,7 @@ class CellUidConfig(AdaptBaseModel): "step from centroid jitter must not veto ordinary storm motion", ) heading_change_penalty_weight: float = Field( - 0.0, + 0.3, ge=0.0, description="Optional cost penalty per radian of heading change (0 = diagnostic only)", ) @@ -303,14 +314,14 @@ class CellUidConfig(AdaptBaseModel): "to absorb segmentation and optical-flow uncertainty", ) minimum_candidate_overlap: float = Field( - 0.20, + 0.10, ge=0.0, le=1.0, description="Hard gate: min fraction of the candidate cell covered by the projected hull " "(Opc = intersection / candidate area)", ) minimum_projected_overlap: float = Field( - 0.20, + 0.10, ge=0.0, le=1.0, description="Hard gate: min fraction of the projected hull covered by the candidate cell " diff --git a/src/adapt/contracts/decisions.py b/src/adapt/contracts/decisions.py index 63e74aa..3d57fda 100644 --- a/src/adapt/contracts/decisions.py +++ b/src/adapt/contracts/decisions.py @@ -87,18 +87,35 @@ class TrackingUnmatched: best_stage: str | None reason: str # NO_CANDIDATE | ALL_REJECTED_OVERLAP | ALL_REJECTED_KINEMATIC | # LOST_ASSIGNMENT | SPLIT_CHILD | MERGE_SOURCE | RESET | FIRST_SCAN + # How close the best candidate came, so a near-miss is distinguishable from + # no candidate at all without re-joining tracking_candidates. + best_opc: float | None = None + best_ocp: float | None = None + best_cost: float | None = None @dataclass(frozen=True) class TrackingSplitMergeTest: - """Every hull overlap tested against ``split_overlap`` — pass or fail.""" + """Every hull overlap tested — pass or fail. + + MERGE compares ``hull_fraction`` against the merge threshold, SPLIT compares + ``cell_fraction`` against the split one; ``overlap_fraction`` always holds + whichever was used for the decision. + """ kind: str # SPLIT | MERGE continuing_cell_uid: str tested_cell_label: int - overlap_fraction: float + overlap_fraction: float # the fraction actually compared against `threshold` threshold: float passed: bool + # Both normalisations plus the raw areas, so the choice of denominator stays + # auditable: MERGE decides on `hull_fraction`, SPLIT on `cell_fraction`. + intersection_px: int | None = None + hull_px: int | None = None + tested_px: int | None = None + hull_fraction: float | None = None + cell_fraction: float | None = None @dataclass(frozen=True) @@ -126,6 +143,9 @@ class SegmentationFrame: n_dropped_small: int n_dropped_large: int n_cells: int + # Carried-seed basins the minimum-size filter spared (0 when the exemption + # is off), so the rescue's contribution can be separated from the filter's. + n_size_exempt: int = 0 @dataclass(frozen=True) diff --git a/src/adapt/execution/nodes/tracking.py b/src/adapt/execution/nodes/tracking.py index 3dd22c3..73b765e 100644 --- a/src/adapt/execution/nodes/tracking.py +++ b/src/adapt/execution/nodes/tracking.py @@ -86,6 +86,7 @@ class TrackingModule(BaseModule): def build_config(cls, cfg) -> TrackingConfig: return TrackingConfig( split_overlap=cfg.tracker.split_overlap_threshold, + merge_overlap=cfg.tracker.merge_overlap_threshold, core_field_threshold=cfg.tracker.core_field_threshold, uid_width=cfg.tracker.cell_uid.width, field_var=cfg.global_.tracking_field, diff --git a/src/adapt/modules/tracking/config.py b/src/adapt/modules/tracking/config.py index 1df29b1..2711bc0 100644 --- a/src/adapt/modules/tracking/config.py +++ b/src/adapt/modules/tracking/config.py @@ -14,7 +14,8 @@ class TrackingConfig(BaseModel): model_config = ConfigDict(frozen=True) - split_overlap: float + split_overlap: float # Opc gate on the born cell (SPLIT) + merge_overlap: float # hull-coverage gate on the dissipating cell (MERGE) core_field_threshold: float uid_width: int field_var: str diff --git a/src/adapt/modules/tracking/decision_log.py b/src/adapt/modules/tracking/decision_log.py index 643ea55..462fcde 100644 --- a/src/adapt/modules/tracking/decision_log.py +++ b/src/adapt/modules/tracking/decision_log.py @@ -151,11 +151,18 @@ def _rank_costs(self) -> None: for rank, (_, j) in enumerate(sorted(costs), start=1): self._pairs[(i, j)]["cost_rank"] = rank - def _best(self, rows: list[tuple[int, dict]]) -> tuple[int | None, str | None]: + def _best(self, rows: list[tuple[int, dict]]): + """The candidate that got furthest, with how close it came. + + Returns ``(label, stage, opc, ocp, cost)``. The overlaps and cost make a + near-miss measurable directly: a cell whose best candidate reached + Opc 0.19 against a 0.20 gate is a different failure from one that had + no candidate at all. + """ if not rows: - return None, None + return None, None, None, None, None label, row = max(rows, key=lambda item: _STAGE_ORDER[item[1]["last_stage"]]) - return label, row["last_stage"] + return label, row["last_stage"], row.get("opc"), row.get("ocp"), row.get("cost") def _reason(self, rows: list[tuple[int, dict]]) -> str: if self.reset_code is not None: @@ -174,17 +181,35 @@ def _unmatched_prev(self): if i in self._matched_prev: continue rows = [(self.curr_labels[j], r) for (p, j), r in self._pairs.items() if p == i] - best_label, best_stage = self._best(rows) + best_label, best_stage, b_opc, b_ocp, b_cost = self._best(rows) reason = "MERGE_SOURCE" if cell.uid in self._merge_sources else self._reason(rows) yield TrackingUnmatched( - "prev", cell.uid, cell.label, len(rows), best_label, best_stage, reason + "prev", + cell.uid, + cell.label, + len(rows), + best_label, + best_stage, + reason, + best_opc=b_opc, + best_ocp=b_ocp, + best_cost=b_cost, ) def _unmatched_curr(self): for j, uid in sorted(self._born_uids.items()): rows = [(self.prev[i].label, r) for (i, c), r in self._pairs.items() if c == j] - best_label, best_stage = self._best(rows) + best_label, best_stage, b_opc, b_ocp, b_cost = self._best(rows) reason = "SPLIT_CHILD" if j in self._split_children else self._reason(rows) yield TrackingUnmatched( - "curr", uid, self.curr_labels[j], len(rows), best_label, best_stage, reason + "curr", + uid, + self.curr_labels[j], + len(rows), + best_label, + best_stage, + reason, + best_opc=b_opc, + best_ocp=b_ocp, + best_cost=b_cost, ) diff --git a/src/adapt/modules/tracking/module.py b/src/adapt/modules/tracking/module.py index be1587c..9ccca3f 100644 --- a/src/adapt/modules/tracking/module.py +++ b/src/adapt/modules/tracking/module.py @@ -126,6 +126,7 @@ class CellTracker: def __init__(self, config): self.split_overlap = config.split_overlap + self.merge_overlap = config.merge_overlap self.core_threshold = config.core_field_threshold self.field_var = config.field_var # generic field; not assumed to be reflectivity self.labels_var = config.labels_var @@ -771,11 +772,44 @@ def record(i: int, c: int, method: MatchMethod) -> dict: return events @staticmethod - def _hull_overlap_fraction(proj_labels: np.ndarray, cell_id: int, mask: np.ndarray) -> float: - """Fraction of a previous cell's projected hull covered by ``mask``.""" + def _overlap_areas( + proj_labels: np.ndarray, cell_id: int, mask: np.ndarray + ) -> tuple[int, int, int]: + """Intersection and both areas (px) for a projected hull against a cell mask.""" proj_mask = proj_labels == cell_id - denom = float(np.sum(proj_mask)) - return float(np.sum(mask & proj_mask)) / denom if denom else 0.0 + return ( + int(np.sum(mask & proj_mask)), + int(np.sum(proj_mask)), + int(np.sum(mask)), + ) + + @staticmethod + def _hull_overlap_fraction(proj_labels: np.ndarray, cell_id: int, mask: np.ndarray) -> float: + """Fraction of a previous cell's projected hull covered by ``mask``. + + The MERGE criterion: the hull belongs to the *dissipating* cell, so this + asks how completely that vanishing cell went into ``mask``. It reaches + 1.0 when a small cell is wholly absorbed by a larger one. + """ + inter, hull_px, _ = CellTracker._overlap_areas(proj_labels, cell_id, mask) + return inter / hull_px if hull_px else 0.0 + + @staticmethod + def _cell_overlap_fraction(proj_labels: np.ndarray, cell_id: int, mask: np.ndarray) -> float: + """Fraction of ``mask`` explained by a previous cell's projected hull (Opc). + + The SPLIT criterion, and the mirror of the merge one: here the hull + belongs to the *continuing parent* and ``mask`` is a born fragment, so + normalising by the hull would ask the fragment to account for the whole + parent — impossible for a split child by construction (its ceiling is + child area / parent hull area, observed at 0.72 on KHTX 2025-06-17 with + a median parent/child area ratio of 3.3, so no split could ever pass a + 0.8 threshold). Normalising by the born cell's area instead asks the + question that matters: is this new cell explained by the parent? It is + the same quantity as ``minimum_candidate_overlap`` in the main gate. + """ + inter, _, mask_px = CellTracker._overlap_areas(proj_labels, cell_id, mask) + return inter / mask_px if mask_px else 0.0 def _new_track_node(self, cell: dict, curr_time) -> int: """Allocate a new track index + identity and add the observation node.""" @@ -793,7 +827,13 @@ def _detect_splits( born: list[int], curr_time, ) -> tuple[set[int], list[dict]]: - """A born cell overlapping a continuing parent's projected hull is a SPLIT child.""" + """A born cell explained by a continuing parent's projected hull is a SPLIT child. + + The test is ``Opc`` — intersection over the *born* cell's area — not the + hull-normalised fraction the merge test uses. See + ``_cell_overlap_fraction`` for why the hull denominator makes a split + structurally undetectable. + """ split_born: set[int] = set() events: list[dict] = [] for b_idx in born: @@ -801,7 +841,8 @@ def _detect_splits( best_parent, best_overlap = None, 0.0 for prev_idx, curr_node in matched_prev.items(): cell_id = self.graph.get_node_attr(prev_node_ids[prev_idx], "cell_id") - overlap = self._hull_overlap_fraction(proj_labels, cell_id, b_mask) + inter, hull_px, tested_px = self._overlap_areas(proj_labels, cell_id, b_mask) + overlap = inter / tested_px if tested_px else 0.0 self._log.split_merge_test( TrackingSplitMergeTest( "SPLIT", @@ -810,6 +851,11 @@ def _detect_splits( float(overlap), self.split_overlap, overlap >= self.split_overlap, + intersection_px=inter, + hull_px=hull_px, + tested_px=tested_px, + hull_fraction=(inter / hull_px if hull_px else 0.0), + cell_fraction=float(overlap), ) ) if overlap >= self.split_overlap and overlap > best_overlap: @@ -840,20 +886,25 @@ def _detect_merges( cell_id = self.graph.get_node_attr(d_node, "cell_id") best_target, best_overlap = None, 0.0 for c_idx, curr_node in matched_curr.items(): - overlap = self._hull_overlap_fraction( - proj_labels, cell_id, curr_cells[c_idx]["mask"] - ) + c_mask = curr_cells[c_idx]["mask"] + inter, hull_px, tested_px = self._overlap_areas(proj_labels, cell_id, c_mask) + overlap = inter / hull_px if hull_px else 0.0 self._log.split_merge_test( TrackingSplitMergeTest( "MERGE", str(self.graph.get_node_attr(curr_node, "cell_uid")), int(cell_id), float(overlap), - self.split_overlap, - overlap >= self.split_overlap, + self.merge_overlap, + overlap >= self.merge_overlap, + intersection_px=inter, + hull_px=hull_px, + tested_px=tested_px, + hull_fraction=float(overlap), + cell_fraction=(inter / tested_px if tested_px else 0.0), ) ) - if overlap >= self.split_overlap and overlap > best_overlap: + if overlap >= self.merge_overlap and overlap > best_overlap: best_target, best_overlap = curr_node, overlap if best_target is not None: self.graph.add_edge(d_node, best_target, edge_type="MERGE", cost=0.0) diff --git a/src/adapt/persistence/track_store.py b/src/adapt/persistence/track_store.py index 7f13334..a2b6307 100644 --- a/src/adapt/persistence/track_store.py +++ b/src/adapt/persistence/track_store.py @@ -68,6 +68,14 @@ ("is_split_source_here", "INTEGER"), ("is_merge_source_here", "INTEGER"), ("is_terminated_after_here", "INTEGER"), + # A merge ends the absorbed cell's track and the survivor keeps its own uid + # and age, so the longer history would otherwise be unrecoverable from this + # table. These name the absorbed cell and the lifetime it had reached at + # this scan (the oldest one, when several merge into the same target). + # Identity is deliberately NOT transferred — a consumer that wants the + # longer lineage reconstructs it from these. + ("merged_from_cell_uid", "TEXT"), + ("merged_from_age_seconds", "REAL"), ) _CBS_NOT_NULL = {"run_id", "scan_id", "scan_time", "cell_label", "cell_uid"} @@ -282,6 +290,14 @@ def write_scan( uid_col = _uid_col(tracked_cells_df) self._reject_duplicate_identities(scan_id, tracked_cells_df, uid_col) cell_uids = tracked_cells_df[uid_col].astype(str).unique().tolist() + # A merge source has just died, so it is absent from tracked_cells_df + # — but its first_seen_time is exactly what merged_from_age_seconds + # needs, so pull those uids in too. + if not cell_events_df.empty and "source_cell_uid" in cell_events_df: + merge_sources = cell_events_df.loc[ + cell_events_df["event_type"] == "MERGE", "source_cell_uid" + ].dropna() + cell_uids += [str(u) for u in merge_sources.unique() if str(u) not in cell_uids] placeholders = ",".join("?" * len(cell_uids)) first_seen_rows = conn.execute( "SELECT cell_uid, first_seen_time FROM cell_tracks " @@ -549,6 +565,31 @@ def _build_cells_rows( except ValueError: scan_dt = None + def _age_at_scan(uid: str) -> float | None: + """Lifetime a track had reached by this scan, from its first_seen time.""" + if scan_dt is None or not first_seen_map or uid not in first_seen_map: + return None + try: + return max(0.0, (scan_dt - from_scan_iso(first_seen_map[uid])).total_seconds()) + except ValueError: + return None + + # Absorbed-cell provenance per merge target: keep the oldest source when + # several merge into the same survivor, since that is the history a + # consumer would otherwise lose. + merged_from: dict[int, tuple[str, float | None]] = {} + if not cell_events_df.empty: + for _, ev in cell_events_df.iterrows(): + if ev["event_type"] != "MERGE": + continue + tcl, suid = ev.get("target_cell_label"), ev.get("source_cell_uid") + if pd.isna(tcl) or not suid or pd.isna(suid): + continue + age = _age_at_scan(str(suid)) + prev = merged_from.get(int(tcl)) + if prev is None or (age or 0.0) > (prev[1] or 0.0): + merged_from[int(tcl)] = (str(suid), age) + rows = [] uid_col = _uid_col(tracked_cells_df) for _, tc in tracked_cells_df.iterrows(): @@ -584,6 +625,8 @@ def _build_cells_rows( "is_split_source_here": 0, "is_merge_source_here": 0, "is_terminated_after_here": 0, + "merged_from_cell_uid": merged_from.get(cl, (None, None))[0], + "merged_from_age_seconds": merged_from.get(cl, (None, None))[1], } # Merge all cell_stats columns if cl in stats_map: diff --git a/tests/modules/tracking/test_split_merge_normalisation.py b/tests/modules/tracking/test_split_merge_normalisation.py new file mode 100644 index 0000000..ad464b0 --- /dev/null +++ b/tests/modules/tracking/test_split_merge_normalisation.py @@ -0,0 +1,64 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""The split and merge tests normalise by different areas. + +Before these were separated, both divided the intersection by the *projected +hull* area. For a merge that is the dissipating cell's own hull, so the fraction +can reach 1.0. For a split the hull belongs to the continuing parent while the +tested mask is a born fragment, so the fraction is bounded above by +child area / parent hull area — a split could not pass a high threshold no +matter how cleanly it split. These tests pin both denominators. +""" + +import numpy as np +import pytest + +from adapt.modules.tracking.module import CellTracker + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def parent_and_child(): + """A 40 px parent hull and a 10 px born fragment fully inside it.""" + proj = np.zeros((10, 10), dtype=np.int32) + proj[0:4, 0:10] = 1 # parent hull, 40 px + child = np.zeros((10, 10), dtype=bool) + child[0:1, 0:10] = True # 10 px, entirely within the hull + return proj, child + + +def test_hull_and_cell_fractions_use_different_denominators(parent_and_child): + proj, child = parent_and_child + inter, hull_px, tested_px = CellTracker._overlap_areas(proj, 1, child) + assert (inter, hull_px, tested_px) == (10, 40, 10) + # merge normalisation: how much of the hull was covered + assert CellTracker._hull_overlap_fraction(proj, 1, child) == pytest.approx(0.25) + # split normalisation: how much of the born cell the parent explains + assert CellTracker._cell_overlap_fraction(proj, 1, child) == pytest.approx(1.0) + + +def test_a_perfectly_explained_child_fails_the_hull_normalisation(parent_and_child): + """The exact failure that made splits undetectable: a child wholly inside + its parent's hull still scores only 0.25 when normalised by the hull.""" + proj, child = parent_and_child + assert CellTracker._hull_overlap_fraction(proj, 1, child) < 0.7 + assert CellTracker._cell_overlap_fraction(proj, 1, child) >= 0.7 + + +def test_a_wholly_absorbed_cell_reaches_one_on_the_merge_normalisation(): + """The merge case is unaffected: a small cell absorbed by a big one + covers all of its own hull.""" + proj = np.zeros((10, 10), dtype=np.int32) + proj[0:2, 0:2] = 1 # dissipating cell's hull, 4 px + survivor = np.zeros((10, 10), dtype=bool) + survivor[0:6, 0:6] = True # the large continuing cell + assert CellTracker._hull_overlap_fraction(proj, 1, survivor) == pytest.approx(1.0) + + +def test_empty_hull_and_empty_mask_are_zero_not_an_error(): + empty = np.zeros((4, 4), dtype=np.int32) + mask = np.ones((4, 4), dtype=bool) + assert CellTracker._hull_overlap_fraction(empty, 1, mask) == 0.0 + assert CellTracker._cell_overlap_fraction(empty, 1, np.zeros((4, 4), dtype=bool)) == 0.0 diff --git a/tests/modules/tracking/test_tracker_scan_local_outputs.py b/tests/modules/tracking/test_tracker_scan_local_outputs.py index 743c124..dec4ae6 100644 --- a/tests/modules/tracking/test_tracker_scan_local_outputs.py +++ b/tests/modules/tracking/test_tracker_scan_local_outputs.py @@ -23,7 +23,10 @@ def config(): import shutil param = ParamConfig() + # Split and merge are separate thresholds now; this fixture wants both + # loose, which is what the single value used to mean. param.tracker.split_overlap_threshold = 0.4 + param.tracker.merge_overlap_threshold = 0.4 user = UserConfig(base_dir=str(Path(d)), radar="TEST_RADAR") internal = resolve_config(param, user, None) return TrackingModule.build_config(internal) @@ -263,6 +266,7 @@ def _make_config(split_overlap=0.4): try: param = ParamConfig() param.tracker.split_overlap_threshold = split_overlap + param.tracker.merge_overlap_threshold = split_overlap user = UserConfig(base_dir=str(Path(d)), radar="TEST_RADAR") internal = resolve_config(param, user, None) return TrackingModule.build_config(internal) diff --git a/tests/persistence/test_merge_history.py b/tests/persistence/test_merge_history.py new file mode 100644 index 0000000..36b270f --- /dev/null +++ b/tests/persistence/test_merge_history.py @@ -0,0 +1,75 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""A merge target records which cell it absorbed and how old that cell was. + +A merge ends the absorbed cell's track while the survivor keeps its own uid and +age, so without these columns the longer history is unrecoverable from +``cells_by_scan``. Identity is deliberately not transferred — the survivor's +own ``cell_uid`` and ``age_seconds`` are untouched. +""" + +import pandas as pd +import pytest + +from adapt.persistence.track_store import TrackStore + +pytestmark = pytest.mark.unit + +SCAN_ISO = "2025-06-17T16:00:00Z" + + +def _rows(events: list[dict], first_seen: dict[str, str]) -> dict[int, dict]: + tracked = pd.DataFrame([{"cell_label": 1, "cell_uid": "SURVIVOR"}]) + built = TrackStore._build_cells_rows( + TrackStore.__new__(TrackStore), + run_id="R", + scan_id="S", + scan_iso=SCAN_ISO, + cell_stats_df=pd.DataFrame([{"cell_label": 1}]), + tracked_cells_df=tracked, + cell_events_df=pd.DataFrame(events), + adjacency={}, + first_seen_map=first_seen, + ) + return {r["cell_label"]: r for r in built} + + +def _merge(source_uid: str) -> dict: + return { + "event_type": "MERGE", + "source_cell_uid": source_uid, + "target_cell_uid": "SURVIVOR", + "target_cell_label": 1, + } + + +def test_merge_records_the_absorbed_cell_and_its_lifetime(): + row = _rows( + [_merge("ABSORBED")], + {"SURVIVOR": "2025-06-17T15:55:00Z", "ABSORBED": "2025-06-17T15:00:00Z"}, + )[1] + assert row["merged_from_cell_uid"] == "ABSORBED" + assert row["merged_from_age_seconds"] == pytest.approx(3600.0) + # identity is NOT transferred: the survivor keeps its own, shorter age + assert row["cell_uid"] == "SURVIVOR" + assert row["age_seconds"] == pytest.approx(300.0) + + +def test_the_oldest_source_wins_when_several_merge_at_once(): + row = _rows( + [_merge("YOUNG"), _merge("OLD")], + { + "SURVIVOR": "2025-06-17T15:55:00Z", + "YOUNG": "2025-06-17T15:50:00Z", + "OLD": "2025-06-17T14:00:00Z", + }, + )[1] + assert row["merged_from_cell_uid"] == "OLD" + assert row["merged_from_age_seconds"] == pytest.approx(7200.0) + + +def test_a_cell_with_no_merge_carries_nulls(): + row = _rows([], {"SURVIVOR": "2025-06-17T15:55:00Z"})[1] + assert row["merged_from_cell_uid"] is None + assert row["merged_from_age_seconds"] is None From 1038237f92b3e22b517cf83fd375232634ee6228 Mon Sep 17 00:00:00 2001 From: Bhupendra Raut Date: Tue, 22 Sep 2026 17:35:55 -0500 Subject: [PATCH 04/12] =?UTF-8?q?ENH:=20downloader=20backpressure=20?= =?UTF-8?q?=E2=80=94=20pause/resume=20on=20queue=20depth?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/adapt/configuration/schemas/internal.py | 2 + src/adapt/configuration/schemas/param.py | 20 ++++ src/adapt/configuration/schemas/user.py | 2 + src/adapt/modules/acquisition/module.py | 72 +++++++++++++- src/adapt/runtime/orchestrator.py | 43 ++++++--- .../test_downloader_backpressure.py | 96 +++++++++++++++++++ tests/runtime/test_orchestrator.py | 6 +- 7 files changed, 226 insertions(+), 15 deletions(-) create mode 100644 tests/modules/acquisition/test_downloader_backpressure.py diff --git a/src/adapt/configuration/schemas/internal.py b/src/adapt/configuration/schemas/internal.py index af96a5e..ef0eac4 100644 --- a/src/adapt/configuration/schemas/internal.py +++ b/src/adapt/configuration/schemas/internal.py @@ -49,6 +49,8 @@ class InternalDownloaderConfig(AdaptBaseModel): start_time: str | None end_time: str | None min_file_size: int + max_queue_size: int = 100 + queue_resume_fraction: float = 0.10 class InternalRegridderConfig(AdaptBaseModel): diff --git a/src/adapt/configuration/schemas/param.py b/src/adapt/configuration/schemas/param.py index 6bdb26e..63b87ec 100644 --- a/src/adapt/configuration/schemas/param.py +++ b/src/adapt/configuration/schemas/param.py @@ -64,6 +64,26 @@ class DownloaderConfig(AdaptBaseModel): min_file_size: int = Field( 1024, ge=1, description="Minimum file size in bytes to consider valid" ) + max_queue_size: int = Field( + 100, + ge=1, + description=( + "Scans the downloader may run ahead of the processor. Each queued item is a " + "small message (the volume itself is already committed to the store), so this " + "costs disk for the downloaded volumes only, ~10 MB each. When the queue " + "reaches this many items the downloader pauses; processing never pauses." + ), + ) + queue_resume_fraction: float = Field( + 0.10, + gt=0.0, + lt=1.0, + description=( + "The downloader resumes once the queue has drained to this fraction of " + "max_queue_size (at least 1 item). The gap between the two levels stops the " + "downloader flapping on every consumed scan." + ), + ) class RegridderConfig(AdaptBaseModel): diff --git a/src/adapt/configuration/schemas/user.py b/src/adapt/configuration/schemas/user.py index c1b7d49..6d5b692 100644 --- a/src/adapt/configuration/schemas/user.py +++ b/src/adapt/configuration/schemas/user.py @@ -124,6 +124,8 @@ class UserDownloaderConfig(_UserSection): max_fetch_retries: int | None = None start_time: str | None = None end_time: str | None = None + max_queue_size: int | None = None + queue_resume_fraction: float | None = None class UserAnalyzerConfig(_UserSection): diff --git a/src/adapt/modules/acquisition/module.py b/src/adapt/modules/acquisition/module.py index d4780a2..edc9016 100644 --- a/src/adapt/modules/acquisition/module.py +++ b/src/adapt/modules/acquisition/module.py @@ -11,6 +11,7 @@ import contextlib import io import logging +import queue import tempfile import threading import time @@ -128,6 +129,13 @@ def __init__( self.latest_minutes = config.downloader.latest_minutes self.start_time = config.downloader.start_time self.end_time = config.downloader.end_time + # Backpressure with hysteresis: pause downloading when the queue reaches + # `high`, resume once it has drained to `low`. The processor is never + # paused — only this thread waits. + self._queue_high = config.downloader.max_queue_size + self._queue_low = max(1, int(self._queue_high * config.downloader.queue_resume_fraction)) + self._queue_pauses = 0 + self._queue_paused_seconds = 0.0 self.result_queue = result_queue self._acquire = acquire @@ -378,6 +386,60 @@ def _download_realtime(self) -> list: # Scan fetching and processing # ======================================================================== + # ======================================================================== + # Queue backpressure + # ======================================================================== + + def queue_backpressure_stats(self) -> dict: + """How often, and for how long, the downloader waited on the processor.""" + return { + "queue_high": self._queue_high, + "queue_low": self._queue_low, + "pauses": self._queue_pauses, + "paused_seconds": round(self._queue_paused_seconds, 1), + } + + def _wait_for_queue_room(self, poll_seconds: float = 0.5) -> None: + """Block this thread while the queue is at its high-water mark. + + Returns as soon as the queue has drained to the low-water mark, or a stop + is requested. Only the downloader waits; the processor keeps consuming. + The two levels are deliberately apart so a full queue does not turn into + one download per consumed scan. + """ + if self.result_queue is None or self.result_queue.qsize() < self._queue_high: + return + self._queue_pauses += 1 + t0 = time.monotonic() + logger.info( + "Queue at %d/%d — pausing downloads until it drains to %d", + self.result_queue.qsize(), + self._queue_high, + self._queue_low, + ) + while not self.stopped() and self.result_queue.qsize() > self._queue_low: + self._sleep(poll_seconds) + waited = time.monotonic() - t0 + self._queue_paused_seconds += waited + if not self.stopped(): + logger.info( + "Queue at %d — resuming downloads after %.0f s", self.result_queue.qsize(), waited + ) + + def _put_until_stopped(self, message: dict, timeout: float = 1.0) -> bool: + """Put with a bounded wait so a stop request is never ignored. + + The hysteresis pause normally guarantees room, so this only ever waits + if something else filled the queue. Returns False if stopped first. + """ + while not self.stopped(): + try: + self.result_queue.put(message, timeout=timeout) + return True + except queue.Full: + continue + return False + def _parse_time_range(self) -> tuple: """Parse ISO timestamps to datetime objects.""" start = datetime.fromisoformat(self.start_time.replace("Z", "+00:00")) @@ -487,6 +549,12 @@ def _process_scans(self, scans: list) -> list: if already_queued: continue + # Hold here (not after the download) so the queue depth is exactly + # how far downloads run ahead of processing. + self._wait_for_queue_room() + if self.stopped(): + break + if self._acquire.is_acquired(source_uri): message = self._acquire.acquire_existing(source_uri, scan_time=scan.scan_time) else: @@ -502,8 +570,8 @@ def _process_scans(self, scans: list) -> list: local.unlink(missing_ok=True) acquired.append(source_uri) - if self.result_queue is not None: - self.result_queue.put(message) + if self.result_queue is not None and not self._put_until_stopped(message): + break with self._known_files_lock: self._known_files.add(source_uri) queued += 1 diff --git a/src/adapt/runtime/orchestrator.py b/src/adapt/runtime/orchestrator.py index 9974ac5..920dc7d 100644 --- a/src/adapt/runtime/orchestrator.py +++ b/src/adapt/runtime/orchestrator.py @@ -125,7 +125,7 @@ class PipelineOrchestrator: def __init__( self, config: "InternalConfig", - max_queue_size: int = 20, + max_queue_size: int | None = None, close_repository_on_stop: bool = True, ): """Initialize orchestrator with fully resolved runtime configuration. @@ -137,13 +137,20 @@ def __init__( Already contains all directory paths, run ID, and validated settings. max_queue_size : int, optional - Maximum size of inter-thread communication queues (default: 100). + Override for ``config.downloader.max_queue_size``. Left unset the + configured value is used. """ self.config = config - self.max_queue_size = max_queue_size + self.max_queue_size = ( + max_queue_size if max_queue_size is not None else config.downloader.max_queue_size + ) - # Queue for downloader -> processor communication - self.downloader_queue: queue.Queue[object] = queue.Queue(maxsize=max_queue_size) + # Queue for downloader -> processor communication. Bounded as a hard + # backstop only: the downloader pauses itself at this level and resumes + # at queue_resume_fraction of it (see AwsNexradDownloader._wait_for_queue_room), + # so in normal operation a put() never blocks and the processor is + # never starved. + self.downloader_queue: queue.Queue[object] = queue.Queue(maxsize=self.max_queue_size) # Fail fast: the pipeline never runs against an uninitialized root. self.store = Store.open(config.base_dir) @@ -534,6 +541,15 @@ def _check_historical_complete(self) -> bool: processed, expected = self.downloader.get_historical_progress() logger.info("Downloader complete: %d/%d files queued", processed, expected) + if hasattr(self.downloader, "queue_backpressure_stats"): + bp = self.downloader.queue_backpressure_stats() + logger.info( + "Downloader backpressure: paused %d time(s), %.0f s total (high %d / low %d)", + bp["pauses"], + bp["paused_seconds"], + bp["queue_high"], + bp["queue_low"], + ) # Stop the downloader (ends ingestion) so the queue can only shrink. self.downloader.stop() @@ -563,7 +579,6 @@ def _drain_queue(self, q: queue.Queue, name: str, timeout: int = 300): stay marked "downloaded" (not "analyzed") and resume on the next run. """ wait_count = 0 - start_time = time.time() last_size = q.qsize() while q.qsize() > 0: @@ -581,13 +596,19 @@ def _drain_queue(self, q: queue.Queue, name: str, timeout: int = 300): logger.info("Waiting for %s queue: %d remaining", name, current_size) time.sleep(1) - # Check timeout both by iteration count and elapsed time - if wait_count > timeout // 5 or (time.time() - start_time) > timeout: + # Abort only on a genuine stall: no item consumed for `timeout` + # seconds. There is deliberately NO cap on total elapsed time — a + # healthy processor working through a full queue of 20 slow scans + # can legitimately take longer than any fixed budget, and the old + # absolute cap silently dropped the tail of a run (22 of 34 scans + # processed on KHTX 2021-05-04, no error, run marked completed). + if wait_count > timeout: logger.warning( - "%s queue drain timeout (%d/%d seconds)", + "%s queue stalled: no progress for %d s with %d file(s) remaining — " + "abandoning them (they resume on the next run)", name, - int(time.time() - start_time), - timeout, + wait_count, + current_size, ) break diff --git a/tests/modules/acquisition/test_downloader_backpressure.py b/tests/modules/acquisition/test_downloader_backpressure.py new file mode 100644 index 0000000..d4095c2 --- /dev/null +++ b/tests/modules/acquisition/test_downloader_backpressure.py @@ -0,0 +1,96 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""The downloader pauses at the queue's high-water mark and resumes at the low one. + +Only the downloader waits — the processor is never paused. The two levels are +apart so a full queue does not degrade into one download per consumed scan. +""" + +from queue import Queue + +import pytest + +from adapt.configuration.schemas.user import UserDownloaderConfig +from adapt.modules.acquisition.module import AwsNexradDownloader + +pytestmark = pytest.mark.unit + + +def _downloader(make_config, q, fake_gateway, fake_aws_conn, scans, high=10, frac=0.3): + cfg = make_config( + downloader=UserDownloaderConfig(max_queue_size=high, queue_resume_fraction=frac) + ) + return AwsNexradDownloader( + cfg, result_queue=q, acquire=fake_gateway, conn=fake_aws_conn(scans), sleeper=lambda _: None + ) + + +def test_levels_come_from_config(make_config, fake_gateway, fake_aws_conn): + d = _downloader(make_config, Queue(), fake_gateway, fake_aws_conn, [], high=100, frac=0.1) + assert d._queue_high == 100 + assert d._queue_low == 10 + + +def test_low_water_is_at_least_one(make_config, fake_gateway, fake_aws_conn): + d = _downloader(make_config, Queue(), fake_gateway, fake_aws_conn, [], high=3, frac=0.1) + assert d._queue_low == 1 + + +def test_no_pause_below_high_water(make_config, fake_gateway, fake_aws_conn, fake_scan): + q = Queue(maxsize=10) + scans = [fake_scan(f"s{i}") for i in range(5)] + d = _downloader(make_config, q, fake_gateway, fake_aws_conn, scans, high=10) + + d._process_scans(scans) + + assert q.qsize() == 5 + assert d.queue_backpressure_stats()["pauses"] == 0 + + +def test_pauses_at_high_water_and_resumes_at_low( + make_config, fake_gateway, fake_aws_conn, fake_scan +): + """With the queue full, downloading waits until a consumer drains it to `low`.""" + q = Queue(maxsize=4) + scans = [fake_scan(f"s{i}") for i in range(6)] + d = _downloader(make_config, q, fake_gateway, fake_aws_conn, scans, high=4, frac=0.5) + assert d._queue_low == 2 + + # A consumer that removes one item per poll, recording the queue depth it saw. + seen: list[int] = [] + + def consume_one(_seconds): + seen.append(q.qsize()) + q.get_nowait() + + d._sleep = consume_one + d._process_scans(scans) + + # 4 downloaded before the first pause; the pause released once qsize <= 2 + # (two polls), then the remaining 2 were downloaded without a second pause. + stats = d.queue_backpressure_stats() + assert stats["pauses"] == 1 + assert seen == [4, 3] + assert len(fake_gateway.acquire_file_calls) == 6 + assert q.qsize() == 4 # 6 put, 2 consumed + + +def test_stop_request_releases_a_paused_downloader( + make_config, fake_gateway, fake_aws_conn, fake_scan +): + q = Queue(maxsize=2) + scans = [fake_scan(f"s{i}") for i in range(5)] + d = _downloader(make_config, q, fake_gateway, fake_aws_conn, scans, high=2, frac=0.5) + + polls = [] + + def stop_on_first_poll(_seconds): + polls.append(q.qsize()) + d.stop() + + d._sleep = stop_on_first_poll + d._process_scans(scans) + + assert polls == [2] + assert len(fake_gateway.acquire_file_calls) == 2 # nothing downloaded while stopped diff --git a/tests/runtime/test_orchestrator.py b/tests/runtime/test_orchestrator.py index 4ec3078..5474e56 100644 --- a/tests/runtime/test_orchestrator.py +++ b/tests/runtime/test_orchestrator.py @@ -92,10 +92,12 @@ def test_orchestrator_builds_working_observability_from_config(pipeline_config): def test_orchestrator_queue_wiring(pipeline_config): - """Orchestrator creates queues with correct size limits.""" + """The queue bound comes from config.downloader.max_queue_size unless overridden.""" orch = PipelineOrchestrator(pipeline_config) + assert orch.downloader_queue.maxsize == pipeline_config.downloader.max_queue_size == 100 - assert orch.downloader_queue.maxsize == 20 + orch = PipelineOrchestrator(pipeline_config, max_queue_size=7) + assert orch.downloader_queue.maxsize == 7 def test_orchestrator_stop_is_idempotent(pipeline_config): From 7a5b45436921ac7c0ed0520b5547cc4edc9e8c8b Mon Sep 17 00:00:00 2001 From: Bhupendra Raut Date: Tue, 22 Sep 2026 17:36:47 -0500 Subject: [PATCH 05/12] ENH: seed-carried cells exempt from min-size filter --- src/adapt/configuration/schemas/internal.py | 6 +- src/adapt/configuration/schemas/param.py | 34 ++++++--- .../schemas/segmenter_methods.py | 2 +- src/adapt/configuration/schemas/user.py | 45 +++++++++++- src/adapt/execution/nodes/detection.py | 16 ++-- src/adapt/modules/detection/config.py | 6 +- src/adapt/modules/detection/module.py | 73 +++++++++++++++---- tests/configuration/test_config_resolution.py | 8 +- tests/configuration/test_seed_carry_config.py | 54 ++++++++++---- .../detection/test_detection_config.py | 8 +- .../detection/test_segmentation_decisions.py | 9 ++- .../modules/detection/test_segmenter_init.py | 2 +- .../detection/test_segmenter_morphology.py | 2 +- .../test_segmenter_pyart_conv_strat.py | 6 +- .../detection/test_size_filter_exemption.py | 69 ++++++++++++++++++ 15 files changed, 271 insertions(+), 69 deletions(-) create mode 100644 tests/modules/detection/test_size_filter_exemption.py diff --git a/src/adapt/configuration/schemas/internal.py b/src/adapt/configuration/schemas/internal.py index ef0eac4..2ab3e23 100644 --- a/src/adapt/configuration/schemas/internal.py +++ b/src/adapt/configuration/schemas/internal.py @@ -75,12 +75,12 @@ class InternalSegmenterConfig(AdaptBaseModel): ] min_cellsize_gridpoint: int max_cellsize_gridpoint: int | None - closing_kernel: tuple[int, int] + closing_radius: int filter_by_size: bool h_maxima: float - seed_carry: bool - seed_carry_max_frames: int + seed_carry_frames: int seed_carry_min_separation: int + carried_exempt_size_filter: bool = False threshold_params: ThresholdParams conv_strat_raut_params: ConvStratRautParams conv_strat_yuter_params: ConvStratYuterParams diff --git a/src/adapt/configuration/schemas/param.py b/src/adapt/configuration/schemas/param.py index 63b87ec..12b6a86 100644 --- a/src/adapt/configuration/schemas/param.py +++ b/src/adapt/configuration/schemas/param.py @@ -122,23 +122,39 @@ class SegmenterConfig(AdaptBaseModel): # Shared labeling-backend parameters (applied to every method's mask) min_cellsize_gridpoint: int = Field(5, ge=1) max_cellsize_gridpoint: int | None = Field(None, ge=1) - closing_kernel: tuple[int, int] = (1, 1) + closing_radius: int = Field( + 2, + ge=0, + description=( + "Radius (grid points) of the disk used to close the convective mask: a " + "dilation followed by an erosion with the same disk. Fills gaps and " + "indentations narrower than ~2r and rounds the boundary without changing " + "the extent of large features. 0 disables it." + ), + ) filter_by_size: bool = True h_maxima: float = Field(5.0, gt=0, description="h-maxima height for cell seeding (dBZ)") # Previous-frame seeding: the projection module's one-step-ahead labels # become extra watershed markers (union with h-maxima seeds, never a # replacement). Off by default; off is byte-identical to no seeding. - seed_carry: bool = Field( - False, description="Seed the watershed with the previous frame's advected cell cores" + seed_carry_frames: int = Field( + 1, + ge=0, + description="How many consecutive frames a core may be carried without independent " + "detection. 0 disables the carry entirely; 1 carries only from the immediately " + "preceding frame (a core not re-detected next frame is dropped); N allows N frames", ) - seed_carry_max_frames: int = Field( - 2, - ge=1, - description="Drop a carried core after this many consecutive frames without " - "independent detection", + carried_exempt_size_filter: bool = Field( + False, + description="Spare carried-seed cells from min_cellsize_gridpoint. Off by default: " + "on KHTX 2021-05-04 the spared basins (1-4 px) sat wholly inside the parent's " + "projected hull (Opc 1.0) but covered ~4% of it, so the tracker's " + "minimum_projected_overlap gate rejected them anyway — 60 of 63 became one-scan " + "orphan tracks and 3 continued. The rescue moved from the size filter to the " + "tracking gate rather than succeeding. Kept as an option for study arms", ) seed_carry_min_separation: int = Field( - 3, + 5, ge=1, description="Minimum pixel distance from any h-maxima seed for a carried core " "to be admitted", diff --git a/src/adapt/configuration/schemas/segmenter_methods.py b/src/adapt/configuration/schemas/segmenter_methods.py index 6f6af67..79e8130 100644 --- a/src/adapt/configuration/schemas/segmenter_methods.py +++ b/src/adapt/configuration/schemas/segmenter_methods.py @@ -30,7 +30,7 @@ class ThresholdParams(AdaptBaseModel): """Fixed reflectivity-threshold method.""" threshold: float = Field( - 30.0, description="Reflectivity threshold in dBZ; cells have reflectivity above it" + 35.0, description="Reflectivity threshold in dBZ; cells have reflectivity above it" ) diff --git a/src/adapt/configuration/schemas/user.py b/src/adapt/configuration/schemas/user.py index 6d5b692..0d03c17 100644 --- a/src/adapt/configuration/schemas/user.py +++ b/src/adapt/configuration/schemas/user.py @@ -49,12 +49,51 @@ class UserSegmenterConfig(_UserSection): method: str | None = None min_cellsize_gridpoint: int | None = None max_cellsize_gridpoint: int | None = None - closing_kernel: tuple[int, int] | None = None + closing_radius: int | None = None filter_by_size: bool | None = None h_maxima: float | None = None - seed_carry: bool | None = None - seed_carry_max_frames: int | None = None + seed_carry_frames: int | None = None seed_carry_min_separation: int | None = None + carried_exempt_size_filter: bool | None = None + + @model_validator(mode="before") + @classmethod + def fold_legacy_closing_kernel(cls, data): + """Accept the retired ``closing_kernel: [w, h]`` rectangle as a disk radius. + + ``(1, 1)`` was a no-op and maps to 0; a w x h rectangle maps to + ``max(w, h) // 2`` so the smoothing scale is preserved. + """ + if isinstance(data, dict) and "closing_kernel" in data: + data = dict(data) + kernel = data.pop("closing_kernel") + if kernel is not None and data.get("closing_radius") is None: + w, h = kernel + data["closing_radius"] = max(int(w), int(h)) // 2 + return data + + @model_validator(mode="before") + @classmethod + def fold_legacy_seed_carry(cls, data): + """Accept the retired ``seed_carry`` / ``seed_carry_max_frames`` pair. + + One integer replaced them: 0 is off, N carries for N frames. A legacy + config maps as ``seed_carry: false -> 0`` and + ``seed_carry: true -> seed_carry_max_frames`` (default 2, its old + default), so existing runs keep their behaviour. + """ + if not isinstance(data, dict): + return data + legacy_on = data.pop("seed_carry", None) + legacy_max = data.pop("seed_carry_max_frames", None) + if legacy_on is None and legacy_max is None: + return data + if data.get("seed_carry_frames") is None: + if legacy_on is False: + data["seed_carry_frames"] = 0 + elif legacy_on is True or legacy_max is not None: + data["seed_carry_frames"] = 2 if legacy_max is None else int(legacy_max) + return data @field_validator("method", mode="before") @classmethod diff --git a/src/adapt/execution/nodes/detection.py b/src/adapt/execution/nodes/detection.py index d642e37..251660e 100644 --- a/src/adapt/execution/nodes/detection.py +++ b/src/adapt/execution/nodes/detection.py @@ -93,23 +93,23 @@ def build_config(cls, cfg) -> DetectionConfig: # Checked at startup: at run time "prior scan has no projected_ds" is # also the legitimate state of the second scan, so it cannot be told # apart from a disabled projection module there. - if seg.seed_carry and not _projection_enabled(cfg): + if seg.seed_carry_frames > 0 and not _projection_enabled(cfg): raise ValueError( - "segmenter.seed_carry needs the projection module (its advected labels " - "are the seeds carried forward) — enable projection or turn seed_carry off" + "segmenter.seed_carry_frames > 0 needs the projection module (its advected " + "labels are the seeds carried forward) — enable projection or set it to 0" ) method_params = getattr(seg, f"{seg.method}_params").model_dump() return DetectionConfig( method=seg.method, method_params=method_params, - closing_kernel=seg.closing_kernel, + closing_radius=seg.closing_radius, filter_by_size=seg.filter_by_size, min_cellsize_gridpoint=seg.min_cellsize_gridpoint, max_cellsize_gridpoint=seg.max_cellsize_gridpoint, h_maxima=seg.h_maxima, - seed_carry=seg.seed_carry, - seed_carry_max_frames=seg.seed_carry_max_frames, + seed_carry_frames=seg.seed_carry_frames, seed_carry_min_separation=seg.seed_carry_min_separation, + carried_exempt_size_filter=seg.carried_exempt_size_filter, reflectivity_var=cfg.global_.tracking_field, labels_var=CELL_LABELS_VAR, z_level=cfg.global_.z_level, @@ -131,7 +131,7 @@ def run(self, context: dict) -> dict: num_cells = int(labels.max().item()) seed_carry = ( tuple(int(age) for age in np.atleast_1d(labels.attrs["carry_age"])) - if config.seed_carry and num_cells + if config.seed_carry_frames > 0 and num_cells else () ) @@ -150,7 +150,7 @@ def _carried(prior: dict | None, config: DetectionConfig) -> CarriedLabels | Non no motion field yet (projection first runs at the second scan and is skipped after a too-large gap): the carry is undefined without one. """ - if not config.seed_carry or prior is None: + if config.seed_carry_frames == 0 or prior is None: return None if "projected_ds" not in prior: logger.debug("seed_carry: prior scan has no projected_ds; nothing carried") diff --git a/src/adapt/modules/detection/config.py b/src/adapt/modules/detection/config.py index 30969cf..07cd410 100644 --- a/src/adapt/modules/detection/config.py +++ b/src/adapt/modules/detection/config.py @@ -22,14 +22,14 @@ class DetectionConfig(BaseModel): method: str method_params: dict[str, Any] - closing_kernel: tuple[int, int] + closing_radius: int filter_by_size: bool min_cellsize_gridpoint: int max_cellsize_gridpoint: int | None h_maxima: float - seed_carry: bool - seed_carry_max_frames: int + seed_carry_frames: int seed_carry_min_separation: int + carried_exempt_size_filter: bool reflectivity_var: str labels_var: str z_level: float diff --git a/src/adapt/modules/detection/module.py b/src/adapt/modules/detection/module.py index e2390a8..2bc6157 100644 --- a/src/adapt/modules/detection/module.py +++ b/src/adapt/modules/detection/module.py @@ -215,7 +215,7 @@ class RadarCellSegmenter: Resolved parameters for the selected method. For ``threshold`` this is ``{"threshold": }``; for the pyart methods it is the algorithm's keyword arguments (splatted into the pyart call, recorded in the output). - - `closing_kernel` : tuple of int, default (1, 1) + - `closing_radius` : int, default 2 (disk radius for mask closing; 0 = off) Morphological closing footprint. (1, 1) means no closing. - `filter_by_size` : bool, default True Whether to apply cell size filtering. @@ -247,14 +247,16 @@ def __init__(self, config): """ self.method = config.method self.method_params = config.method_params - self.kernel_size = config.closing_kernel + self.closing_radius = config.closing_radius self.filter_by_size = config.filter_by_size self.min_gridpoints = config.min_cellsize_gridpoint self.max_gridpoints = config.max_cellsize_gridpoint self.h_maxima = config.h_maxima - self.seed_carry = config.seed_carry - self.seed_carry_max_frames = config.seed_carry_max_frames + self.seed_carry_frames = config.seed_carry_frames + # One integer governs the carry: 0 is off, N allows N frames. + self.seed_carry = config.seed_carry_frames > 0 self.seed_carry_min_separation = config.seed_carry_min_separation + self.carried_exempt_size_filter = config.carried_exempt_size_filter self.refl_name = config.reflectivity_var self.labels_name = config.labels_var self.z_level = config.z_level @@ -395,15 +397,34 @@ def _binary_to_labels( carried: CarriedLabels | None, ) -> tuple[np.ndarray, tuple[int, ...]]: """Morphology, detect cells, filter; returns labels and carry age per label.""" - from skimage.morphology import closing, footprint_rectangle - - closed_mask = closing(binary_mask, footprint_rectangle(self.kernel_size)) + from skimage.morphology import closing, disk + + # Dilate then erode with the same disk: gaps and indentations narrower + # than ~2r are filled and the boundary is rounded, while the outer extent + # of anything larger than the disk is unchanged. + closed_mask = ( + closing(binary_mask, disk(self.closing_radius)) + if self.closing_radius > 0 + else binary_mask + ) components, _ = label(closed_mask, structure=_COMPONENT_CONNECTIVITY) basins, admitted, seed_rows = self._label_maxtree(closed_mask, field, carried, components) + # Basins whose marker came from a carried seed, so the size filter can + # spare them (see _filter_and_relabel). + exempt = ( + frozenset( + int(r["marker_id"]) + for r in seed_rows + if r["origin"] == "CARRIED" and r["decision"] == "SEEDED" and r["marker_id"] + ) + if self.carried_exempt_size_filter + else frozenset() + ) + # if there are any cells, filter and/or renumber - labels = self._filter_and_relabel(basins) if basins.max() > 0 else basins + labels = self._filter_and_relabel(basins, exempt) if basins.max() > 0 else basins labels = labels.astype(np.int32) # A marker pixel keeps its own label through watershed and relabeling, @@ -438,6 +459,18 @@ def _decide( markers = [r for r in seed_rows if r["marker_id"] is not None] dropped = [r for r in markers if r["basin_px"] and r["final_label"] == 0] n_dropped_small = sum(r["basin_px"] < self.min_gridpoints for r in dropped) + # Carried basins that survived only because the exemption spared them. + n_size_exempt = ( + sum( + r["origin"] == "CARRIED" + and r["decision"] == "SEEDED" + and r["final_label"] != 0 + and 0 < (r["basin_px"] or 0) < self.min_gridpoints + for r in markers + ) + if self.carried_exempt_size_filter and self.filter_by_size + else 0 + ) component_rows = [] for cid in range(1, int(components.max()) + 1): @@ -472,6 +505,7 @@ def _decide( n_dropped_small=n_dropped_small, n_dropped_large=len(dropped) - n_dropped_small, n_cells=int(labels.max()), + n_size_exempt=n_size_exempt, ) seeds = tuple(SegmentationSeed(**row) for row in seed_rows) return SegmentationDecisions(frame, seeds, tuple(component_rows)) @@ -576,7 +610,7 @@ def _admit( footprint_px=int(footprint.sum()), ) rows.append(entry) - if age + 1 > self.seed_carry_max_frames: + if age + 1 > self.seed_carry_frames: entry["decision"] = "EXPIRED" continue if not binary[row, col]: @@ -621,15 +655,28 @@ def _admit( ) return tuple(admitted), rows - def _filter_and_relabel(self, labels: np.ndarray) -> np.ndarray: - """Filter, renumber by size.""" + def _filter_and_relabel( + self, labels: np.ndarray, exempt_markers: frozenset[int] = frozenset() + ) -> np.ndarray: + """Filter, renumber by size. + + ``exempt_markers`` are basin labels the minimum-size filter must not + drop. A carried seed is only ever admitted where a previous cell lost + its core, so its basin is small by construction; deleting it undoes the + rescue and leaves the previous cell with no successor, which the tracker + then reports as a termination. Only the minimum-size test is waived — + the maximum still applies, since a runaway basin is a different fault. + """ labels_unique, counts = np.unique(labels, return_counts=True) keep_mask = labels_unique > 0 if self.filter_by_size: if self.min_gridpoints > 1: - keep_mask &= counts >= self.min_gridpoints - num_small = np.sum((labels_unique > 0) & (counts < self.min_gridpoints)) + too_small = counts < self.min_gridpoints + if exempt_markers: + too_small &= ~np.isin(labels_unique, list(exempt_markers)) + keep_mask &= ~too_small + num_small = np.sum((labels_unique > 0) & too_small) if num_small > 0: logger.debug(f"Removed {num_small} small (< {self.min_gridpoints})") diff --git a/tests/configuration/test_config_resolution.py b/tests/configuration/test_config_resolution.py index 16f4d0c..8c19452 100644 --- a/tests/configuration/test_config_resolution.py +++ b/tests/configuration/test_config_resolution.py @@ -62,11 +62,11 @@ class TestUserConfigAliases: """Test UserConfig flat aliases map correctly.""" def test_threshold_lives_in_per_method_block(self): - """threshold is now a per-method parameter, defaulting to 30 dBZ.""" + """threshold is now a per-method parameter, defaulting to 35 dBZ.""" user = UserConfig(base_dir="/tmp", radar="KHTX") config = resolve_config(ParamConfig(), user, None) - assert config.segmenter.threshold_params.threshold == 30.0 + assert config.segmenter.threshold_params.threshold == 35.0 def test_radar_id_alias(self): """radar_id flat alias maps to downloader.radar_id.""" @@ -213,8 +213,8 @@ def test_segmenter_defaults(self): user = UserConfig(base_dir="/tmp", radar="KHTX") config = resolve_config(ParamConfig(), user, None) - assert config.segmenter.threshold_params.threshold == 30.0 - assert config.segmenter.closing_kernel == (1, 1) + assert config.segmenter.threshold_params.threshold == 35.0 + assert config.segmenter.closing_radius == 2 assert config.segmenter.min_cellsize_gridpoint == 5 assert config.segmenter.filter_by_size is True diff --git a/tests/configuration/test_seed_carry_config.py b/tests/configuration/test_seed_carry_config.py index a6fb479..d167847 100644 --- a/tests/configuration/test_seed_carry_config.py +++ b/tests/configuration/test_seed_carry_config.py @@ -13,28 +13,42 @@ pytestmark = pytest.mark.unit -def test_seed_carry_is_off_by_default(): +def test_seed_carry_defaults_to_one_frame(): + """One integer governs the carry: 1 means only from the preceding frame.""" seg = SegmenterConfig() - assert seg.seed_carry is False - assert seg.seed_carry_max_frames == 2 - assert seg.seed_carry_min_separation == 3 + assert seg.seed_carry_frames == 1 + assert seg.seed_carry_min_separation == 5 + assert seg.carried_exempt_size_filter is False -@pytest.mark.parametrize("field", ["seed_carry_max_frames", "seed_carry_min_separation"]) -def test_seed_carry_bounds_below_one_are_rejected(field): +def test_seed_carry_can_be_disabled(): + assert SegmenterConfig(seed_carry_frames=0).seed_carry_frames == 0 + + +@pytest.mark.parametrize("field", ["seed_carry_frames", "seed_carry_min_separation"]) +def test_seed_carry_bounds_below_zero_are_rejected(field): + with pytest.raises(ValidationError): + SegmenterConfig(**{field: -1}) + + +def test_seed_carry_min_separation_below_one_is_rejected(): with pytest.raises(ValidationError): - SegmenterConfig(**{field: 0}) + SegmenterConfig(seed_carry_min_separation=0) + + +def test_legacy_seed_carry_pair_is_folded_into_the_integer(): + """The retired bool + max_frames pair still loads, keeping its meaning.""" + assert UserSegmenterConfig(seed_carry=False).seed_carry_frames == 0 + assert UserSegmenterConfig(seed_carry=True).seed_carry_frames == 2 + assert UserSegmenterConfig(seed_carry=True, seed_carry_max_frames=4).seed_carry_frames == 4 def test_build_config_passes_seed_carry_fields(make_config): cfg = make_config( - segmenter=UserSegmenterConfig( - seed_carry=True, seed_carry_max_frames=4, seed_carry_min_separation=5 - ) + segmenter=UserSegmenterConfig(seed_carry_frames=4, seed_carry_min_separation=5) ) det = DetectModule.build_config(cfg) - assert det.seed_carry is True - assert det.seed_carry_max_frames == 4 + assert det.seed_carry_frames == 4 assert det.seed_carry_min_separation == 5 @@ -47,11 +61,21 @@ def test_build_config_passes_seed_carry_fields(make_config): ], ) def test_build_config_raises_when_seed_carry_is_on_without_projection(make_config, selection): - cfg = make_config(segmenter=UserSegmenterConfig(seed_carry=True)).model_copy(update=selection) + cfg = make_config(segmenter=UserSegmenterConfig(seed_carry_frames=1)).model_copy( + update=selection + ) with pytest.raises(ValueError, match="projection"): DetectModule.build_config(cfg) def test_build_config_accepts_seed_carry_when_projection_is_enabled(make_config): - cfg = make_config(segmenter=UserSegmenterConfig(seed_carry=True)) - assert DetectModule.build_config(cfg).seed_carry is True + cfg = make_config(segmenter=UserSegmenterConfig(seed_carry_frames=1)) + assert DetectModule.build_config(cfg).seed_carry_frames == 1 + + +def test_legacy_closing_kernel_folds_to_a_disk_radius(): + """closing_kernel [w, h] is retired; it maps to closing_radius = max(w, h) // 2.""" + assert UserSegmenterConfig(closing_kernel=(1, 1)).closing_radius == 0 # was a no-op + assert UserSegmenterConfig(closing_kernel=(5, 5)).closing_radius == 2 + assert UserSegmenterConfig(closing_kernel=(3, 7)).closing_radius == 3 + assert UserSegmenterConfig(closing_radius=1, closing_kernel=(9, 9)).closing_radius == 1 diff --git a/tests/modules/detection/test_detection_config.py b/tests/modules/detection/test_detection_config.py index 8d68e87..1cd111b 100644 --- a/tests/modules/detection/test_detection_config.py +++ b/tests/modules/detection/test_detection_config.py @@ -17,13 +17,13 @@ def _make() -> DetectionConfig: return DetectionConfig( method="threshold", method_params={"threshold": 30.0}, - closing_kernel=(3, 3), + closing_radius=1, filter_by_size=True, min_cellsize_gridpoint=5, max_cellsize_gridpoint=None, h_maxima=3.0, - seed_carry=False, - seed_carry_max_frames=2, + seed_carry_frames=0, + carried_exempt_size_filter=True, seed_carry_min_separation=3, reflectivity_var="reflectivity", labels_var="cell_labels", @@ -36,7 +36,7 @@ def test_holds_all_required_fields(self): cfg = _make() assert cfg.method == "threshold" assert cfg.method_params == {"threshold": 30.0} - assert cfg.closing_kernel == (3, 3) + assert cfg.closing_radius == 1 assert cfg.min_cellsize_gridpoint == 5 assert cfg.max_cellsize_gridpoint is None assert cfg.h_maxima == 3.0 diff --git a/tests/modules/detection/test_segmentation_decisions.py b/tests/modules/detection/test_segmentation_decisions.py index 9e3c8f1..2bc3ebe 100644 --- a/tests/modules/detection/test_segmentation_decisions.py +++ b/tests/modules/detection/test_segmentation_decisions.py @@ -39,7 +39,14 @@ def _carried(*blocks: tuple[int, int], ages=None) -> CarriedLabels: def _segmenter(make_detection_config, **kw) -> RadarCellSegmenter: - seg_kw = {"filter_by_size": False, "h_maxima": 5.0, "seed_carry": True, **kw} + # closing_radius=0: these tests count basin pixels on synthetic squares + seg_kw = { + "filter_by_size": False, + "h_maxima": 5.0, + "seed_carry": True, + "closing_radius": 0, + **kw, + } return RadarCellSegmenter( make_detection_config(threshold=35.0, segmenter=UserSegmenterConfig(**seg_kw)) ) diff --git a/tests/modules/detection/test_segmenter_init.py b/tests/modules/detection/test_segmenter_init.py index de3fea5..b573f4e 100644 --- a/tests/modules/detection/test_segmenter_init.py +++ b/tests/modules/detection/test_segmenter_init.py @@ -14,7 +14,7 @@ def test_threshold_fixture_config(detection_module_config): """The threshold test fixture yields a threshold segmenter with expert values.""" seg = RadarCellSegmenter(detection_module_config) assert seg.method == "threshold" - assert seg.method_params["threshold"] == 30.0 + assert seg.method_params["threshold"] == 35.0 assert seg.filter_by_size is True diff --git a/tests/modules/detection/test_segmenter_morphology.py b/tests/modules/detection/test_segmenter_morphology.py index d820447..45d1bbc 100644 --- a/tests/modules/detection/test_segmenter_morphology.py +++ b/tests/modules/detection/test_segmenter_morphology.py @@ -28,7 +28,7 @@ def test_close_cells_with_closing(close_cells_ds, make_detection_config): config = make_detection_config( threshold=30, - segmenter=UserSegmenterConfig(filter_by_size=False, closing_kernel=(2, 2)), + segmenter=UserSegmenterConfig(filter_by_size=False, closing_radius=1), ) seg = RadarCellSegmenter(config) diff --git a/tests/modules/detection/test_segmenter_pyart_conv_strat.py b/tests/modules/detection/test_segmenter_pyart_conv_strat.py index 4daf61b..f1d98f4 100644 --- a/tests/modules/detection/test_segmenter_pyart_conv_strat.py +++ b/tests/modules/detection/test_segmenter_pyart_conv_strat.py @@ -42,13 +42,13 @@ def _config(method: str) -> DetectionConfig: return DetectionConfig( method=method, method_params=_method_params(method), - closing_kernel=(1, 1), + closing_radius=0, filter_by_size=True, min_cellsize_gridpoint=3, max_cellsize_gridpoint=None, h_maxima=3.0, - seed_carry=False, - seed_carry_max_frames=2, + seed_carry_frames=0, + carried_exempt_size_filter=True, seed_carry_min_separation=3, reflectivity_var="reflectivity", labels_var="cell_labels", diff --git a/tests/modules/detection/test_size_filter_exemption.py b/tests/modules/detection/test_size_filter_exemption.py new file mode 100644 index 0000000..3072734 --- /dev/null +++ b/tests/modules/detection/test_size_filter_exemption.py @@ -0,0 +1,69 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Carried-seed basins are spared by the minimum-size filter. + +A carried seed is only ever admitted where a previous cell lost its core, so +its basin is small by construction. Deleting it undoes the rescue and leaves +the previous cell with no successor, which the tracker then reports as a +termination — the filter silently cancelling the carry. Only the minimum-size +test is waived; the maximum still applies. +""" + +import numpy as np +import pytest + +from adapt.configuration.schemas.user import UserSegmenterConfig +from adapt.modules.detection.module import RadarCellSegmenter + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def basins(): + """One basin below the size floor (2 px) and one above it (9 px).""" + out = np.zeros((10, 10), dtype=np.int32) + out[0, 0:2] = 1 + out[5:8, 5:8] = 2 + return out + + +def _segmenter(make_detection_config, **kw): + return RadarCellSegmenter( + make_detection_config( + threshold=35.0, + segmenter=UserSegmenterConfig(filter_by_size=True, min_cellsize_gridpoint=5, **kw), + ) + ) + + +def test_a_small_basin_is_dropped_without_an_exemption(make_detection_config, basins): + seg = _segmenter(make_detection_config) + out = seg._filter_and_relabel(basins) + assert sorted(np.unique(out[out > 0])) == [1] + assert int((out == 1).sum()) == 9 # only the large basin survived + + +def test_an_exempt_basin_survives_the_size_floor(make_detection_config, basins): + seg = _segmenter(make_detection_config) + out = seg._filter_and_relabel(basins, frozenset({1})) + kept = sorted(np.unique(out[out > 0])) + assert kept == [1, 2] + # labels are still size-ranked, so the 9 px basin takes rank 1 + assert int((out == 1).sum()) == 9 + assert int((out == 2).sum()) == 2 + + +def test_the_maximum_size_filter_still_applies_to_exempt_basins(make_detection_config, basins): + seg = RadarCellSegmenter( + make_detection_config( + threshold=35.0, + segmenter=UserSegmenterConfig( + filter_by_size=True, min_cellsize_gridpoint=5, max_cellsize_gridpoint=4 + ), + ) + ) + out = seg._filter_and_relabel(basins, frozenset({1, 2})) + # basin 2 (9 px) exceeds the maximum and goes, basin 1 (2 px) is spared + assert sorted(np.unique(out[out > 0])) == [1] + assert int((out == 1).sum()) == 2 From ed0bc97eb93b55b75e38ea716ebdde8d5ace90e6 Mon Sep 17 00:00:00 2001 From: Bhupendra Raut Date: Tue, 22 Sep 2026 17:37:03 -0500 Subject: [PATCH 06/12] ENH: max_scan_gap_minutes and min_motion_threshold config knobs --- src/adapt/configuration/schemas/internal.py | 3 +- src/adapt/configuration/schemas/param.py | 19 ++++++---- src/adapt/configuration/schemas/user.py | 2 +- src/adapt/execution/nodes/projection.py | 2 +- src/adapt/execution/nodes/tracking.py | 2 +- src/adapt/modules/projection/config.py | 2 +- .../test_scan_gap_and_thresholds.py | 38 +++++++++++++++++++ .../projection/test_projector_validation.py | 4 +- .../modules/tracking/test_tracker_quality.py | 8 +++- 9 files changed, 63 insertions(+), 17 deletions(-) create mode 100644 tests/configuration/test_scan_gap_and_thresholds.py diff --git a/src/adapt/configuration/schemas/internal.py b/src/adapt/configuration/schemas/internal.py index 2ab3e23..a3660c4 100644 --- a/src/adapt/configuration/schemas/internal.py +++ b/src/adapt/configuration/schemas/internal.py @@ -123,6 +123,7 @@ class InternalGlobalConfig(AdaptBaseModel): """Runtime global settings.""" z_level: float + max_scan_gap_minutes: float = Field(default=10.0, gt=0.0) tracking_field: str coord_names: InternalCoordNamesConfig @@ -143,7 +144,6 @@ class InternalProjectorConfig(AdaptBaseModel): """Runtime projection configuration.""" method: str - max_time_interval_minutes: int max_projection_steps: int = Field(ge=1, le=10) # Capped at 10 nan_fill_value: float flow_params: InternalFlowParamsConfig @@ -173,7 +173,6 @@ class InternalCellUidConfig(AdaptBaseModel): split_overlap_threshold: float = Field(default=0.7, ge=0.0, le=1.0) merge_overlap_threshold: float = Field(default=0.7, ge=0.0, le=1.0) core_field_threshold: float = Field(default=40.0, ge=0.0) - max_tracking_gap_minutes: float = Field(default=20.0, gt=0.0) max_speed_ms: float = Field(default=40.0, gt=0.0) max_speed_multiplier: float = Field(default=3.0, gt=0.0) acceleration_floor_ms: float = Field(default=10.0, ge=0.0) diff --git a/src/adapt/configuration/schemas/param.py b/src/adapt/configuration/schemas/param.py index 12b6a86..75d7afc 100644 --- a/src/adapt/configuration/schemas/param.py +++ b/src/adapt/configuration/schemas/param.py @@ -182,6 +182,16 @@ class GlobalConfig(AdaptBaseModel): """Global pipeline settings.""" z_level: float = Field(2000.0, description="Analysis altitude in meters") + max_scan_gap_minutes: float = Field( + 10.0, + gt=0.0, + description=( + "The one scan-gap bound, shared by every stage that pairs frames. Beyond it " + "no seed is carried, no projection is computed and tracking resets. Set it to " + "roughly twice the nominal scan cadence; a single value keeps the stages from " + "disagreeing about whether a gap is crossable" + ), + ) tracking_field: str = Field( "reflectivity", description=( @@ -215,11 +225,10 @@ class ProjectorConfig(AdaptBaseModel): """Cell projection configuration.""" method: Literal["adapt_default"] = "adapt_default" - max_time_interval_minutes: int = Field(30, ge=1) max_projection_steps: int = Field(3, ge=1, le=10) nan_fill_value: float = 0.0 flow_params: FlowParamsConfig = Field(default_factory=FlowParamsConfig) # type: ignore[arg-type] - min_motion_threshold: float = Field(0.5, ge=0) + min_motion_threshold: float = Field(1.0, ge=0) max_flow_magnitude: float = Field( 20.0, gt=0, @@ -315,12 +324,6 @@ class CellUidConfig(AdaptBaseModel): core_field_threshold: float = Field( 40.0, ge=0.0, description="Field threshold for the core-area output (e.g. dBZ for radar)" ) - max_tracking_gap_minutes: float = Field( - 20.0, - gt=0.0, - description="Hard limit: scan gaps above this terminate all tracks and restart " - "(no matching attempted across the gap)", - ) max_speed_ms: float = Field( 40.0, gt=0.0, diff --git a/src/adapt/configuration/schemas/user.py b/src/adapt/configuration/schemas/user.py index 0d03c17..141f65d 100644 --- a/src/adapt/configuration/schemas/user.py +++ b/src/adapt/configuration/schemas/user.py @@ -108,6 +108,7 @@ class UserGlobalConfig(_UserSection): """User-facing global config.""" z_level: float | None = None + max_scan_gap_minutes: float | None = None tracking_field: str | None = None coord_names: dict[str, str] | None = None @@ -124,7 +125,6 @@ class UserProjectorConfig(_UserSection): """User-facing projector config.""" method: str | None = None - max_time_interval_minutes: int | None = None max_projection_steps: int | None = None nan_fill_value: float | None = None flow_params: dict[str, Any] | None = None diff --git a/src/adapt/execution/nodes/projection.py b/src/adapt/execution/nodes/projection.py index 73401b9..9a2ce68 100644 --- a/src/adapt/execution/nodes/projection.py +++ b/src/adapt/execution/nodes/projection.py @@ -47,7 +47,7 @@ def build_config(cls, cfg) -> ProjectionConfig: return ProjectionConfig( method=cfg.projector.method, nan_fill_value=cfg.projector.nan_fill_value, - max_time_interval_minutes=cfg.projector.max_time_interval_minutes, + max_time_interval_minutes=cfg.global_.max_scan_gap_minutes, max_projection_steps=cfg.projector.max_projection_steps, pyr_scale=cfg.projector.flow_params.pyr_scale, levels=cfg.projector.flow_params.levels, diff --git a/src/adapt/execution/nodes/tracking.py b/src/adapt/execution/nodes/tracking.py index 73b765e..4d9e191 100644 --- a/src/adapt/execution/nodes/tracking.py +++ b/src/adapt/execution/nodes/tracking.py @@ -91,7 +91,7 @@ def build_config(cls, cfg) -> TrackingConfig: uid_width=cfg.tracker.cell_uid.width, field_var=cfg.global_.tracking_field, labels_var=CELL_LABELS_VAR, - max_tracking_gap_minutes=cfg.tracker.max_tracking_gap_minutes, + max_tracking_gap_minutes=cfg.global_.max_scan_gap_minutes, max_speed_ms=cfg.tracker.max_speed_ms, max_speed_multiplier=cfg.tracker.max_speed_multiplier, acceleration_floor_ms=cfg.tracker.acceleration_floor_ms, diff --git a/src/adapt/modules/projection/config.py b/src/adapt/modules/projection/config.py index 3736e47..0aaf442 100644 --- a/src/adapt/modules/projection/config.py +++ b/src/adapt/modules/projection/config.py @@ -16,7 +16,7 @@ class ProjectionConfig(BaseModel): method: str nan_fill_value: float - max_time_interval_minutes: int + max_time_interval_minutes: float # fed from global_.max_scan_gap_minutes max_projection_steps: int pyr_scale: float levels: int diff --git a/tests/configuration/test_scan_gap_and_thresholds.py b/tests/configuration/test_scan_gap_and_thresholds.py new file mode 100644 index 0000000..343faa8 --- /dev/null +++ b/tests/configuration/test_scan_gap_and_thresholds.py @@ -0,0 +1,38 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""One scan-gap bound, and independent split/merge thresholds.""" + +import pytest + +from adapt.configuration.schemas.param import ParamConfig +from adapt.execution.nodes.projection import ProjectionModule +from adapt.execution.nodes.tracking import TrackingModule + +pytestmark = pytest.mark.unit + + +def test_scan_gap_lives_only_on_global(): + """The per-module gap fields are gone, so the stages cannot disagree.""" + param = ParamConfig() + assert param.global_.max_scan_gap_minutes == 10.0 + assert not hasattr(param.projector, "max_time_interval_minutes") + assert not hasattr(param.tracker, "max_tracking_gap_minutes") + + +def test_every_stage_reads_the_same_gap(make_config): + cfg = make_config(**{"global": {"max_scan_gap_minutes": 7.5}}) + assert cfg.global_.max_scan_gap_minutes == 7.5 + assert ProjectionModule.build_config(cfg).max_time_interval_minutes == 7.5 + assert TrackingModule.build_config(cfg).max_tracking_gap_minutes == 7.5 + + +def test_split_and_merge_thresholds_are_independent(make_config): + param = ParamConfig() + assert param.tracker.split_overlap_threshold == 0.6 + assert param.tracker.merge_overlap_threshold == 0.7 + + cfg = make_config(tracker={"split_overlap_threshold": 0.5}) + tracking = TrackingModule.build_config(cfg) + assert tracking.split_overlap == 0.5 + assert tracking.merge_overlap == 0.7 diff --git a/tests/modules/projection/test_projector_validation.py b/tests/modules/projection/test_projector_validation.py index 1307f25..1da0c17 100644 --- a/tests/modules/projection/test_projector_validation.py +++ b/tests/modules/projection/test_projector_validation.py @@ -16,9 +16,9 @@ def test_validate_requires_two_datasets(simple_labeled_ds_pair, make_projection_ def test_projection_skipped_if_time_gap_too_large(simple_labeled_ds_pair, make_projection_config): """Projection skipped when time gap exceeds max interval.""" - from adapt.configuration.schemas.user import UserProjectorConfig + from adapt.configuration.schemas.user import UserGlobalConfig - config = make_projection_config(projector=UserProjectorConfig(max_time_interval_minutes=1)) + config = make_projection_config(global_=UserGlobalConfig(max_scan_gap_minutes=1)) proj = RadarCellProjector(config) out = proj.project(simple_labeled_ds_pair) diff --git a/tests/modules/tracking/test_tracker_quality.py b/tests/modules/tracking/test_tracker_quality.py index 76df2cf..b4b5804 100644 --- a/tests/modules/tracking/test_tracker_quality.py +++ b/tests/modules/tracking/test_tracker_quality.py @@ -30,8 +30,14 @@ def _make_config(**overrides): try: param = ParamConfig() param.tracker.split_overlap_threshold = 0.4 + param.tracker.merge_overlap_threshold = 0.4 for key, val in overrides.items(): - setattr(param.tracker, key, val) + # The scan-gap bound is global now — one value shared by the seed + # carry, projection and tracking — so it no longer lives on tracker. + if key == "max_tracking_gap_minutes": + param.global_.max_scan_gap_minutes = val + else: + setattr(param.tracker, key, val) user = UserConfig(base_dir=str(Path(d)), radar="TEST_RADAR") internal = resolve_config(param, user, None) return TrackingModule.build_config(internal) From 360faea517ccccf54e9314a7a7aa23e31c1b831d Mon Sep 17 00:00:00 2001 From: Bhupendra Raut Date: Tue, 22 Sep 2026 17:37:07 -0500 Subject: [PATCH 07/12] ENH: basemap tile-server identity and cache location --- src/adapt/utils/basemap.py | 22 ++++++++++++++++++++++ tests/utils/__init__.py | 0 tests/utils/test_basemap.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 src/adapt/utils/basemap.py create mode 100644 tests/utils/__init__.py create mode 100644 tests/utils/test_basemap.py diff --git a/src/adapt/utils/basemap.py b/src/adapt/utils/basemap.py new file mode 100644 index 0000000..db28750 --- /dev/null +++ b/src/adapt/utils/basemap.py @@ -0,0 +1,22 @@ +"""Tile-server identity and cache location shared by every basemap consumer. + +Tile providers require a stable User-Agent naming the application and a +contact (OSM answers anonymous or rotating agents with 403) and require tiles +to be cached rather than re-downloaded on every run. The consumers that call +contextily pass ``tile_headers()`` on every fetch and point contextily's tile +cache at ``TILE_CACHE_DIR`` once at import. +""" + +from importlib.metadata import version +from pathlib import Path + +TILE_CACHE_DIR = Path.home() / ".adapt" / "tiles" + +_CONTACT = "https://github.com/RBhupi/Adapt" + + +def tile_headers() -> dict[str, str]: + """HTTP headers for tile requests. The key is lower-case on purpose: contextily + merges ``{"user-agent": , **headers}``, so only an identical key + replaces its anonymous agent.""" + return {"user-agent": f"Adapt/{version('arm-adapt')} (+{_CONTACT})"} diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/utils/test_basemap.py b/tests/utils/test_basemap.py new file mode 100644 index 0000000..65700fa --- /dev/null +++ b/tests/utils/test_basemap.py @@ -0,0 +1,28 @@ +"""Tile-server identity and cache location shared by every basemap consumer. + +Tile providers require a stable User-Agent that names the application and a +contact (OSM blocks anonymous or rotating agents with 403), and they require +tiles to be cached rather than re-downloaded on every run. +""" + +from importlib.metadata import version +from pathlib import Path + +from adapt.utils.basemap import TILE_CACHE_DIR, tile_headers + + +def test_user_agent_names_adapt_with_version_and_contact(): + ua = tile_headers()["user-agent"] + assert ua.startswith(f"Adapt/{version('arm-adapt')} ") + assert "https://" in ua # contact URL, as the OSM tile usage policy requires + + +def test_user_agent_overrides_contextily_default_when_merged(): + # contextily builds ``{"user-agent": , **headers}``: the same + # lower-case key must win, or the anonymous agent would still be sent. + merged = {"user-agent": "contextily-deadbeef", **tile_headers()} + assert merged["user-agent"].startswith("Adapt/") + + +def test_tile_cache_lives_in_central_adapt_directory(): + assert Path.home() / ".adapt" / "tiles" == TILE_CACHE_DIR From dda876d124df3e491133b5f085941ddf8e9ca78b Mon Sep 17 00:00:00 2001 From: Bhupendra Raut Date: Tue, 22 Sep 2026 17:37:09 -0500 Subject: [PATCH 08/12] ADD: adapt_dev conda environment for local development --- environment-dev.yml | 56 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 environment-dev.yml diff --git a/environment-dev.yml b/environment-dev.yml new file mode 100644 index 0000000..fef8624 --- /dev/null +++ b/environment-dev.yml @@ -0,0 +1,56 @@ +# Development environment for Adapt. +# +# mamba env create -f environment-dev.yml # creates `adapt_dev` +# mamba activate adapt_dev +# pip install -e . --no-deps --no-build-isolation # Adapt itself, editable +# +# Every dependency comes from conda-forge; Adapt is the only pip install and it +# is editable so source edits take effect immediately. --no-deps keeps pip from +# re-resolving anything conda already provided; --no-build-isolation reuses the +# setuptools/setuptools_scm in the env instead of fetching a build sandbox. +# +# Verified 2026-09-21: 1437 tests pass (python 3.12, arm_pyart 2.3.0, opencv 5.0). +# The 4 conv_strat_raut tests that fail under arm_pyart 2.2.0 pass here. +name: adapt_dev +channels: + - conda-forge +dependencies: + - python=3.12 + - pip + # build backend for the editable install (pyproject [build-system]) + - setuptools>=77 + - wheel + - setuptools_scm>=8 + - git # setuptools_scm reads the version from git + # runtime (pyproject [project.dependencies]) + - numpy + - xarray + - pandas + - scipy + - scikit-image + - matplotlib + - networkx + - h5py + - netcdf4 + - pydantic>=2 + - pyyaml + - arm_pyart + - s3fs>=2021.11 # pyart requires it; the solver otherwise picks 0.4.2 + - py-opencv # conda-forge name for opencv-python + - boto3 + - cmweather + - pyproj + # optional extras + - contextily # [maps] + - ffmpeg # movie consumer; its tests skip without it + - pyarrow # parquet I/O for analysis scripts + # dev (pyproject [project.optional-dependencies].dev) + - pytest>=7 + - pytest-cov + - import-linter>=2 + - pre-commit>=4 + # docs (pyproject [project.optional-dependencies].docs) + - sphinx>=7 + - pydata-sphinx-theme>=0.15 + - sphinx-autodoc-typehints + - myst-parser>=2 \ No newline at end of file From a35affe109d1f60713e1cbf660e84909fdde6005 Mon Sep 17 00:00:00 2001 From: Bhupendra Raut Date: Tue, 22 Sep 2026 17:37:25 -0500 Subject: [PATCH 09/12] ENH: max_scan_gap_minutes and min_motion_threshold config knobs (cont.) --- src/adapt/runtime/processor.py | 4 +++- src/adapt/runtime/run_reporter.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/adapt/runtime/processor.py b/src/adapt/runtime/processor.py index 1213db9..9afec1f 100644 --- a/src/adapt/runtime/processor.py +++ b/src/adapt/runtime/processor.py @@ -170,7 +170,9 @@ def __init__( # Rolling scan history (replaces per-phase segmented_history) self._scan_history: list[dict] = [] self._max_history = config.processor.max_history - self._max_time_gap_minutes = config.projector.max_time_interval_minutes + # One bound for every stage that pairs frames (see global_.max_scan_gap_minutes): + # beyond it no seed is carried, no projection runs and tracking resets. + self._max_time_gap_minutes = config.global_.max_scan_gap_minutes self._last_skipped = False # ── Lifecycle ───────────────────────────────────────────────────────────── diff --git a/src/adapt/runtime/run_reporter.py b/src/adapt/runtime/run_reporter.py index 861a528..2c63a57 100644 --- a/src/adapt/runtime/run_reporter.py +++ b/src/adapt/runtime/run_reporter.py @@ -102,7 +102,7 @@ def format_header(start: RunStart) -> str: f"· steps {c.projector.max_projection_steps}" ), "tracking": lambda c: ( - f"gap {c.tracker.max_tracking_gap_minutes:g}min " + f"gap {c.global_.max_scan_gap_minutes:g}min " f"· overlap≥{c.tracker.minimum_candidate_overlap:g} · vmax {c.tracker.max_speed_ms:g}m/s" ), } From 07624d630b2ed24279a4231d1e781ae7122937b2 Mon Sep 17 00:00:00 2001 From: Bhupendra Raut Date: Wed, 23 Sep 2026 17:39:12 -0500 Subject: [PATCH 10/12] trackerv2 --- .../schemas/collection_decisions.sql | 168 +- src/adapt/configuration/schemas/internal.py | 55 +- src/adapt/configuration/schemas/param.py | 124 +- src/adapt/contracts/__init__.py | 20 +- src/adapt/contracts/decisions.py | 195 +- src/adapt/contracts/tracking.py | 11 +- src/adapt/execution/nodes/tracking.py | 34 +- src/adapt/modules/tracking/README.md | 366 +--- src/adapt/modules/tracking/config.py | 32 +- src/adapt/modules/tracking/decision_log.py | 261 +-- src/adapt/modules/tracking/events.py | 210 +-- src/adapt/modules/tracking/graph.py | 127 -- src/adapt/modules/tracking/latent.py | 63 + src/adapt/modules/tracking/lineage.py | 71 + .../modules/tracking/matching/assignment.py | 145 +- .../modules/tracking/matching/candidate.py | 43 - src/adapt/modules/tracking/matching/gates.py | 119 ++ .../modules/tracking/matching/geometry.py | 226 +-- .../modules/tracking/matching/hungarian.py | 38 +- .../modules/tracking/matching/validation.py | 45 - src/adapt/modules/tracking/models.py | 65 +- src/adapt/modules/tracking/module.py | 1676 ++++++++--------- src/adapt/modules/tracking/motion.py | 85 - src/adapt/modules/tracking/observations.py | 318 ++++ src/adapt/persistence/decision_store.py | 15 +- src/adapt/persistence/track_store.py | 79 +- src/adapt/runtime/run_reporter.py | 3 +- .../test_full_config_roundtrip.py | 4 +- .../test_scan_gap_and_thresholds.py | 2 +- tests/configuration/test_tracker_config.py | 61 + tests/configuration/test_user_passthrough.py | 4 +- tests/contracts/test_decisions_contract.py | 46 +- tests/execution/nodes/test_tracking_node.py | 6 +- tests/modules/tracking/test_assignment.py | 96 +- .../tracking/test_candidate_generator.py | 58 - tests/modules/tracking/test_decision_log.py | 175 +- tests/modules/tracking/test_gates.py | 149 ++ tests/modules/tracking/test_geometry.py | 240 ++- .../tracking/test_hungarian_component.py | 34 +- tests/modules/tracking/test_lineage.py | 106 ++ tests/modules/tracking/test_motion.py | 38 - .../test_split_merge_normalisation.py | 64 - .../modules/tracking/test_tracker_quality.py | 176 +- .../test_tracker_scan_local_outputs.py | 12 +- tests/modules/tracking/test_tracker_v2.py | 304 +++ tests/modules/tracking/test_tracking_graph.py | 89 - tests/modules/tracking/test_validation.py | 53 - .../test_collection_provisioning.py | 10 +- tests/persistence/test_decision_store.py | 132 +- .../persistence/test_track_store_products.py | 93 + tests/validation/test_check_functions.py | 6 + 51 files changed, 3483 insertions(+), 3069 deletions(-) delete mode 100644 src/adapt/modules/tracking/graph.py create mode 100644 src/adapt/modules/tracking/latent.py create mode 100644 src/adapt/modules/tracking/lineage.py delete mode 100644 src/adapt/modules/tracking/matching/candidate.py create mode 100644 src/adapt/modules/tracking/matching/gates.py delete mode 100644 src/adapt/modules/tracking/matching/validation.py delete mode 100644 src/adapt/modules/tracking/motion.py create mode 100644 src/adapt/modules/tracking/observations.py create mode 100644 tests/configuration/test_tracker_config.py delete mode 100644 tests/modules/tracking/test_candidate_generator.py create mode 100644 tests/modules/tracking/test_gates.py create mode 100644 tests/modules/tracking/test_lineage.py delete mode 100644 tests/modules/tracking/test_motion.py delete mode 100644 tests/modules/tracking/test_split_merge_normalisation.py create mode 100644 tests/modules/tracking/test_tracker_v2.py delete mode 100644 tests/modules/tracking/test_tracking_graph.py delete mode 100644 tests/modules/tracking/test_validation.py diff --git a/src/adapt/configuration/schemas/collection_decisions.sql b/src/adapt/configuration/schemas/collection_decisions.sql index a42a6f8..7fa6956 100644 --- a/src/adapt/configuration/schemas/collection_decisions.sql +++ b/src/adapt/configuration/schemas/collection_decisions.sql @@ -5,108 +5,148 @@ -- the public API. Rows mirror the frozen record fields plus run/scan stamps. -- Config thresholds are not repeated here: join run_id to the run registry. --- user_version 2: split/merge tests carry both normalisations and their areas; --- tracking_unmatched carries the best candidate's overlaps and cost; --- segmentation_frames counts size-filter exemptions for carried cells. --- CREATE TABLE IF NOT EXISTS does not migrate an existing file — a collection --- written under version 1 keeps its old columns, so start a new base_dir when --- the new fields are needed. -PRAGMA user_version = 2; +-- user_version 3: tracker v2 (grown overlaps, curved gate, additive cost with a +-- no-link ceiling, crossing test, latent tracks, identity rule). Its records go +-- to tables with new names — tracking_scans, _cells, _pairs, _lineage, +-- _identity, _latent — so a decisions.db written by the previous tracker keeps +-- its tracking_frames/_candidates/_unmatched/_split_merge_tests rows untouched +-- and still takes new runs. CREATE TABLE IF NOT EXISTS does not migrate an +-- existing table: a changed column set needs a new table name or a new base_dir. +PRAGMA user_version = 3; -CREATE TABLE IF NOT EXISTS tracking_frames ( +CREATE TABLE IF NOT EXISTS tracking_scans ( run_id TEXT NOT NULL, scan_id TEXT NOT NULL, scan_time TEXT NOT NULL, dt_s REAL, reset_code TEXT, n_prev INTEGER NOT NULL, + n_latent INTEGER NOT NULL, n_curr INTEGER NOT NULL, n_pairs INTEGER NOT NULL, n_pass_overlap INTEGER NOT NULL, - n_pass_kinematic INTEGER NOT NULL, - n_propagated INTEGER NOT NULL, - n_hungarian INTEGER NOT NULL, + n_pass_speed INTEGER NOT NULL, + n_pass_cost INTEGER NOT NULL, + n_components INTEGER NOT NULL, + n_continue INTEGER NOT NULL, + n_crossing_excluded INTEGER NOT NULL, n_split INTEGER NOT NULL, n_merge INTEGER NOT NULL, + n_resumed INTEGER NOT NULL, + n_identity_transfer INTEGER NOT NULL, n_initiation INTEGER NOT NULL, + n_latent_created INTEGER NOT NULL, n_termination INTEGER NOT NULL, PRIMARY KEY (run_id, scan_id) ); -CREATE TABLE IF NOT EXISTS tracking_candidates ( +CREATE TABLE IF NOT EXISTS tracking_cells ( + run_id TEXT NOT NULL, + scan_id TEXT NOT NULL, + scan_time TEXT NOT NULL, + cell_label INTEGER NOT NULL, + cell_uid TEXT NOT NULL, + area_km2 REAL NOT NULL, + mass REAL NOT NULL, -- Σ e, e = excess above the cell threshold + mean_excess REAL NOT NULL, -- ē = mass / area (per pixel) + centre_x REAL NOT NULL, -- e²-weighted centre, metres + centre_y REAL NOT NULL, + score REAL NOT NULL, -- identity score S = log A + γ log ē (−inf: no excess) + core_area_km2 REAL NOT NULL, -- diagnostic, above tracker.core_field_threshold + fate TEXT NOT NULL, -- CONTINUE | RESUMED | SPLIT_CHILD | INITIATION + PRIMARY KEY (run_id, scan_id, cell_label) +); +CREATE INDEX IF NOT EXISTS idx_tracking_cells_uid ON tracking_cells (run_id, cell_uid); + +CREATE TABLE IF NOT EXISTS tracking_pairs ( run_id TEXT NOT NULL, scan_id TEXT NOT NULL, scan_time TEXT NOT NULL, prev_cell_uid TEXT NOT NULL, prev_cell_label INTEGER NOT NULL, curr_cell_label INTEGER NOT NULL, - track_steps_before INTEGER NOT NULL, - hull_area_px INTEGER NOT NULL, - hull_centroid_x REAL NOT NULL, - hull_centroid_y REAL NOT NULL, - curr_area_px INTEGER NOT NULL, - curr_centroid_x REAL NOT NULL, - curr_centroid_y REAL NOT NULL, + source TEXT NOT NULL, -- LIVE footprint or LATENT track + steps INTEGER NOT NULL, -- scan intervals spanned + growth_radius_km REAL NOT NULL, + footprint_area_px INTEGER NOT NULL, + footprint_grown_px INTEGER NOT NULL, + cell_grown_px INTEGER NOT NULL, intersection_px INTEGER NOT NULL, - opc REAL NOT NULL, - ocp REAL NOT NULL, - overlap_passed INTEGER NOT NULL, - speed_ms REAL, - previous_speed_ms REAL, - accel_cap_ms REAL, - kinematic_code TEXT, - displacement_m REAL, - length_scale_m REAL, - heading_change_deg REAL, - cost REAL, - component_n_prev INTEGER, - component_n_curr INTEGER, + o_c REAL NOT NULL, -- |H+ ∩ M+| / |M+| + o_h REAL NOT NULL, -- |H+ ∩ M+| / |H+| + u REAL NOT NULL, -- 1 − √(o_c o_h) + predicted_x REAL NOT NULL, + predicted_y REAL NOT NULL, + curr_centre_x REAL NOT NULL, + curr_centre_y REAL NOT NULL, + d REAL NOT NULL, -- shape-aware residual √(δᵀ Σ⁻¹ δ) + speed_ms REAL NOT NULL, + prev_speed_ms REAL, + speed_limit_ms REAL, -- v_prev + Δv_max; NULL on a track's first step + heading_change_deg REAL, -- NULL when a step is below two grid lengths + h REAL NOT NULL, + cost REAL NOT NULL, -- u + w_d d + h + gate TEXT NOT NULL, -- PASS | OVERLAP | SPEED | SPEED_CHANGE | COST + component_id INTEGER, cost_rank INTEGER, - match_method TEXT, - last_stage TEXT NOT NULL, - outcome TEXT NOT NULL, + margin REAL, + crossing_with_uid TEXT, + outcome TEXT NOT NULL, -- CONTINUE | RESUMED | LOST | CROSSING | REJECTED PRIMARY KEY (run_id, scan_id, prev_cell_uid, curr_cell_label) ); -CREATE INDEX IF NOT EXISTS idx_tracking_candidates_prev - ON tracking_candidates (run_id, prev_cell_uid); +CREATE INDEX IF NOT EXISTS idx_tracking_pairs_prev ON tracking_pairs (run_id, prev_cell_uid); -CREATE TABLE IF NOT EXISTS tracking_unmatched ( +CREATE TABLE IF NOT EXISTS tracking_lineage ( run_id TEXT NOT NULL, scan_id TEXT NOT NULL, scan_time TEXT NOT NULL, - side TEXT NOT NULL, + seq INTEGER NOT NULL, + side TEXT NOT NULL, -- curr orphan | prev unlinked cell cell_uid TEXT NOT NULL, cell_label INTEGER NOT NULL, - n_candidates INTEGER NOT NULL, - best_candidate_label INTEGER, - best_stage TEXT, - best_opc REAL, - best_ocp REAL, - best_cost REAL, - reason TEXT NOT NULL, - PRIMARY KEY (run_id, scan_id, side, cell_uid) + hypothesis TEXT NOT NULL, -- RESUME | SPLIT | INITIATION | MERGE | LATENT + partner_uid TEXT, + partner_label INTEGER, + fraction REAL, + cost REAL, + threshold REAL, + passed INTEGER NOT NULL, + chosen INTEGER NOT NULL, + reason TEXT, + PRIMARY KEY (run_id, scan_id, seq) ); -CREATE INDEX IF NOT EXISTS idx_tracking_unmatched_uid - ON tracking_unmatched (run_id, cell_uid); -CREATE TABLE IF NOT EXISTS tracking_split_merge_tests ( +CREATE TABLE IF NOT EXISTS tracking_identity ( run_id TEXT NOT NULL, scan_id TEXT NOT NULL, scan_time TEXT NOT NULL, - kind TEXT NOT NULL, - continuing_cell_uid TEXT NOT NULL, - tested_cell_label INTEGER NOT NULL, - overlap_fraction REAL NOT NULL, -- the fraction compared against `threshold` - threshold REAL NOT NULL, - passed INTEGER NOT NULL, - -- Both normalisations and the raw areas: MERGE decides on hull_fraction, - -- SPLIT on cell_fraction. Kept side by side so the denominator is auditable. - intersection_px INTEGER, - hull_px INTEGER, - tested_px INTEGER, - hull_fraction REAL, - cell_fraction REAL, - PRIMARY KEY (run_id, scan_id, kind, continuing_cell_uid, tested_cell_label) + kind TEXT NOT NULL, -- SPLIT | MERGE + identity_uid TEXT NOT NULL, + candidate_side TEXT NOT NULL, + candidate_label INTEGER NOT NULL, + score REAL NOT NULL, + distance REAL NOT NULL, + winner INTEGER NOT NULL, + rule TEXT NOT NULL, -- SCORE | NEAREST + transferred INTEGER NOT NULL, + PRIMARY KEY (run_id, scan_id, kind, identity_uid, candidate_side, candidate_label) +); + +CREATE TABLE IF NOT EXISTS tracking_latent ( + run_id TEXT NOT NULL, + scan_id TEXT NOT NULL, + scan_time TEXT NOT NULL, + cell_uid TEXT NOT NULL, + last_label INTEGER NOT NULL, + last_scan_id TEXT NOT NULL, + age INTEGER NOT NULL, + origin TEXT NOT NULL, -- MERGED | TERMINATED + merged_into_uid TEXT, + status TEXT NOT NULL, -- CREATED | CARRIED | RESUMED | EXPIRED + footprint_px INTEGER NOT NULL, + predicted_x REAL NOT NULL, + predicted_y REAL NOT NULL, + PRIMARY KEY (run_id, scan_id, cell_uid) ); CREATE TABLE IF NOT EXISTS segmentation_frames ( diff --git a/src/adapt/configuration/schemas/internal.py b/src/adapt/configuration/schemas/internal.py index a3660c4..e6c74f8 100644 --- a/src/adapt/configuration/schemas/internal.py +++ b/src/adapt/configuration/schemas/internal.py @@ -161,27 +161,58 @@ class InternalAnalyzerConfig(AdaptBaseModel): adjacency_min_touching_boundary_pixels: int = Field(ge=1) +# Tracker v1 keys → the v2 parameter that replaced them. A config written for +# the previous tracker must fail loudly, not run with silently ignored values. +_REMOVED_TRACKER_KEYS = { + "minimum_candidate_overlap": "max_overlap_mismatch", + "minimum_projected_overlap": "max_overlap_mismatch", + "projected_hull_buffer_km": "growth_max_km / growth_size_km", + "length_scale": "residual_weight", + "geometry_length_scale_km": "residual_weight", + "max_speed_multiplier": "max_acceleration_ms2", + "acceleration_floor_ms": "max_acceleration_ms2", + "heading_change_penalty_weight": "heading_weight", +} + + class InternalTrackerConfig(AdaptBaseModel): """Runtime tracking configuration.""" + @model_validator(mode="before") + @classmethod + def _reject_removed_keys(cls, data): + if isinstance(data, dict): + removed = sorted(k for k in data if k in _REMOVED_TRACKER_KEYS) + if removed: + hints = ", ".join(f"{k} → {_REMOVED_TRACKER_KEYS[k]}" for k in removed) + raise ValueError( + f"tracker: {hints} — these keys belong to the previous tracker; " + "replace them with the new parameters (see `adapt config`)" + ) + return data + class InternalCellUidConfig(AdaptBaseModel): """Runtime cell UID configuration.""" width: int = Field(ge=1) alphabet: Literal["base36_upper"] - split_overlap_threshold: float = Field(default=0.7, ge=0.0, le=1.0) - merge_overlap_threshold: float = Field(default=0.7, ge=0.0, le=1.0) - core_field_threshold: float = Field(default=40.0, ge=0.0) - max_speed_ms: float = Field(default=40.0, gt=0.0) - max_speed_multiplier: float = Field(default=3.0, gt=0.0) - acceleration_floor_ms: float = Field(default=10.0, ge=0.0) - heading_change_penalty_weight: float = Field(default=0.0, ge=0.0) - projected_hull_buffer_km: float = Field(default=1.0, gt=0.0) - minimum_candidate_overlap: float = Field(default=0.20, ge=0.0, le=1.0) - minimum_projected_overlap: float = Field(default=0.20, ge=0.0, le=1.0) - length_scale: Literal["hull_equiv_diameter", "sum_radii", "fixed_km"] = "hull_equiv_diameter" - geometry_length_scale_km: float = Field(default=5.0, gt=0.0) + cell_threshold: float + polarity: Literal[1, -1] + growth_max_km: float = Field(ge=0.0) + growth_size_km: float = Field(gt=0.0) + max_overlap_mismatch: float = Field(gt=0.0, le=1.0) + max_link_cost: float = Field(gt=0.0) + residual_weight: float = Field(ge=0.0) + heading_weight: float = Field(ge=0.0) + max_speed_ms: float = Field(gt=0.0) + max_acceleration_ms2: float = Field(ge=0.0) + split_overlap_threshold: float = Field(gt=0.0, le=1.0) + merge_overlap_threshold: float = Field(gt=0.0, le=1.0) + latent_scans: int = Field(ge=0) + identity_intensity_weight: float = Field(ge=0.0) + identity_score_margin: float = Field(ge=0.0) + core_field_threshold: float cell_uid: InternalCellUidConfig diff --git a/src/adapt/configuration/schemas/param.py b/src/adapt/configuration/schemas/param.py index 75d7afc..b713fa4 100644 --- a/src/adapt/configuration/schemas/param.py +++ b/src/adapt/configuration/schemas/param.py @@ -148,8 +148,8 @@ class SegmenterConfig(AdaptBaseModel): False, description="Spare carried-seed cells from min_cellsize_gridpoint. Off by default: " "on KHTX 2021-05-04 the spared basins (1-4 px) sat wholly inside the parent's " - "projected hull (Opc 1.0) but covered ~4% of it, so the tracker's " - "minimum_projected_overlap gate rejected them anyway — 60 of 63 became one-scan " + "projected hull (Opc 1.0) but covered ~4% of it, so the (v1) tracker's " + "projected-overlap gate rejected them anyway — 60 of 63 became one-scan " "orphan tracks and 3 continued. The rescue moved from the size filter to the " "tracking gate rather than succeeding. Kept as an option for study arms", ) @@ -301,80 +301,90 @@ class CellUidConfig(AdaptBaseModel): width: int = Field(10, ge=1) alphabet: Literal["base36_upper"] = "base36_upper" - split_overlap_threshold: float = Field( - 0.6, - ge=0.0, - le=1.0, - description=( - "Min fraction of the BORN cell explained by the continuing parent's " - "projected hull to confirm a SPLIT (Opc = intersection / born cell area)" - ), + cell_threshold: float = Field( + 30.0, + description="Intensity at the cell edge (field units). The excess e = polarity·(f − " + "cell_threshold)₊ weights the cell centre (by e²) and the identity score", ) - merge_overlap_threshold: float = Field( - 0.7, + polarity: Literal[1, -1] = Field( + 1, + description="+1 when cells are above the threshold (reflectivity), −1 when below " + "(brightness temperature)", + ) + growth_max_km: float = Field( + 2.0, ge=0.0, - le=1.0, - description=( - "Min fraction of the DISSIPATING cell's projected hull covered by the " - "continuing cell to confirm a MERGE (intersection / projected hull area). " - "Separate from split_overlap_threshold because the two tests normalise by " - "different areas; a single value governed both before they were split apart." - ), + description="Growth r_max (km) of the smallest footprints and cells: " + "r = r_max·(1 − √A/ℓ0)₊ evens out the one-grid-cell registration error across cell sizes", ) - core_field_threshold: float = Field( - 40.0, ge=0.0, description="Field threshold for the core-area output (e.g. dBZ for radar)" + growth_size_km: float = Field( + 12.0, gt=0.0, description="Size ℓ0 (km, √area) at and above which cells are not grown" ) - max_speed_ms: float = Field( - 40.0, + max_overlap_mismatch: float = Field( + 0.6, gt=0.0, - description="Hard physical cap (m/s); candidate pairs above this are rejected pre-matching", + le=1.0, + description="Curved overlap gate u_max: a pair is a candidate when u = 1 − √(O^c·O^h) " + "on the grown shapes is at most this", ) - max_speed_multiplier: float = Field( - 3.0, + max_link_cost: float = Field( + 1.8, gt=0.0, - description="Hard acceleration cap: reject if candidate speed exceeds this times the " - "track's mean step speed", + description="Cost ceiling c_max: the cost of leaving a cell unlinked; a link is accepted " + "only if its cost c = u + w_d·d + h is at most this", ) - acceleration_floor_ms: float = Field( - 10.0, + residual_weight: float = Field( + 0.25, ge=0.0, - description="The acceleration cap never falls below this speed (m/s): a slow prior " - "step from centroid jitter must not veto ordinary storm motion", + description="w_d on the shape-aware centre residual d; 0.25 makes w_d·d the displacement " + "over the diameter for a round cell", ) - heading_change_penalty_weight: float = Field( - 0.3, + heading_weight: float = Field( + 1.0, ge=0.0, - description="Optional cost penalty per radian of heading change (0 = diagnostic only)", + description="w_h on the heading term w_h·(1 − cos Δθ)/2, applied when both steps exceed " + "two grid lengths", ) - projected_hull_buffer_km: float = Field( - 1.0, + max_speed_ms: float = Field(40.0, gt=0.0, description="Hard speed cap v_max (m/s) of any link") + max_acceleration_ms2: float = Field( + 0.04, + ge=0.0, + description="a_max (m/s²) in the additive speed-change gate " + "v ≤ v_prev + a_max·Δt + 2√2·σ_x/Δt (σ_x = one grid length)", + ) + split_overlap_threshold: float = Field( + 0.65, gt=0.0, - description="Dilation radius (km) applied to projected hulls before candidate generation, " - "to absorb segmentation and optical-flow uncertainty", + le=1.0, + description="τ_s: an orphan is a split child when a continuing parent's grown footprint " + "covers at least this share of it (O^c)", ) - minimum_candidate_overlap: float = Field( - 0.10, - ge=0.0, + merge_overlap_threshold: float = Field( + 0.7, + gt=0.0, le=1.0, - description="Hard gate: min fraction of the candidate cell covered by the projected hull " - "(Opc = intersection / candidate area)", + description="τ_m: an unlinked cell merged into a continuing cell covering at least this " + "share of its grown footprint (O^h)", ) - minimum_projected_overlap: float = Field( - 0.10, + latent_scans: int = Field( + 2, + ge=0, + description="N: a track that ends or merges away is carried by the flow for up to N scans " + "and may be resumed; 0 or 1 ends it at once", + ) + identity_intensity_weight: float = Field( + 1.0, ge=0.0, - le=1.0, - description="Hard gate: min fraction of the projected hull covered by the candidate cell " - "(Ocp = intersection / projected hull area)", + description="γ in the identity score S = log A + γ·log ē (1: intensity mass, 0: area)", ) - length_scale: Literal["hull_equiv_diameter", "sum_radii", "fixed_km"] = Field( - "hull_equiv_diameter", - description="Characteristic length L that normalises centroid displacement in the " - "geometry-first cost (cost = m + d/L). Swappable for experimentation", + identity_score_margin: float = Field( + 0.33, + ge=0.0, + description="δ_S: scores closer than this are within noise and the identity follows the " + "candidate nearer the predicted position", ) - geometry_length_scale_km: float = Field( - 5.0, - gt=0.0, - description="Fixed characteristic length (km) used only when length_scale='fixed_km'", + core_field_threshold: float = Field( + 40.0, description="Field threshold for the diagnostic core-area output (not a decision)" ) cell_uid: CellUidConfig = Field(default_factory=CellUidConfig) # type: ignore[arg-type] diff --git a/src/adapt/contracts/__init__.py b/src/adapt/contracts/__init__.py index 980c9b5..0478e63 100644 --- a/src/adapt/contracts/__init__.py +++ b/src/adapt/contracts/__init__.py @@ -30,11 +30,13 @@ SegmentationDecisions, SegmentationFrame, SegmentationSeed, - TrackingCandidate, + TrackingCell, TrackingDecisions, - TrackingFrame, - TrackingSplitMergeTest, - TrackingUnmatched, + TrackingIdentity, + TrackingLatent, + TrackingLineage, + TrackingPair, + TrackingScan, check_segmentation_decisions, check_tracking_decisions, ) @@ -82,10 +84,12 @@ "PersistenceMeta", "ScanRecord", # decision records — analysis-only, written to decisions.db - "TrackingFrame", - "TrackingCandidate", - "TrackingUnmatched", - "TrackingSplitMergeTest", + "TrackingScan", + "TrackingCell", + "TrackingPair", + "TrackingLineage", + "TrackingIdentity", + "TrackingLatent", "TrackingDecisions", "SegmentationFrame", "SegmentationSeed", diff --git a/src/adapt/contracts/decisions.py b/src/adapt/contracts/decisions.py index 3d57fda..2d830fc 100644 --- a/src/adapt/contracts/decisions.py +++ b/src/adapt/contracts/decisions.py @@ -18,112 +18,162 @@ @dataclass(frozen=True) -class TrackingFrame: - """One scan pair: how the frame was classified and what each stage yielded.""" +class TrackingScan: + """One scan pair: how it was classified and what each step yielded.""" dt_s: float | None reset_code: str | None # FIRST_SCAN | NON_MONOTONIC_TIME | TRACK_GAP_EXCEEDED | NO_PROJECTIONS n_prev: int + n_latent: int # latent tracks offered to this scan n_curr: int n_pairs: int n_pass_overlap: int - n_pass_kinematic: int - n_propagated: int - n_hungarian: int + n_pass_speed: int + n_pass_cost: int + n_components: int + n_continue: int + n_crossing_excluded: int n_split: int n_merge: int + n_resumed: int + n_identity_transfer: int n_initiation: int + n_latent_created: int n_termination: int @dataclass(frozen=True) -class TrackingCandidate: - """One (previous cell, current cell) pair that candidate generation produced. +class TrackingCell: + """One current cell: the quantities the decisions used, and its fate. + + ``mass`` is Σe and ``mean_excess`` ē over the cell (e = excess above the + cell threshold); the centre is the e²-weighted one (metres); ``score`` is + the identity score S. ``core_area_km2`` is diagnostic only. + """ + + cell_label: int + cell_uid: str + area_km2: float + mass: float + mean_excess: float + centre_x: float + centre_y: float + score: float + core_area_km2: float + fate: str # CONTINUE | RESUMED | SPLIT_CHILD | INITIATION + + +@dataclass(frozen=True) +class TrackingPair: + """One candidate pair: a footprint (live or latent) against a current cell. - ``last_stage`` is the furthest stage the pair reached (OVERLAP, KINEMATIC, - ASSIGNMENT); ``outcome`` is CONTINUE, REJECTED_OVERLAP, REJECTED_KINEMATIC - or LOST_ASSIGNMENT. Fields of stages the pair never reached are None. + Every quantity is computed for every pair; ``gate`` names the first check + it failed (OVERLAP, SPEED, SPEED_CHANGE, COST) or PASS. ``outcome`` is + CONTINUE, RESUMED, LOST (passed, not chosen), CROSSING (excluded because it + crossed ``crossing_with_uid``'s link) or REJECTED (failed a gate). """ prev_cell_uid: str prev_cell_label: int curr_cell_label: int - track_steps_before: int - hull_area_px: int - hull_centroid_x: float - hull_centroid_y: float - curr_area_px: int - curr_centroid_x: float - curr_centroid_y: float + source: str # LIVE | LATENT + steps: int # scan intervals the link spans + growth_radius_km: float + footprint_area_px: int + footprint_grown_px: int + cell_grown_px: int intersection_px: int - opc: float - ocp: float - overlap_passed: bool - speed_ms: float | None - previous_speed_ms: float | None - accel_cap_ms: float | None - kinematic_code: str | None - displacement_m: float | None - length_scale_m: float | None + o_c: float + o_h: float + u: float + predicted_x: float + predicted_y: float + curr_centre_x: float + curr_centre_y: float + d: float + speed_ms: float + prev_speed_ms: float | None + speed_limit_ms: float | None heading_change_deg: float | None - cost: float | None - component_n_prev: int | None - component_n_curr: int | None + h: float + cost: float + gate: str + component_id: int | None cost_rank: int | None - match_method: str | None - last_stage: str + margin: float | None + crossing_with_uid: str | None outcome: str @dataclass(frozen=True) -class TrackingUnmatched: - """A previous cell that did not continue, or a current cell that was born.""" +class TrackingLineage: + """One hypothesis for a cell the assignment left over, and whether it was chosen. + + Current orphans (side ``curr``) weigh RESUME, SPLIT and INITIATION; + unlinked previous cells (side ``prev``) weigh MERGE and LATENT. + ``fraction`` is O^c for a split and O^h for a merge; ``cost`` is the + resumption cost; ``reason`` says why a hypothesis failed, or IDENTITY when + the identity rule made the assigned cell a split child or merge source. + """ - side: str # prev | curr + side: str cell_uid: str cell_label: int - n_candidates: int - best_candidate_label: int | None - best_stage: str | None - reason: str # NO_CANDIDATE | ALL_REJECTED_OVERLAP | ALL_REJECTED_KINEMATIC | - # LOST_ASSIGNMENT | SPLIT_CHILD | MERGE_SOURCE | RESET | FIRST_SCAN - # How close the best candidate came, so a near-miss is distinguishable from - # no candidate at all without re-joining tracking_candidates. - best_opc: float | None = None - best_ocp: float | None = None - best_cost: float | None = None + hypothesis: str + partner_uid: str | None + partner_label: int | None + fraction: float | None + cost: float | None + threshold: float | None + passed: bool + chosen: bool + reason: str | None @dataclass(frozen=True) -class TrackingSplitMergeTest: - """Every hull overlap tested — pass or fail. +class TrackingIdentity: + """One contender for an identity contested by a split or a merge. - MERGE compares ``hull_fraction`` against the merge threshold, SPLIT compares - ``cell_fraction`` against the split one; ``overlap_fraction`` always holds - whichever was used for the decision. + ``identity_uid`` is the identity at stake; ``candidate_label`` is a previous + cell (merge) or a current cell (split). ``transferred`` is True when the + winner is not the cell the assignment linked. """ kind: str # SPLIT | MERGE - continuing_cell_uid: str - tested_cell_label: int - overlap_fraction: float # the fraction actually compared against `threshold` - threshold: float - passed: bool - # Both normalisations plus the raw areas, so the choice of denominator stays - # auditable: MERGE decides on `hull_fraction`, SPLIT on `cell_fraction`. - intersection_px: int | None = None - hull_px: int | None = None - tested_px: int | None = None - hull_fraction: float | None = None - cell_fraction: float | None = None + identity_uid: str + candidate_side: str # prev | curr + candidate_label: int + score: float + distance: float + winner: bool + rule: str # SCORE | NEAREST + transferred: bool + + +@dataclass(frozen=True) +class TrackingLatent: + """A latent track at this scan: created, carried, resumed or expired.""" + + cell_uid: str + last_label: int + last_scan_id: str + age: int # scans since the last observation + origin: str # MERGED | TERMINATED + merged_into_uid: str | None + status: str # CREATED | CARRIED | RESUMED | EXPIRED + footprint_px: int + predicted_x: float + predicted_y: float @dataclass(frozen=True) class TrackingDecisions: - frame: TrackingFrame - candidates: tuple[TrackingCandidate, ...] - unmatched: tuple[TrackingUnmatched, ...] - split_merge_tests: tuple[TrackingSplitMergeTest, ...] + scan: TrackingScan + cells: tuple[TrackingCell, ...] + pairs: tuple[TrackingPair, ...] + lineage: tuple[TrackingLineage, ...] + identity: tuple[TrackingIdentity, ...] + latent: tuple[TrackingLatent, ...] # ── Segmentation ───────────────────────────────────────────────────────────── @@ -207,15 +257,18 @@ def check_tracking_decisions(decisions: TrackingDecisions) -> None: require( isinstance(decisions, TrackingDecisions), "tracking_decisions must be TrackingDecisions" ) - frame = decisions.frame + scan = decisions.scan + require( + scan.n_termination <= scan.n_prev + scan.n_latent, + "tracking_decisions: more terminations than previous and latent tracks", + ) require( - frame.n_termination <= frame.n_prev, - "tracking_decisions: more terminations than previous cells", + len(decisions.pairs) == scan.n_pairs, + f"tracking_decisions: {len(decisions.pairs)} pair rows for {scan.n_pairs} pairs", ) require( - len(decisions.candidates) == frame.n_pairs, - f"tracking_decisions: {len(decisions.candidates)} candidate rows for " - f"{frame.n_pairs} generated pairs", + len(decisions.cells) == scan.n_curr, + f"tracking_decisions: {len(decisions.cells)} cell rows for {scan.n_curr} current cells", ) diff --git a/src/adapt/contracts/tracking.py b/src/adapt/contracts/tracking.py index 6a95be6..6cd8c19 100644 --- a/src/adapt/contracts/tracking.py +++ b/src/adapt/contracts/tracking.py @@ -36,7 +36,16 @@ "event_group_id", ] -_VALID_EVENT_TYPES = {"CONTINUE", "SPLIT", "MERGE", "INITIATION", "TERMINATION"} +# LATENT: a track that vanished is kept for possible resumption; RESUMED: it was. +_VALID_EVENT_TYPES = { + "CONTINUE", + "SPLIT", + "MERGE", + "INITIATION", + "TERMINATION", + "LATENT", + "RESUMED", +} def assert_tracked_cells(df: pd.DataFrame) -> None: diff --git a/src/adapt/execution/nodes/tracking.py b/src/adapt/execution/nodes/tracking.py index 4d9e191..a376da0 100644 --- a/src/adapt/execution/nodes/tracking.py +++ b/src/adapt/execution/nodes/tracking.py @@ -40,7 +40,8 @@ class TrackingModule(BaseModule): tracked_cells : pd.DataFrame Per-cell observations for the current scan with cell_uid/cell_label. cell_events : pd.DataFrame - Explicit event rows for CONTINUE, SPLIT, MERGE, INITIATION, TERMINATION. + Explicit event rows for CONTINUE, SPLIT, MERGE, INITIATION, TERMINATION, + LATENT (a track kept for resumption) and RESUMED. analysis_ds : xr.Dataset ``projected_ds`` plus the cell_uid LUTs (``cell_uid`` for this scan's labels, ``registration_cell_uid`` for the previous scan's) — the dataset @@ -84,23 +85,28 @@ class TrackingModule(BaseModule): @classmethod def build_config(cls, cfg) -> TrackingConfig: + t = cfg.tracker return TrackingConfig( - split_overlap=cfg.tracker.split_overlap_threshold, - merge_overlap=cfg.tracker.merge_overlap_threshold, - core_field_threshold=cfg.tracker.core_field_threshold, - uid_width=cfg.tracker.cell_uid.width, + uid_width=t.cell_uid.width, field_var=cfg.global_.tracking_field, labels_var=CELL_LABELS_VAR, max_tracking_gap_minutes=cfg.global_.max_scan_gap_minutes, - max_speed_ms=cfg.tracker.max_speed_ms, - max_speed_multiplier=cfg.tracker.max_speed_multiplier, - acceleration_floor_ms=cfg.tracker.acceleration_floor_ms, - heading_change_penalty_weight=cfg.tracker.heading_change_penalty_weight, - projected_hull_buffer_km=cfg.tracker.projected_hull_buffer_km, - minimum_candidate_overlap=cfg.tracker.minimum_candidate_overlap, - minimum_projected_overlap=cfg.tracker.minimum_projected_overlap, - length_scale=cfg.tracker.length_scale, - geometry_length_scale_km=cfg.tracker.geometry_length_scale_km, + cell_threshold=t.cell_threshold, + polarity=t.polarity, + growth_max_km=t.growth_max_km, + growth_size_km=t.growth_size_km, + max_overlap_mismatch=t.max_overlap_mismatch, + max_link_cost=t.max_link_cost, + residual_weight=t.residual_weight, + heading_weight=t.heading_weight, + max_speed_ms=t.max_speed_ms, + max_acceleration_ms2=t.max_acceleration_ms2, + split_overlap=t.split_overlap_threshold, + merge_overlap=t.merge_overlap_threshold, + latent_scans=t.latent_scans, + identity_intensity_weight=t.identity_intensity_weight, + identity_score_margin=t.identity_score_margin, + core_field_threshold=t.core_field_threshold, ) def __init__(self) -> None: diff --git a/src/adapt/modules/tracking/README.md b/src/adapt/modules/tracking/README.md index c5dfd2c..124846c 100644 --- a/src/adapt/modules/tracking/README.md +++ b/src/adapt/modules/tracking/README.md @@ -1,322 +1,74 @@ # Storm Cell Tracking Module -**Author**: Adapt Development Team -**Status**: Production-ready **Module Name**: `tracking` -## Overview - -The tracking module performs tracking-only association of segmented radar cells across consecutive scans using projected mask overlap and a multi-term matching cost. It emits scan-local tracking observations, explicit lineage events, and same-scan adjacency translated into track identity space. - -## Features - -- **Registration-driven**: matches against projected (advected) cell hulls -- **Geometry-first, field-agnostic**: bidirectional overlap + `m + d/L` cost; the field is only a centroid weight -- **Constraint propagation before optimisation**: Hungarian only for ambiguous connected components -- **Graph-Based Lineage**: stores complete tracking history as a directed graph -- **Explicit Events**: emits CONTINUE, SPLIT, MERGE, INITIATION, TERMINATION rows per scan - -## Architecture - -The scientific layer is decomposed into focused, single-responsibility files under -`adapt/modules/tracking/`. `CellTracker` (in `module.py`) is orchestration only; -it delegates to: - -The tracker is **geometry-first** and **field-agnostic**: the pixel field (reflectivity, -brightness temperature, vertical wind, …) is used only as a centroid weight, never -assumed to be reflectivity. Association is resolved as progressively stronger -constraints with optimisation as the last resort. +Gives each segmented cell an identity that persists from scan to scan and records +where tracks begin, end, split, merge, pause and resume. Causal: a scan's decisions +use only that scan and earlier ones and are never revised. Field-agnostic: the field +enters only through the excess `e = polarity·(f − cell_threshold)₊`. + +## Algorithm (per scan pair) + +1. **Predict.** Footprint `H_i` = previous cell advected by the dense flow + (`cell_projections[0]`). Predicted centre = last centre moved as the footprint moved. +2. **Grow** footprint and every cell by `r = r_max (1 − √A/ℓ0)₊` (distance transform, + from the boundary). Candidates: grown shapes intersect. +3. **Overlap** on the grown shapes: `O^c = I/|M⁺|`, `O^h = I/|H⁺|`, + `u = 1 − √(O^c O^h)`; curved gate `u ≤ u_max`. +4. **Position** = e²-weighted centre; residual `d = √(δᵀ Σ⁻¹ δ)` with `Σ` the + footprint's second moment; heading `h = w_h (1 − cos Δθ)/2` when both steps exceed + two grid lengths. +5. **Hard gates**: `v ≤ v_max`; `v ≤ v_prev + a_max Δt + 2√2 σ_x/Δt` (σ_x = grid length). +6. **Cost** `c = u + w_d d + h ≤ c_max`. Joint assignment per group of competing cells + (Hungarian) with a no-link option at `c_max`; links that cross are never chosen + (the costlier of a crossing pair is excluded), then a scan-wide crossing check. +7. **Lineage** of what is left: an orphan resumes a latent track (same gates and cost + over the elapsed time, no crossing in any intervening interval), else is a split + child (`O^c ≥ τ_s`), else an initiation. An unlinked track is a merge source + (`O^h ≥ τ_m`) or not; either way it becomes **latent** for up to `N` scans, carried by + the mean flow inside its footprint, and terminates when not resumed. +8. **Identity** at splits and merges: larger `S = log A + γ log ē`; within `δ_S`, the + candidate nearer the predicted position. + +## Files | File | Responsibility | |------|----------------| -| `module.py` | `CellTracker` — per-scan flow, state, delegation | -| `graph.py` | `TrackingGraph` (the only `networkx` home) | -| `matching/geometry.py` | overlap (Opc/Ocp), hull dilation, length-scale strategies, `pair_cost`, centroids | -| `matching/candidate.py` | `CandidateGenerator` — buffered high-recall candidate pairs | -| `matching/validation.py` | `GeometricValidator` — bidirectional-overlap hard gate | -| `matching/assignment.py` | `ConstraintPropagator` + `AssignmentGraph` (connected components) | -| `matching/hungarian.py` | `HungarianMatcher` — per-component assignment (the only `scipy` optimisation home) | -| `motion.py` | `MotionValidator` (speed/accel caps) + heading-change helpers | -| `models.py` | `MatchMethod` / `TrackingError` enums, `MatchDiagnostics`, `TrackMotionState` | -| `identity.py` | stable `cell_uid` generation | -| `events.py` | lineage event-row builders + diagnostics assembly | +| `module.py` | `CellTracker` — per-scan flow and state (live tracks, latent tracks, recent intervals) | +| `observations.py` | grid, cells, tracks, footprints; evidence and first failed gate of every pair | +| `matching/geometry.py` | growth, grown overlaps, excess/mass, e²-centre, second moment, Mahalanobis | +| `matching/gates.py` | speed gates, heading term, segment crossing, drop-the-costlier rule | +| `matching/assignment.py` | components, per-component assignment with crossing exclusion and margins | +| `matching/hungarian.py` | Hungarian with a no-link column (the only `scipy` optimisation home) | +| `lineage.py` | split/merge partner choice, identity score and rule | +| `latent.py` | `LatentTrack`, footprint carry by the flow | +| `events.py` | `cell_events` rows | +| `decision_log.py` | `ScanLog` → `TrackingDecisions` (decisions.db) | +| `identity.py`, `lut.py` | cell uids; label → uid lookup tables | | `config.py` | frozen `TrackingConfig` | -The node layer (`adapt/execution/nodes/tracking.py`) keeps the `BaseModule` wrapper, -`registry.register`, `build_config`, contracts, and persistence specs — no engine -imports ever live under `modules/tracking/`. - -### Matching hierarchy - -Each frame pair is resolved in this order (registration-driven, optimisation last): - -``` -scan-gap classification (physical time; hard reset on excess gap / non-monotonic time) - ↓ -registration projected hulls (previous labels advected by the full flow step) - ↓ -dilate hulls by projected_hull_buffer_km → liberal candidate pairs (high recall) - ↓ -hard gate: bidirectional overlap (Opc, Ocp) + kinematic speed/acceleration caps - ↓ -deterministic constraint propagation (mutually-unique matches → PROPAGATED) - ↓ -connected components → Hungarian inside ambiguous groups only (→ HUNGARIAN) - ↓ -split / merge detection - ↓ -initiation / termination -``` - -Bidirectional overlap — `Opc = |I|/|candidate|` and `Ocp = |I|/|projected hull|`, both -required above their thresholds — rejects tiny cells inside large hulls, merged cells -engulfing a prediction, and grazing contacts that a one-sided fraction or IoU would pass. -The buffer widens candidate *recall* only; the gate and cost use the un-buffered hull. - -## Usage - -### As Part of Pipeline - -```python -# Automatic via pipeline DAG -# tracking module runs after projection module - -# Context inputs: -# - projected_ds: xr.Dataset (from ProjectionModule) -# - cell_stats: pd.DataFrame (from AnalysisModule) -# - cell_adjacency: pd.DataFrame (from AnalysisModule) -# - config: InternalConfig -# -# Context outputs: -# - tracked_cells: pd.DataFrame -# - track_events: pd.DataFrame -# - tracked_cell_adjacency: pd.DataFrame -``` - -### Standalone - -```python -from adapt.modules.tracking.module import CellTracker -from adapt.schemas import init_runtime_config - -# Initialize -config = init_runtime_config(user_config) -tracker = CellTracker(config) - -# Track one scan at a time (scan-local outputs) -tracked_cells, cell_events = tracker.track(projected_ds, cell_stats) -``` - -## Configuration - -Defaults live in `adapt.configuration.schemas.param.TrackerConfig`. Key knobs: - -```python -projected_hull_buffer_km: float = 1.0 # dilation radius for candidate recall -minimum_candidate_overlap: float = 0.20 # Opc gate -minimum_projected_overlap: float = 0.20 # Ocp gate -length_scale: str = "hull_equiv_diameter" # hull_equiv_diameter | sum_radii | fixed_km -geometry_length_scale_km: float = 5.0 # used only when length_scale == "fixed_km" -max_tracking_gap_minutes: float = 20.0 # hard reset when a scan gap exceeds this -max_speed_ms: float = 40.0 # kinematic velocity cap -max_speed_multiplier: float = 3.0 # kinematic acceleration cap -heading_change_penalty_weight: float = 0.0 # optional crossing-track penalty -split_overlap_threshold: float = 0.8 # SPLIT / MERGE overlap threshold -core_field_threshold: float = 40.0 # core-area output threshold (field units) -``` - -Override in user config: - -```yaml -tracker: - minimum_candidate_overlap: 0.25 - length_scale: sum_radii - max_speed_ms: 35.0 -``` - -## Data Outputs - -### Tracked Cells - -One row per tracked cell observation in the current scan: - -| Column | Type | Description | -|--------|------|-------------| -| `time` | datetime64 | Observation timestamp | -| `track_index` | int | Deterministic track index (starts at 1) | -| `track_id` | str | Deterministic UUID (derived from run_id + track_index) | -| `cell_label` | int | Cell label from segmentation | -| `area` | float | Cell area (km²) | -| `centroid_x`, `centroid_y` | float | Cell center coordinates | -| `mean_reflectivity` | float | Average dBZ | -| `max_reflectivity` | float | Peak dBZ | -| `core_area` | float | Area above core threshold (km²) | -| `n_connected_cells` | int | Number of adjacent tracked neighbors in this scan | -| `connected_track_ids_json` | str | JSON list of adjacent `track_id` values | - -### Track Events - -Explicit lineage/event rows for the current scan: - -| Column | Type | Description | -|--------|------|-------------| -| `time` | datetime64 | Scan timestamp | -| `event_type` | str | CONTINUE \| SPLIT \| MERGE \| INITIATION \| TERMINATION | -| `source_track_id` | str or None | Source track (if applicable) | -| `target_track_id` | str or None | Target track (if applicable) | -| `cost` | float or None | Matching cost (CONTINUE only in v1) | - -### Tracked Cell Adjacency - -Normalized adjacency pairs in track identity space: - -| Column | Type | Description | -|--------|------|-------------| -| `time` | datetime64 | Scan timestamp | -| `track_id_a`, `track_id_b` | str | Adjacent tracks in this scan | -| `touching_boundary_pixels` | int | Boundary-touch count from analysis | - -## Algorithm Details +## Outputs -### Cost Function (geometry-only, `matching/geometry.py`) +- `tracked_cells`: one row per cell (`cell_uid`, `cell_label`, area, e²-centre, field stats, core area). +- `cell_events`: CONTINUE, SPLIT, MERGE, INITIATION, TERMINATION, **LATENT** (track kept + for resumption) and **RESUMED**. `match_method` is ASSIGNED, IDENTITY_MERGE or + IDENTITY_SPLIT (identity transferred by the score rule) or RESUMED; + `source_scan_id` is the scan the source was last observed in. +- `tracking_decisions` → decisions.db: `tracking_scans`, `tracking_cells`, + `tracking_pairs` (every pair, every number, first failed gate, component, rank, + margin, crossing), `tracking_lineage` (every hypothesis), `tracking_identity`, + `tracking_latent`. -The cost is dimensionless and scale-invariant so the same tracker behaves -consistently across radar, satellite, and cloud-mask inputs: +## Configuration (`tracker:`) -``` -cost = m + d / L + heading_penalty -m = 1 − √Opc · √Ocp # overlap mismatch, in [0, 1] -d = ‖ mass_centroid(candidate) − centroid(projected hull) ‖ # metres (prediction residual) -``` - -`√Opc·√Ocp` spreads near-threshold overlaps and compresses near-perfect ones. `d` is -normalised by a **configurable characteristic length** `L` (`length_scale`), so both -terms are O(1) with no invented weights: - -| `length_scale` | L | -|----------------|---| -| `hull_equiv_diameter` (default) | `2·√(|hull|·pixel_area/π)` | -| `sum_radii` | `√(|hull|·pixel_area/π) + √(|cell|·pixel_area/π)` | -| `fixed_km` | `geometry_length_scale_km · 1000` | - -When `heading_change_penalty_weight > 0`, a soft `weight · Δheading` (radians) term is -added for tracks with an established velocity (crossing-track prevention). - -### Assignment - -Most pairs are resolved by **constraint propagation** (mutually-unique matches, no -optimisation → `PROPAGATED`). Only genuinely ambiguous connected components reach -`scipy.optimize.linear_sum_assignment`, run **per component** — never one global matrix -(→ `HUNGARIAN`). - -### Search Region - -Candidates come from the registration projected hull — `cell_projections[0]`, -the previous scan's labels advected by the full flow step to the current scan -time (the minute-resolution `registration_minutes` frames stop short of it by -up to a minute) — dilated by `projected_hull_buffer_km` for recall -(`matching/candidate.py`). - -### Diagnostics - -Every accepted match records a `MatchDiagnostics` row persisted to `cell_events`: -`candidate_opc`, `candidate_ocp`, `candidate_centroid_distance_m`, -`candidate_speed_ms`, `candidate_heading_change_deg`, `candidate_area_ratio`, -`candidate_final_cost`, and `match_method` -(`PROPAGATED` / `HUNGARIAN` / `SPLIT` / `MERGE`). - -## Testing - -Run behavior-driven tests: - -```bash -pytest tests/modules/tracking/ -v -``` - -Tests cover: -- Linear motion tracking -- Cell growth and decay -- New cell birth and death -- Crossing tracks -- Graph structure -- Cost function -- DataFrame outputs - -All tests use synthetic data for reproducibility. - -## Dependencies - -- `numpy`: Array operations -- `pandas`: DataFrame outputs -- `xarray`: Dataset handling -- `scipy`: Hungarian assignment -- `networkx`: Tracking graph storage - -## Performance - -Typical performance for 50 cells per scan: -- **Tracking time**: 10-50 ms per frame pair -- **Memory usage**: ~10 MB for 100 scans -- **Graph size**: Linear with total cell-observations - -## Tracking-Assisted Segmentation Correction (design only — not implemented) - -Projected hulls carry motion-coherence information that can flag segmentation -errors. This is a **designed extension point**, intentionally left unimplemented -(YAGNI) until a concrete use case exists. No hooks or dead code are present today. - -**Motivating cases** - -- *Over-split.* Segmentation fragments one storm into several cells, but a single - continuing parent's projected hull covers all fragments coherently → the - fragments should be re-merged into one tracked object. -- *Over-merge.* Segmentation fuses two storms into one cell, but two distinct - parents project into separable sub-regions → the cell should be re-split. - -**Where it would hook in.** A correction stage would sit between *registration -projected hulls* and *deterministic unique-overlap matching* in the hierarchy -above — i.e. it adjusts the current-frame label field *before* matching, so all -downstream stages operate on the corrected segmentation. It would consume the -same `cell_projections[0]` registration hull plus the per-parent overlap structure -already computed by `OverlapMatcher`. - -**Proposed shape when built.** A registered, swappable -`SegmentationCorrector` strategy (Open/Closed, like detection/tracking backends) -implemented under `modules/`, communicating via a `contracts/` interface, returning -a corrected label array + provenance describing each merge/split it applied. It -must be deterministic and produce no side effects. The correction provenance would -travel as additional diagnostic rows so every change is traceable. - -## Future Enhancements - -Potential improvements identified during development: - -1. **Advanced Split/Merge Logic**: Implement temporary merge identity restoration -2. **Track Smoothing**: Apply Kalman filtering to motion vectors -3. **Motion-coherent segmentation correction**: see the design section above -4. **Parallel Processing**: Process multiple files concurrently -5. **Persistence**: Save/load tracking graph for resumable processing +`cell_threshold`, `polarity`, `growth_max_km` (r_max), `growth_size_km` (ℓ0), +`max_overlap_mismatch` (u_max), `max_link_cost` (c_max), `residual_weight` (w_d), +`heading_weight` (w_h), `max_speed_ms` (v_max), `max_acceleration_ms2` (a_max), +`split_overlap_threshold` (τ_s), `merge_overlap_threshold` (τ_m), `latent_scans` (N), +`identity_intensity_weight` (γ), `identity_score_margin` (δ_S), `core_field_threshold` +(diagnostic only). Keys of the previous tracker fail loudly, naming their replacement. ## References -- **Hungarian Algorithm**: Kuhn, H. W. (1955). "The Hungarian method for the assignment problem" -- **Storm Tracking**: Dixon & Wiener (1993). "TITAN: Thunderstorm Identification, Tracking, Analysis, and Nowcasting" -- **Optical Flow**: Farnebäck, G. (2003). "Two-Frame Motion Estimation Based on Polynomial Expansion" - -## Developer Notes - -This module was implemented as the **first additional default module** in the Adapt system. For detailed developer experience and implementation patterns, see `MODULE_EXTENSION_GUIDE.md` in the repository root. - -Key learnings: -- The two-layer pattern (scientific class + wrapper) works extremely well -- Pydantic config schemas eliminate runtime validation complexity -- Contract validators provide fail-fast guarantees -- Behavior-driven tests with synthetic data are robust and fast - -## License - -Same as Adapt main project. - -## Contact - -For questions or issues, open a GitHub issue in the Adapt repository. +Kuhn (1955); Jaqaman et al. (2008), *Nature Methods*; Dixon & Wiener (1993), TITAN; +Raut et al. (2021), *JAMC*. diff --git a/src/adapt/modules/tracking/config.py b/src/adapt/modules/tracking/config.py index 2711bc0..cb63603 100644 --- a/src/adapt/modules/tracking/config.py +++ b/src/adapt/modules/tracking/config.py @@ -3,8 +3,8 @@ """Tracking module config schema. -Holds exactly the fields CellTracker consumes (geometry-first gate/buffer/ -length-scale params + cell_uid params flattened). Built once at startup by +Holds exactly the fields CellTracker consumes (tracker v2 rule parameters ++ cell_uid params flattened). Built once at startup by TrackingModule.build_config() from the resolved InternalConfig. Frozen. """ @@ -14,19 +14,23 @@ class TrackingConfig(BaseModel): model_config = ConfigDict(frozen=True) - split_overlap: float # Opc gate on the born cell (SPLIT) - merge_overlap: float # hull-coverage gate on the dissipating cell (MERGE) - core_field_threshold: float uid_width: int field_var: str labels_var: str max_tracking_gap_minutes: float - max_speed_ms: float - max_speed_multiplier: float - acceleration_floor_ms: float - heading_change_penalty_weight: float - projected_hull_buffer_km: float - minimum_candidate_overlap: float - minimum_projected_overlap: float - length_scale: str - geometry_length_scale_km: float + cell_threshold: float # f_thr: excess e = polarity·(f − f_thr)₊ + polarity: int # +1 above threshold (reflectivity), −1 below (brightness temperature) + growth_max_km: float # r_max + growth_size_km: float # ℓ0 + max_overlap_mismatch: float # u_max + max_link_cost: float # c_max + residual_weight: float # w_d + heading_weight: float # w_h + max_speed_ms: float # v_max + max_acceleration_ms2: float # a_max + split_overlap: float # τ_s on O^c + merge_overlap: float # τ_m on O^h + latent_scans: int # N + identity_intensity_weight: float # γ + identity_score_margin: float # δ_S + core_field_threshold: float # diagnostic core area only diff --git a/src/adapt/modules/tracking/decision_log.py b/src/adapt/modules/tracking/decision_log.py index 462fcde..b983321 100644 --- a/src/adapt/modules/tracking/decision_log.py +++ b/src/adapt/modules/tracking/decision_log.py @@ -1,215 +1,104 @@ # Copyright © 2026, UChicago Argonne, LLC # See LICENSE for terms and disclaimer. -"""Per-frame log of the tracker's matching decisions. +"""Per-scan log of the tracker's decisions. -``FrameLog`` accumulates, stage by stage, what happened to every candidate -pair of one scan pair and freezes it into ``TrackingDecisions`` — the -analysis-only record persisted to ``decisions.db``. It is a builder: mutable -for the duration of one ``CellTracker.track`` call, immutable afterwards. +``ScanLog`` collects, step by step, the records of one ``CellTracker.track`` +call and freezes them into ``TrackingDecisions`` — the analysis-only record +persisted to ``decisions.db``. Mutable while the scan is tracked, immutable +afterwards. """ +from collections import Counter from dataclasses import dataclass, field from adapt.contracts import ( - TrackingCandidate, + TrackingCell, TrackingDecisions, - TrackingFrame, - TrackingSplitMergeTest, - TrackingUnmatched, + TrackingIdentity, + TrackingLatent, + TrackingLineage, + TrackingPair, + TrackingScan, ) -__all__ = ["FrameLog", "LogCell", "STAGE_ASSIGNMENT", "STAGE_KINEMATIC", "STAGE_OVERLAP"] - -STAGE_OVERLAP = "OVERLAP" -STAGE_KINEMATIC = "KINEMATIC" -STAGE_ASSIGNMENT = "ASSIGNMENT" -_STAGE_ORDER = {STAGE_OVERLAP: 0, STAGE_KINEMATIC: 1, STAGE_ASSIGNMENT: 2} - - -@dataclass(frozen=True) -class LogCell: - """Identity of one previous cell as the log sees it.""" - - uid: str - label: int - steps_before: int = 0 +__all__ = ["ScanLog"] @dataclass -class FrameLog: - """Builder for one frame pair's ``TrackingDecisions``.""" - - dt_s: float | None - prev: list[LogCell] - curr_labels: list[int] - reset_code: str | None = None - _pairs: dict[tuple[int, int], dict] = field(default_factory=dict) - _tests: list[TrackingSplitMergeTest] = field(default_factory=list) - _matched_prev: dict[int, str] = field(default_factory=dict) # prev idx -> method - _split_children: set[int] = field(default_factory=set) - _merge_sources: set[str] = field(default_factory=set) # prev cell uids - _born_uids: dict[int, str] = field(default_factory=dict) +class ScanLog: + """Builder for one scan's ``TrackingDecisions``. - # ── stage recorders ──────────────────────────────────────────────────── + ``counts`` holds the scan totals the records alone do not give + (previous and latent tracks, components, crossing exclusions, merges, + identity transfers, latent creations, terminations). + """ - def candidate(self, i: int, j: int, **geometry) -> None: - """Stage 2/3: a generated pair with its overlap-gate result.""" - self._pairs[(i, j)] = { - **geometry, - "speed_ms": None, - "previous_speed_ms": None, - "accel_cap_ms": None, - "kinematic_code": None, - "displacement_m": None, - "length_scale_m": None, - "heading_change_deg": None, - "cost": None, - "component_n_prev": None, - "component_n_curr": None, + dt_s: float | None + reset_code: str | None + counts: Counter = field(default_factory=Counter) + pairs: dict[tuple[str, int], dict] = field(default_factory=dict) + cells: dict[int, TrackingCell] = field(default_factory=dict) + lineage: list[TrackingLineage] = field(default_factory=list) + identity: list[TrackingIdentity] = field(default_factory=list) + latent: list[TrackingLatent] = field(default_factory=list) + + def pair(self, uid: str, label: int, **values) -> None: + self.pairs[(uid, label)] = { + "component_id": None, "cost_rank": None, - "match_method": None, - "last_stage": STAGE_OVERLAP, - "outcome": "CONTINUE" if geometry["overlap_passed"] else "REJECTED_OVERLAP", + "margin": None, + "crossing_with_uid": None, + **values, } - def kinematic(self, i: int, j: int, **motion) -> None: - """Stage 4: the kinematic gate's numbers; a code means rejection.""" - row = self._pairs[(i, j)] - row.update(motion, last_stage=STAGE_KINEMATIC) - if motion["kinematic_code"] is not None: - row["outcome"] = "REJECTED_KINEMATIC" - - def costed(self, i: int, j: int, **cost) -> None: - """Stage 5: the pair survived every gate and carries a cost.""" - self._pairs[(i, j)].update(cost, last_stage=STAGE_ASSIGNMENT, outcome="LOST_ASSIGNMENT") - - def matched(self, i: int, j: int, method: str) -> None: - """Stage 6: the pair won its component.""" - self._pairs[(i, j)].update(match_method=method, outcome="CONTINUE") - self._matched_prev[i] = method - - def component(self, prevs: list[int], currs: list[int]) -> None: - """Stage 6: the component (prev x curr) every costed pair was resolved in.""" - for i, j in self._pairs: - if i in prevs and j in currs and self._pairs[(i, j)]["cost"] is not None: - self._pairs[(i, j)].update(component_n_prev=len(prevs), component_n_curr=len(currs)) - - def split_merge_test(self, test: TrackingSplitMergeTest) -> None: - self._tests.append(test) - - def split_child(self, j: int) -> None: - self._split_children.add(j) - - def merge_source(self, prev_uid: str) -> None: - self._merge_sources.add(prev_uid) - - def born(self, j: int, uid: str) -> None: - self._born_uids[j] = uid - - # ── freeze ───────────────────────────────────────────────────────────── + def update_pair(self, uid: str, label: int, **values) -> None: + self.pairs[(uid, label)].update(values) def freeze(self) -> TrackingDecisions: self._rank_costs() - candidates = tuple( - TrackingCandidate( - prev_cell_uid=self.prev[i].uid, - prev_cell_label=self.prev[i].label, - curr_cell_label=self.curr_labels[j], - track_steps_before=self.prev[i].steps_before, - **row, - ) - for (i, j), row in sorted(self._pairs.items()) + pairs = tuple( + TrackingPair(prev_cell_uid=uid, curr_cell_label=label, **row) + for (uid, label), row in sorted(self.pairs.items()) ) - unmatched = tuple(self._unmatched_prev()) + tuple(self._unmatched_curr()) - rows = list(self._pairs.values()) - methods = list(self._matched_prev.values()) - frame = TrackingFrame( + gates = Counter(p.gate for p in pairs) + fates = Counter(c.fate for c in self.cells.values()) + scan = TrackingScan( dt_s=self.dt_s, reset_code=self.reset_code, - n_prev=len(self.prev), - n_curr=len(self.curr_labels), - n_pairs=len(rows), - n_pass_overlap=sum(r["overlap_passed"] for r in rows), - n_pass_kinematic=sum(r["cost"] is not None for r in rows), - n_propagated=methods.count("PROPAGATED"), - n_hungarian=methods.count("HUNGARIAN"), - n_split=len(self._split_children), - n_merge=len(self._merge_sources), - n_initiation=len(self._born_uids) - len(self._split_children), - n_termination=len(self.prev) - len(self._matched_prev), + n_prev=self.counts["prev"], + n_latent=self.counts["latent"], + n_curr=len(self.cells), + n_pairs=len(pairs), + n_pass_overlap=len(pairs) - gates["OVERLAP"], + n_pass_speed=gates["PASS"] + gates["COST"], + n_pass_cost=gates["PASS"], + n_components=self.counts["components"], + n_continue=fates["CONTINUE"], + n_crossing_excluded=self.counts["crossing_excluded"], + n_split=fates["SPLIT_CHILD"], + n_merge=self.counts["merge"], + n_resumed=fates["RESUMED"], + n_identity_transfer=self.counts["identity_transfer"], + n_initiation=fates["INITIATION"], + n_latent_created=self.counts["latent_created"], + n_termination=self.counts["termination"], + ) + return TrackingDecisions( + scan, + tuple(self.cells[k] for k in sorted(self.cells)), + pairs, + tuple(self.lineage), + tuple(self.identity), + tuple(self.latent), ) - return TrackingDecisions(frame, candidates, unmatched, tuple(self._tests)) def _rank_costs(self) -> None: - """Rank each previous cell's costed pairs, cheapest first (1-based).""" - by_prev: dict[int, list[tuple[float, int]]] = {} - for (i, j), row in self._pairs.items(): - if row["cost"] is not None: - by_prev.setdefault(i, []).append((row["cost"], j)) - for i, costs in by_prev.items(): - for rank, (_, j) in enumerate(sorted(costs), start=1): - self._pairs[(i, j)]["cost_rank"] = rank - - def _best(self, rows: list[tuple[int, dict]]): - """The candidate that got furthest, with how close it came. - - Returns ``(label, stage, opc, ocp, cost)``. The overlaps and cost make a - near-miss measurable directly: a cell whose best candidate reached - Opc 0.19 against a 0.20 gate is a different failure from one that had - no candidate at all. - """ - if not rows: - return None, None, None, None, None - label, row = max(rows, key=lambda item: _STAGE_ORDER[item[1]["last_stage"]]) - return label, row["last_stage"], row.get("opc"), row.get("ocp"), row.get("cost") - - def _reason(self, rows: list[tuple[int, dict]]) -> str: - if self.reset_code is not None: - return self.reset_code if self.reset_code == "FIRST_SCAN" else "RESET" - if not rows: - return "NO_CANDIDATE" - stages = {row["last_stage"] for _, row in rows} - if STAGE_ASSIGNMENT in stages: - return "LOST_ASSIGNMENT" - if STAGE_KINEMATIC in stages: - return "ALL_REJECTED_KINEMATIC" - return "ALL_REJECTED_OVERLAP" - - def _unmatched_prev(self): - for i, cell in enumerate(self.prev): - if i in self._matched_prev: - continue - rows = [(self.curr_labels[j], r) for (p, j), r in self._pairs.items() if p == i] - best_label, best_stage, b_opc, b_ocp, b_cost = self._best(rows) - reason = "MERGE_SOURCE" if cell.uid in self._merge_sources else self._reason(rows) - yield TrackingUnmatched( - "prev", - cell.uid, - cell.label, - len(rows), - best_label, - best_stage, - reason, - best_opc=b_opc, - best_ocp=b_ocp, - best_cost=b_cost, - ) - - def _unmatched_curr(self): - for j, uid in sorted(self._born_uids.items()): - rows = [(self.prev[i].label, r) for (i, c), r in self._pairs.items() if c == j] - best_label, best_stage, b_opc, b_ocp, b_cost = self._best(rows) - reason = "SPLIT_CHILD" if j in self._split_children else self._reason(rows) - yield TrackingUnmatched( - "curr", - uid, - self.curr_labels[j], - len(rows), - best_label, - best_stage, - reason, - best_opc=b_opc, - best_ocp=b_ocp, - best_cost=b_cost, - ) + """Rank each footprint's admissible pairs, cheapest first (1-based).""" + by_prev: dict[str, list[tuple[float, int]]] = {} + for (uid, label), row in self.pairs.items(): + if row["gate"] == "PASS": + by_prev.setdefault(uid, []).append((row["cost"], label)) + for uid, costs in by_prev.items(): + for rank, (_, label) in enumerate(sorted(costs), start=1): + self.pairs[(uid, label)]["cost_rank"] = rank diff --git a/src/adapt/modules/tracking/events.py b/src/adapt/modules/tracking/events.py index 3da5248..da3f0e0 100644 --- a/src/adapt/modules/tracking/events.py +++ b/src/adapt/modules/tracking/events.py @@ -1,31 +1,29 @@ # Copyright © 2026, UChicago Argonne, LLC # See LICENSE for terms and disclaimer. -"""Lineage event-row builders. +"""Lineage event rows persisted as ``cell_events``. -Pure functions that turn graph nodes into the explicit event dicts persisted as -``cell_events`` (CONTINUE / SPLIT / MERGE / INITIATION / TERMINATION). Each takes -the graph and the track_index→(cell_uid, signature) identity map; no tracker state. +One row per event: CONTINUE, SPLIT, MERGE, INITIATION, TERMINATION, LATENT (a +track kept for possible resumption) and RESUMED. ``source_scan_id`` / +``source_scan_time`` name the scan the source cell was last observed in — the +previous scan, except for a resumed or an expiring latent track. """ +from dataclasses import dataclass + import pandas as pd -from adapt.modules.tracking.graph import TrackingGraph -from adapt.modules.tracking.models import MatchDiagnostics, MatchMethod from adapt.utils.time import normalize_time_scalar __all__ = [ "DIAGNOSTIC_COLUMNS", "EVENT_COLUMNS", + "EndPoint", + "LinkDiagnostics", "build_cell_events_dataframe", - "event_continue", - "event_initiation", - "event_merge", - "event_split", - "event_termination", + "event_row", ] -# Per-accepted-match explainability columns (null for INITIATION / TERMINATION). DIAGNOSTIC_COLUMNS = [ "candidate_opc", "candidate_ocp", @@ -44,160 +42,88 @@ "target_cell_uid", "source_cell_label", "target_cell_label", + "source_scan_id", + "source_scan_time", "cost", "is_dominant", "event_group_id", *DIAGNOSTIC_COLUMNS, ] -Identity = dict[int, tuple[str, str]] - - -def _diagnostic_fields(diag: MatchDiagnostics | None) -> dict: - """Map a MatchDiagnostics into the ``candidate_*`` / ``match_method`` columns.""" - if diag is None: - return dict.fromkeys(DIAGNOSTIC_COLUMNS) - return { - "candidate_opc": diag.opc, - "candidate_ocp": diag.ocp, - "candidate_centroid_distance_m": diag.centroid_distance_m, - "candidate_speed_ms": diag.speed_ms, - "candidate_heading_change_deg": diag.heading_change_deg, - "candidate_area_ratio": diag.area_ratio, - "candidate_final_cost": diag.final_cost, - "match_method": diag.match_method, - } - - -def _uid(graph: TrackingGraph, identity: Identity, node_id: int) -> str: - track_index = int(graph.get_node_attr(node_id, "track_index")) - if track_index not in identity: - raise ValueError(f"Missing cell identity for track_index={track_index}") - return identity[track_index][0] - -def _time_key(time_val) -> str: - """Stable ISO8601 time key for event grouping.""" - tv = normalize_time_scalar(time_val) - return pd.Timestamp(tv).isoformat() - - -def event_continue( - graph: TrackingGraph, - identity: Identity, - time, - prev_node_id: int, - curr_node_id: int, - cost: float, - diagnostics: MatchDiagnostics | None = None, -) -> dict: - target_cell_uid = _uid(graph, identity, curr_node_id) - return { - "time": time, - "event_type": "CONTINUE", - "source_cell_uid": _uid(graph, identity, prev_node_id), - "target_cell_uid": target_cell_uid, - "source_cell_label": int(graph.get_node_attr(prev_node_id, "cell_id")), - "target_cell_label": int(graph.get_node_attr(curr_node_id, "cell_id")), - "cost": float(cost), - "is_dominant": True, - "event_group_id": f"{_time_key(time)}:CONTINUE:{target_cell_uid}", - **_diagnostic_fields(diagnostics), - } +@dataclass(frozen=True) +class EndPoint: + """A cell at one end of an event.""" + uid: str + label: int | None # None: an expiring merge's survivor, observed scans ago -def event_split( - graph: TrackingGraph, - identity: Identity, - time, - parent_node_id: int, - child_node_id: int, -) -> dict: - parent_uid = _uid(graph, identity, parent_node_id) - return { - "time": time, - "event_type": "SPLIT", - "source_cell_uid": parent_uid, - "target_cell_uid": _uid(graph, identity, child_node_id), - "source_cell_label": int(graph.get_node_attr(parent_node_id, "cell_id")), - "target_cell_label": int(graph.get_node_attr(child_node_id, "cell_id")), - "cost": None, - "is_dominant": False, - "event_group_id": f"{_time_key(time)}:SPLIT:{parent_uid}", - **_diagnostic_fields(MatchDiagnostics(match_method=MatchMethod.SPLIT)), - } +@dataclass(frozen=True) +class LinkDiagnostics: + """Evidence of an accepted link: grown overlaps (O^c, O^h), centre residual + to the prediction (m), speed, heading change, area ratio, cost, and how the + link was decided.""" -def event_merge( - graph: TrackingGraph, - identity: Identity, - time, - source_node_id: int, - target_node_id: int, -) -> dict: - target_uid = _uid(graph, identity, target_node_id) - return { - "time": time, - "event_type": "MERGE", - "source_cell_uid": _uid(graph, identity, source_node_id), - "target_cell_uid": target_uid, - "source_cell_label": int(graph.get_node_attr(source_node_id, "cell_id")), - "target_cell_label": int(graph.get_node_attr(target_node_id, "cell_id")), - "cost": None, - "is_dominant": False, - "event_group_id": f"{_time_key(time)}:MERGE:{target_uid}", - **_diagnostic_fields(MatchDiagnostics(match_method=MatchMethod.MERGE)), - } + o_c: float + o_h: float + residual_m: float + speed_ms: float + heading_change_deg: float | None + area_ratio: float + cost: float + method: str -def event_initiation(graph: TrackingGraph, identity: Identity, time, node_id: int) -> dict: - target_uid = _uid(graph, identity, node_id) - return { - "time": time, - "event_type": "INITIATION", - "source_cell_uid": None, - "target_cell_uid": target_uid, - "source_cell_label": None, - "target_cell_label": int(graph.get_node_attr(node_id, "cell_id")), - "cost": None, - "is_dominant": False, - "event_group_id": f"{_time_key(time)}:INITIATION:{target_uid}", - } +def _time_key(time_val) -> str: + return pd.Timestamp(normalize_time_scalar(time_val)).isoformat() -def event_termination( - graph: TrackingGraph, - identity: Identity, +def event_row( time, - source_node_id: int, - target_node_id: int | None, + event_type: str, + source: EndPoint | None, + target: EndPoint | None, + *, + source_scan: tuple[str, object] | None = None, + diagnostics: LinkDiagnostics | None = None, + is_dominant: bool = False, ) -> dict: - source_uid = _uid(graph, identity, source_node_id) - target_uid = _uid(graph, identity, target_node_id) if target_node_id is not None else None + """One ``cell_events`` row; the group id is keyed by the cell the event is about.""" + about = target if event_type in {"CONTINUE", "MERGE", "INITIATION", "RESUMED"} else source + if about is None: + raise ValueError(f"{event_type} event without the cell it is about") + diag = dict.fromkeys(DIAGNOSTIC_COLUMNS) + if diagnostics is not None: + diag = { + "candidate_opc": diagnostics.o_c, + "candidate_ocp": diagnostics.o_h, + "candidate_centroid_distance_m": diagnostics.residual_m, + "candidate_speed_ms": diagnostics.speed_ms, + "candidate_heading_change_deg": diagnostics.heading_change_deg, + "candidate_area_ratio": diagnostics.area_ratio, + "candidate_final_cost": diagnostics.cost, + "match_method": diagnostics.method, + } return { "time": time, - "event_type": "TERMINATION", - "source_cell_uid": source_uid, - "target_cell_uid": target_uid, - "source_cell_label": int(graph.get_node_attr(source_node_id, "cell_id")), - "target_cell_label": ( - int(graph.get_node_attr(target_node_id, "cell_id")) - if target_node_id is not None - else None - ), - "cost": None, - "is_dominant": False, - "event_group_id": f"{_time_key(time)}:TERMINATION:{source_uid}", + "event_type": event_type, + "source_cell_uid": source.uid if source else None, + "target_cell_uid": target.uid if target else None, + "source_cell_label": source.label if source else None, + "target_cell_label": target.label if target else None, + "source_scan_id": source_scan[0] if source_scan else None, + "source_scan_time": source_scan[1] if source_scan else None, + "cost": diagnostics.cost if diagnostics else None, + "is_dominant": is_dominant, + "event_group_id": f"{_time_key(time)}:{event_type}:{about.uid}", + **diag, } def build_cell_events_dataframe(events: list[dict]) -> pd.DataFrame: if not events: return pd.DataFrame(columns=EVENT_COLUMNS) - df = pd.DataFrame(events) - for col in EVENT_COLUMNS: - if col not in df.columns: - df[col] = None - df = df[EVENT_COLUMNS] + df = pd.DataFrame(events)[EVENT_COLUMNS] df["time"] = df["time"].apply(lambda t: pd.Timestamp(normalize_time_scalar(t))) return df diff --git a/src/adapt/modules/tracking/graph.py b/src/adapt/modules/tracking/graph.py deleted file mode 100644 index 09cc51b..0000000 --- a/src/adapt/modules/tracking/graph.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright © 2026, UChicago Argonne, LLC -# See LICENSE for terms and disclaimer. - -"""Directed graph storing cell tracking history and lineage. - -Nodes are cell observations at a time; edges are temporal relationships -(CONTINUE, SPLIT, MERGE). This is the only home for ``networkx`` in the package. -""" - -import networkx as nx - -__all__ = ["TrackingGraph"] - - -class TrackingGraph: - """Directed graph storing cell tracking history and lineage. - - Nodes represent cell observations at specific times. - Edges represent temporal relationships (CONTINUE, SPLIT, MERGE). - - Node attributes: - - node_id: unique identifier (int) - - time: observation timestamp - - cell_id: cell label from segmentation - - track_index: tracking index this cell belongs to (starts at 1; 0 = background sentinel) - - area: cell area in km² - - centroid_x, centroid_y: cell center coordinates - - mean_reflectivity: average dBZ - - max_reflectivity: peak dBZ - - core_area: area with Z > threshold dBZ - - Edge attributes: - - edge_type: "CONTINUE", "SPLIT", "MERGE" - - cost: assignment cost (for diagnostics) - """ - - def __init__(self): - """Initialize empty tracking graph.""" - self.graph = nx.DiGraph() - self._node_counter = 0 - self._track_counter = 0 # Will yield 1, 2, 3, ... (0 is background sentinel) - - def add_observation( - self, - time, - cell_id: int, - track_index: int, - area: float, - centroid_x: float, - centroid_y: float, - mean_reflectivity: float, - max_reflectivity: float, - core_area: float, - cell_uid: str, - track_signature: str, - ) -> int: - node_id = self._node_counter - self._node_counter += 1 - - self.graph.add_node( - node_id, - time=time, - cell_id=cell_id, - track_index=track_index, - area=area, - centroid_x=centroid_x, - centroid_y=centroid_y, - mean_reflectivity=mean_reflectivity, - max_reflectivity=max_reflectivity, - core_area=core_area, - cell_uid=cell_uid, - track_signature=track_signature, - ) - return node_id - - def add_edge(self, from_node: int, to_node: int, edge_type: str, cost: float = 0.0): - """Add a temporal relationship edge. - - Parameters - ---------- - from_node : int - Source node ID (earlier time) - to_node : int - Target node ID (later time) - edge_type : str - Edge type: "CONTINUE", "SPLIT", or "MERGE" - cost : float, optional - Assignment cost for diagnostics (default: 0.0) - """ - self.graph.add_edge(from_node, to_node, edge_type=edge_type, cost=cost) - - def get_new_track_index(self) -> int: - """Allocate a new unique track index (starts at 1; 0 is background sentinel).""" - self._track_counter += 1 - return self._track_counter - - def get_node_attr(self, node_id: int, attr: str): - """Get a node attribute value. - - Parameters - ---------- - node_id : int - Node identifier - attr : str - Attribute name - - Returns - ------- - Any - Attribute value, or None if not present - """ - return self.graph.nodes[node_id].get(attr) - - def get_nodes_at_time(self, time) -> list[int]: - """Get all node IDs for a given timestamp. - - Parameters - ---------- - time : datetime-like - Timestamp to query - - Returns - ------- - List[int] - List of node IDs at this time - """ - return [n for n, d in self.graph.nodes(data=True) if d.get("time") == time] diff --git a/src/adapt/modules/tracking/latent.py b/src/adapt/modules/tracking/latent.py new file mode 100644 index 0000000..d6c7110 --- /dev/null +++ b/src/adapt/modules/tracking/latent.py @@ -0,0 +1,63 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Latent tracks: cells that ended or merged away, carried by the flow. + +A latent track keeps the last observed state of its cell and a footprint that +the flow moves on each scan, so a cell that vanishes for a scan — absorbed by +a neighbour, or briefly below the threshold — can resume its identity. +""" + +from dataclasses import dataclass + +import numpy as np + +__all__ = ["ORIGIN_MERGED", "ORIGIN_TERMINATED", "LatentTrack", "carry_footprint"] + +ORIGIN_MERGED = "MERGED" +ORIGIN_TERMINATED = "TERMINATED" + + +@dataclass(frozen=True, eq=False) +class LatentTrack: + """A track without an observation since ``scan_id``, ``age`` scans ago. + + Position fields are metres; ``footprint`` and ``predicted`` are at the scan + being tracked. ``merged_into_uid`` names the survivor for a merged-away track. + """ + + uid: str + label: int + scan_id: str + scan_time: object + time_s: float + centre: tuple[float, float] + step: tuple[float, float] | None + speed: float | None + score: float + footprint: np.ndarray + predicted: tuple[float, float] + age: int + origin: str + merged_into_uid: str | None + + +def carry_footprint( + footprint: np.ndarray, flow_x: np.ndarray, flow_y: np.ndarray +) -> tuple[np.ndarray, tuple[int, int]]: + """Shift a footprint by the mean flow inside it, rounded to whole pixels. + + ``flow_x``/``flow_y`` are the displacement (pixels per scan) from the + footprint's scan to the next. Returns the carried mask (clipped at the + domain edge) and the ``(row, col)`` shift applied. + """ + if not footprint.any(): + return footprint, (0, 0) + d_row = int(round(float(flow_y[footprint].mean()))) + d_col = int(round(float(flow_x[footprint].mean()))) + rows, cols = np.nonzero(footprint) + rows, cols = rows + d_row, cols + d_col + inside = (rows >= 0) & (rows < footprint.shape[0]) & (cols >= 0) & (cols < footprint.shape[1]) + carried = np.zeros_like(footprint) + carried[rows[inside], cols[inside]] = True + return carried, (d_row, d_col) diff --git a/src/adapt/modules/tracking/lineage.py b/src/adapt/modules/tracking/lineage.py new file mode 100644 index 0000000..25c2a12 --- /dev/null +++ b/src/adapt/modules/tracking/lineage.py @@ -0,0 +1,71 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Lineage partner choice and the identity rule at splits and merges. + +A fragment is a split child of the continuing parent whose grown footprint +covers the largest share of it; a vanished cell is a merge source of the +continuing cell covering the largest share of its footprint. When a split or +merge leaves two candidates for one identity, the identity follows the larger +score ``S = log A + γ log ē``; within the noise margin it follows the candidate +nearer the predicted position. Pure functions. +""" + +import math +from collections.abc import Hashable +from dataclasses import dataclass + +__all__ = [ + "RULE_NEAREST", + "RULE_SCORE", + "IdentityCandidate", + "best_partner", + "choose_identity", + "identity_score", +] + +RULE_SCORE = "SCORE" +RULE_NEAREST = "NEAREST" + + +def best_partner(fractions: dict[int, float], threshold: float) -> int | None: + """The key with the largest fraction ≥ ``threshold`` (smaller key on a tie), else None.""" + qualified = [(-f, k) for k, f in fractions.items() if f >= threshold] + return min(qualified)[1] if qualified else None + + +def identity_score(area_km2: float, mean_excess: float, gamma: float) -> float: + """``S = log A + γ log ē``; γ = 1 ranks by intensity mass, γ = 0 by area. + + A cell with no excess (ē = 0) has no core and scores −∞ for γ > 0. + """ + if gamma == 0.0: + return math.log(area_km2) + if mean_excess == 0.0: + return -math.inf + return math.log(area_km2) + gamma * math.log(mean_excess) + + +@dataclass(frozen=True) +class IdentityCandidate: + """One contender for an identity: its score and its shape-aware distance + to the predicted position.""" + + key: Hashable + score: float + distance: float + + +def choose_identity(candidates: list[IdentityCandidate], margin: float) -> tuple[Hashable, str]: + """``(winner key, rule)``. + + Candidates scoring within ``margin`` of the best are indistinguishable by + intensity; among them the nearest wins (RULE_NEAREST). A single clear + leader wins on score (RULE_SCORE). + """ + top = max(c.score for c in candidates) + close = [c for c in candidates if c.score >= top - margin] + if len(close) == 1: + return close[0].key, RULE_SCORE + nearest = min(close, key=lambda c: (c.distance, -c.score)) + return nearest.key, RULE_NEAREST diff --git a/src/adapt/modules/tracking/matching/assignment.py b/src/adapt/modules/tracking/matching/assignment.py index 74b54ab..4d29b60 100644 --- a/src/adapt/modules/tracking/matching/assignment.py +++ b/src/adapt/modules/tracking/matching/assignment.py @@ -1,72 +1,95 @@ # Copyright © 2026, UChicago Argonne, LLC # See LICENSE for terms and disclaimer. -"""Constraint propagation and ambiguity isolation. - -After validation the surviving pairs form a bipartite graph (previous objects ↔ -current cells). ``ConstraintPropagator`` deterministically resolves every pair that -is unique on *both* sides (degree-1 ↔ degree-1), iterating until convergence — this -handles the great majority of ordinary tracks with no optimisation. ``AssignmentGraph`` -then splits whatever remains into independent connected components, so Hungarian -assignment runs on small, locally-ambiguous groups instead of one global matrix. +"""Joint assignment inside each group of competing cells. + +Admissible links form a bipartite graph (previous cells ↔ current cells); +each connected component is decided on its own, so a clear association in one +part of the domain is never altered by an ambiguous one elsewhere. Inside a +component the minimum-cost set of links is found with a no-link option at the +cost ceiling, subject to the crossing test: while the chosen links contain a +crossing pair, the one of the two whose exclusion leaves the cheaper +assignment is excluded (the costlier on a tie) and the component is solved +again. """ +from dataclasses import dataclass +from itertools import combinations + import networkx as nx -__all__ = ["AssignmentGraph", "ConstraintPropagator"] +from adapt.modules.tracking.matching.gates import Point, segments_cross +from adapt.modules.tracking.matching.hungarian import HungarianMatcher + +__all__ = ["ComponentSolution", "Link", "components", "solve_component"] Edge = tuple[int, int] # (prev_idx, curr_idx) -class ConstraintPropagator: - """Iteratively peel deterministic (mutually-unique) matches to convergence.""" - - @staticmethod - def resolve(edges: list[Edge]) -> tuple[list[Edge], list[Edge]]: - """Return (forced_matches, remaining_edges). - - A pair (i, j) is forced when previous i has exactly one surviving candidate - and current j has exactly one surviving predecessor. Removing forced nodes - can make neighbours unique, so the peel repeats until nothing new resolves. - """ - remaining: set[Edge] = set(edges) - forced: list[Edge] = [] - while True: - prev_deg: dict[int, int] = {} - curr_deg: dict[int, int] = {} - for i, j in remaining: - prev_deg[i] = prev_deg.get(i, 0) + 1 - curr_deg[j] = curr_deg.get(j, 0) + 1 - new = [(i, j) for (i, j) in remaining if prev_deg[i] == 1 and curr_deg[j] == 1] - if not new: - break - forced.extend(new) - matched_prev = {i for i, _ in new} - matched_curr = {j for _, j in new} - remaining = { - (i, j) for (i, j) in remaining if i not in matched_prev and j not in matched_curr - } - return sorted(forced), sorted(remaining) - - -class AssignmentGraph: - """Bipartite graph over candidate edges; yields connected components.""" - - def __init__(self, edges: list[Edge]): - self.edges = list(edges) - - def components(self) -> list[tuple[list[int], list[int]]]: - """Connected components as ``(prev_indices, curr_indices)`` sorted lists. - - Only nodes that appear in an edge are included; isolated previous/current - objects (no surviving candidate) are handled by the caller as dissipated/born. - """ - graph = nx.Graph() - for i, j in self.edges: - graph.add_edge(("p", i), ("c", j)) - components: list[tuple[list[int], list[int]]] = [] - for nodes in nx.connected_components(graph): - prevs = sorted(n[1] for n in nodes if n[0] == "p") - currs = sorted(n[1] for n in nodes if n[0] == "c") - components.append((prevs, currs)) - return components +@dataclass(frozen=True) +class Link: + """An admissible link with its cost and its segment (metres).""" + + prev: int + curr: int + cost: float + start: Point + end: Point + + +@dataclass(frozen=True) +class ComponentSolution: + """Chosen links, links excluded for crossing (→ the chosen-set link they crossed), + and each chosen link's margin: the extra total cost of the best assignment without it. + """ + + chosen: list[Edge] + crossing_excluded: dict[Edge, Edge] + margins: dict[Edge, float] + + +def components(edges: list[Edge]) -> list[tuple[list[int], list[int]]]: + """Connected components as sorted ``(prev_indices, curr_indices)``.""" + graph = nx.Graph() + graph.add_edges_from((("p", i), ("c", j)) for i, j in edges) + groups = [] + for nodes in nx.connected_components(graph): + groups.append( + (sorted(n[1] for n in nodes if n[0] == "p"), sorted(n[1] for n in nodes if n[0] == "c")) + ) + return sorted(groups) + + +def solve_component(links: list[Link], no_link_cost: float) -> ComponentSolution: + """Minimum-cost non-crossing links of one component, with no-link at ``no_link_cost``.""" + by_edge = {(lk.prev, lk.curr): lk for lk in links} + prevs = sorted({lk.prev for lk in links}) + currs = sorted({lk.curr for lk in links}) + + def solve(excluded) -> tuple[list[Edge], float]: + costs = {e: lk.cost for e, lk in by_edge.items() if e not in excluded} + chosen = HungarianMatcher.match(prevs, currs, costs, no_link_cost) + total = sum(costs[e] for e in chosen) + no_link_cost * (len(prevs) - len(chosen)) + return chosen, total + + def first_crossing(chosen: list[Edge]) -> tuple[Edge, Edge] | None: + for a, b in combinations(chosen, 2): + la, lb = by_edge[a], by_edge[b] + if segments_cross(la.start, la.end, lb.start, lb.end): + return a, b + return None + + excluded: dict[Edge, Edge] = {} + chosen, total = solve(excluded) + while (pair := first_crossing(chosen)) is not None: + a, b = sorted(pair, key=lambda e: (by_edge[e].cost, e)) # b is the costlier + total_without_a = solve({**excluded, a: b})[1] + total_without_b = solve({**excluded, b: a})[1] + if total_without_a < total_without_b: + excluded[a] = b + else: + excluded[b] = a + chosen, total = solve(excluded) + + margins = {e: solve({**excluded, e: e})[1] - total for e in chosen} + return ComponentSolution(chosen, excluded, margins) diff --git a/src/adapt/modules/tracking/matching/candidate.py b/src/adapt/modules/tracking/matching/candidate.py deleted file mode 100644 index 07c9605..0000000 --- a/src/adapt/modules/tracking/matching/candidate.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright © 2026, UChicago Argonne, LLC -# See LICENSE for terms and disclaimer. - -"""Liberal candidate generation. - -Each projected hull is dilated by the configured buffer, then every current cell -that touches the buffered hull becomes a candidate pair. High recall by design — -ranking and rejection happen in later stages. Pure geometry over boolean masks. -""" - -import numpy as np - -from adapt.modules.tracking.matching.geometry import dilate_hull - -__all__ = ["CandidateGenerator"] - - -class CandidateGenerator: - """Dilate projected hulls and emit every intersecting cell as a candidate.""" - - def __init__(self, buffer_pixels: int): - self.buffer_pixels = buffer_pixels - - def generate( - self, - prev_hulls: list[np.ndarray], - curr_masks: list[np.ndarray], - ) -> tuple[list[np.ndarray], list[tuple[int, int]]]: - """Return (buffered_hulls, candidate_pairs). - - ``buffered_hulls[i]`` is the dilated hull for previous object ``i`` (reused - by downstream overlap/cost). ``candidate_pairs`` are ``(prev_idx, curr_idx)`` - for every current cell intersecting a buffered hull. - """ - buffered = [dilate_hull(h, self.buffer_pixels) for h in prev_hulls] - pairs: list[tuple[int, int]] = [] - for i, hull in enumerate(buffered): - if not hull.any(): - continue - for j, cell in enumerate(curr_masks): - if np.any(hull & cell): - pairs.append((i, j)) - return buffered, pairs diff --git a/src/adapt/modules/tracking/matching/gates.py b/src/adapt/modules/tracking/matching/gates.py new file mode 100644 index 0000000..0afa254 --- /dev/null +++ b/src/adapt/modules/tracking/matching/gates.py @@ -0,0 +1,119 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Physical plausibility of a link: hard speed gates, heading term, crossing. + +A pair failing a speed gate is removed whatever its overlap. The heading term +is soft (part of the cost); the crossing test is hard and parameter-free. +Positions are metres, speeds m/s. Pure functions — no tracker state. +""" + +import math +from collections.abc import Hashable +from dataclasses import dataclass + +__all__ = [ + "GATE_SPEED", + "GATE_SPEED_CHANGE", + "SpeedGate", + "drop_crossing_links", + "heading_term", + "segments_cross", + "speed_change_limit", + "speed_gate", +] + +GATE_SPEED = "SPEED" +GATE_SPEED_CHANGE = "SPEED_CHANGE" + +Point = tuple[float, float] + + +def speed_change_limit(v_prev: float, dt_s: float, a_max: float, sigma_x_m: float) -> float: + """``v_prev + a_max Δt + 2√2 σ_x / Δt``. + + Acceleration over the interval plus the apparent speed change a position + error σ_x at both ends of two consecutive steps produces; the second term + dominates short intervals, so the limit adapts to scan cadence. + """ + return v_prev + a_max * dt_s + 2.0 * math.sqrt(2.0) * sigma_x_m / dt_s + + +@dataclass(frozen=True) +class SpeedGate: + """``code`` is None when the pair passes; ``limit_ms`` is the speed-change limit applied.""" + + code: str | None + limit_ms: float | None + + +def speed_gate( + speed_ms: float, + v_prev: float | None, + dt_s: float, + v_max: float, + a_max: float, + sigma_x_m: float, +) -> SpeedGate: + """Absolute cap first; the additive speed-change limit once the track has a speed.""" + limit = None if v_prev is None else speed_change_limit(v_prev, dt_s, a_max, sigma_x_m) + if speed_ms > v_max: + return SpeedGate(GATE_SPEED, limit) + if limit is not None and speed_ms > limit: + return SpeedGate(GATE_SPEED_CHANGE, limit) + return SpeedGate(None, limit) + + +def heading_term( + prev_step: Point | None, step: Point, weight: float, min_step_m: float +) -> tuple[float, float | None]: + """``(h, Δθ in degrees)`` with ``h = w_h (1 − cos Δθ) / 2``. + + Applied only when both displacements exceed ``min_step_m``; below it the + direction is position noise and the term is (0, None). + """ + if prev_step is None: + return 0.0, None + prev_len, step_len = math.hypot(*prev_step), math.hypot(*step) + if prev_len <= min_step_m or step_len <= min_step_m: + return 0.0, None + cos_turn = (prev_step[0] * step[0] + prev_step[1] * step[1]) / (prev_len * step_len) + cos_turn = min(1.0, max(-1.0, cos_turn)) + return weight * (1.0 - cos_turn) / 2.0, math.degrees(math.acos(cos_turn)) + + +def _orientation(a: Point, b: Point, c: Point) -> int: + cross = (b[0] - a[0]) * (c[1] - a[1]) - (b[1] - a[1]) * (c[0] - a[0]) + return int(cross > 0) - int(cross < 0) + + +def segments_cross(a0: Point, a1: Point, b0: Point, b1: Point) -> bool: + """True when the two segments properly intersect. + + Each segment's endpoints lie strictly on opposite sides of the other's + line. Shared endpoints (split, merge), touching and collinear segments are + not crossings. + """ + return ( + _orientation(a0, a1, b0) * _orientation(a0, a1, b1) < 0 + and _orientation(b0, b1, a0) * _orientation(b0, b1, a1) < 0 + ) + + +def drop_crossing_links( + links: dict[Hashable, tuple[Point, Point, float]], +) -> tuple[list, dict]: + """Keep links cheapest first; a link crossing a kept one is dropped. + + ``links`` maps a key to ``(start, end, cost)``. Returns the kept keys + (cheapest first) and ``{dropped key: key of the kept link it crossed}``. + """ + kept: list = [] + dropped: dict = {} + for key, (start, end, _) in sorted(links.items(), key=lambda kv: (kv[1][2], kv[0])): + blocker = next((k for k in kept if segments_cross(start, end, *links[k][:2])), None) + if blocker is None: + kept.append(key) + else: + dropped[key] = blocker + return kept, dropped diff --git a/src/adapt/modules/tracking/matching/geometry.py b/src/adapt/modules/tracking/matching/geometry.py index d16d7e2..652b748 100644 --- a/src/adapt/modules/tracking/matching/geometry.py +++ b/src/adapt/modules/tracking/matching/geometry.py @@ -1,141 +1,155 @@ # Copyright © 2026, UChicago Argonne, LLC # See LICENSE for terms and disclaimer. -"""Pure geometry primitives for geometry-first matching. +"""Pure geometry of tracker v2, in grid-index space. -Bidirectional overlap fractions, hull dilation (buffering), field-agnostic -centroids, characteristic-length strategies, and the dimensionless pair cost -``cost = m + d/L``. No graph, no I/O, no assumption that the field is reflectivity. +Size-dependent growth of footprints and cells (measured from the shape +boundary), the two overlap fractions on the grown shapes and their combined +mismatch ``u``, the intensity excess above the cell threshold, the +e²-weighted core centre, and the shape-aware (Mahalanobis) residual. No graph, +no I/O; the field is never assumed to be reflectivity. """ import math +from dataclasses import dataclass import numpy as np -from scipy.ndimage import binary_dilation +from scipy.ndimage import distance_transform_edt __all__ = [ - "LENGTH_SCALES", - "bidirectional_overlap", - "buffer_pixels_from_km", - "dilate_hull", - "geometric_mismatch", - "length_scale", - "mask_centroid", - "mass_weighted_centroid", - "pair_cost", + "PairOverlap", + "cell_intensity", + "core_centre", + "footprint_pairs", + "grow", + "growth_radius_km", + "mahalanobis", + "second_moment", ] -# 8-connected structuring element for isotropic dilation. -_STRUCT = np.ones((3, 3), dtype=bool) +# Variance of a uniform unit pixel: keeps a one-pixel shape's moment invertible. +_PIXEL_VARIANCE = 1.0 / 12.0 -def buffer_pixels_from_km(buffer_km: float, pixel_size_m: float) -> int: - """Number of dilation iterations for a buffer distance, given metres-per-pixel.""" - if pixel_size_m <= 0: - raise ValueError("pixel_size_m must be positive") - return int(math.ceil(buffer_km * 1000.0 / pixel_size_m)) +def growth_radius_km(area_km2: float, r_max_km: float, l0_km: float) -> float: + """``r = r_max (1 − √A / ℓ0)₊``: small cells grow by up to r_max, cells of size ℓ0 not.""" + return r_max_km * max(0.0, 1.0 - math.sqrt(area_km2) / l0_km) -def dilate_hull(hull_mask: np.ndarray, buffer_pixels: int) -> np.ndarray: - """Dilate a boolean hull mask by ``buffer_pixels`` (0 → unchanged).""" - if buffer_pixels <= 0: - return hull_mask - return binary_dilation(hull_mask, structure=_STRUCT, iterations=buffer_pixels) +def grow(mask: np.ndarray, radius_px: float) -> np.ndarray: + """Every pixel within ``radius_px`` of the shape (Euclidean, from the boundary).""" + if radius_px <= 0.0 or not mask.any(): + return mask + return distance_transform_edt(~mask) <= radius_px -def bidirectional_overlap(hull: np.ndarray, cell: np.ndarray) -> tuple[float, float]: - """Return (Opc, Ocp). +@dataclass(frozen=True) +class PairOverlap: + """Overlap of a grown footprint H⁺ with one grown current cell M⁺ (same radius).""" - Opc = intersection / candidate area (how much of the cell the hull explains). - Ocp = intersection / hull area (how much of the hull the cell covers). - """ - intersection = float(np.count_nonzero(hull & cell)) - if intersection == 0.0: - return 0.0, 0.0 - candidate_area = float(np.count_nonzero(cell)) - hull_area = float(np.count_nonzero(hull)) - opc = intersection / candidate_area if candidate_area > 0 else 0.0 - ocp = intersection / hull_area if hull_area > 0 else 0.0 - return opc, ocp - - -def mask_centroid(mask: np.ndarray) -> tuple[float, float]: - """Geometric centroid ``(row, col)`` of a boolean mask (index space).""" - ys, xs = np.nonzero(mask) - return float(ys.mean()), float(xs.mean()) + label: int + intersection_px: int + footprint_grown_px: int + cell_grown_px: int + @property + def o_c(self) -> float: + """Share of the grown cell covered: |H⁺∩M⁺| / |M⁺|.""" + return self.intersection_px / self.cell_grown_px -def mass_weighted_centroid(field: np.ndarray, mask: np.ndarray) -> tuple[float, float]: - """Field-weighted centroid ``(row, col)`` in index space. - - Weights are ``field - min(field within mask)`` clamped to ≥ 0, so the result is - well defined for any field (reflectivity, brightness temperature, vertical wind, - …). Falls back to the geometric centroid when the total weight is zero. - """ - ys, xs = np.nonzero(mask) - values = field[ys, xs].astype(float) - weights = values - values.min() - total = float(weights.sum()) - if total <= 0.0: - return float(ys.mean()), float(xs.mean()) - return float((weights * ys).sum() / total), float((weights * xs).sum() / total) - + @property + def o_h(self) -> float: + """Share of the grown footprint covered: |H⁺∩M⁺| / |H⁺|.""" + return self.intersection_px / self.footprint_grown_px -def _equiv_diameter( - hull_area_px: float, cell_area_px: float, pixel_area_m2: float, fixed_km: float -) -> float: - area_m2 = hull_area_px * pixel_area_m2 - return 2.0 * math.sqrt(area_m2 / math.pi) if area_m2 > 0 else 0.0 + @property + def u(self) -> float: + """Overlap mismatch ``1 − √(O^c O^h)``: 0 when the shapes coincide, 1 when they touch.""" + return 1.0 - math.sqrt(self.o_c * self.o_h) -def _sum_radii( - hull_area_px: float, cell_area_px: float, pixel_area_m2: float, fixed_km: float -) -> float: - hull_r = math.sqrt(hull_area_px * pixel_area_m2 / math.pi) if hull_area_px > 0 else 0.0 - cell_r = math.sqrt(cell_area_px * pixel_area_m2 / math.pi) if cell_area_px > 0 else 0.0 - return hull_r + cell_r +def _box(mask: np.ndarray, margin: int) -> tuple[slice, slice]: + rows, cols = np.nonzero(mask) + return ( + slice(max(rows.min() - margin, 0), rows.max() + margin + 1), + slice(max(cols.min() - margin, 0), cols.max() + margin + 1), + ) -def _fixed_km( - hull_area_px: float, cell_area_px: float, pixel_area_m2: float, fixed_km: float -) -> float: - return fixed_km * 1000.0 +def footprint_pairs( + footprint: np.ndarray, labels: np.ndarray, radius_px: float +) -> list[PairOverlap]: + """Every current cell whose grown shape meets the grown footprint. - -# Registered length-scale strategies (name → callable), all with a uniform signature. -LENGTH_SCALES = { - "hull_equiv_diameter": _equiv_diameter, - "sum_radii": _sum_radii, - "fixed_km": _fixed_km, -} + Footprint and cell are grown by the same radius; the fractions are taken + on the grown shapes. Cells are grown whole, wherever they extend. + """ + if not footprint.any(): + return [] + margin = math.ceil(radius_px) + 1 + search = _box(footprint, 2 * margin) + near = grow(footprint[search], 2.0 * radius_px) & (labels[search] > 0) + pairs: list[PairOverlap] = [] + for label in np.unique(labels[search][near]): + cell = labels == label + box = _box(footprint | cell, margin) + h_grown = grow(footprint[box], radius_px) + m_grown = grow(cell[box], radius_px) + intersection = int(np.count_nonzero(h_grown & m_grown)) + if intersection: + pairs.append( + PairOverlap( + int(label), + intersection, + int(np.count_nonzero(h_grown)), + int(np.count_nonzero(m_grown)), + ) + ) + return pairs + + +def _excess(field: np.ndarray, mask: np.ndarray, threshold: float, polarity: int) -> np.ndarray: + values = np.nan_to_num(field[mask].astype(float), nan=threshold) + return np.clip(polarity * (values - threshold), 0.0, None) + + +def cell_intensity( + field: np.ndarray, mask: np.ndarray, threshold: float, polarity: int +) -> tuple[float, float]: + """Intensity mass ``M = Σ e`` and mean excess ``ē = M / A`` of one cell. + + ``e = polarity · (f − threshold)₊``: positive above a reflectivity threshold + (polarity +1) and below a brightness-temperature one (polarity −1), zero at + the cell edge. Missing values carry no excess. + """ + e = _excess(field, mask, threshold, polarity) + return float(e.sum()), float(e.mean()) -def length_scale( - name: str, - hull_area_px: float, - cell_area_px: float, - pixel_area_m2: float, - fixed_km: float, -) -> float: - """Characteristic length L (metres) for the selected strategy.""" - try: - fn = LENGTH_SCALES[name] - except KeyError: - raise ValueError(f"Unknown length_scale '{name}'; valid: {sorted(LENGTH_SCALES)}") from None - return fn(hull_area_px, cell_area_px, pixel_area_m2, fixed_km) +def core_centre( + field: np.ndarray, mask: np.ndarray, threshold: float, polarity: int +) -> tuple[float, float]: + """``(row, col)`` of the e²-weighted centre. + The square pulls the centre towards the intense cores without jumping to + one pixel. A cell with no excess anywhere has no core, so its centre is + its geometric centre. + """ + rows, cols = np.nonzero(mask) + weight = _excess(field, mask, threshold, polarity) ** 2 + total = float(weight.sum()) + if total == 0.0: + return float(rows.mean()), float(cols.mean()) + return float((weight * rows).sum() / total), float((weight * cols).sum() / total) -def geometric_mismatch(opc: float, ocp: float) -> float: - """Overlap mismatch ``m = 1 - sqrt(Opc)*sqrt(Ocp)`` in [0, 1].""" - return 1.0 - math.sqrt(opc) * math.sqrt(ocp) +def second_moment(mask: np.ndarray) -> np.ndarray: + """Second-moment (covariance) matrix of a shape's pixels, (row, col) order, px².""" + points = np.stack(np.nonzero(mask)).astype(float) + centred = points - points.mean(axis=1, keepdims=True) + return centred @ centred.T / points.shape[1] + _PIXEL_VARIANCE * np.eye(2) -def pair_cost(opc: float, ocp: float, displacement_m: float, length_m: float) -> float: - """Dimensionless geometry-first cost ``m + d/L``. - Falls back to the overlap term alone when L is degenerate (zero-area hull). - """ - m = geometric_mismatch(opc, ocp) - if length_m <= 0.0: - return m - return m + displacement_m / length_m +def mahalanobis(delta: np.ndarray, sigma: np.ndarray) -> float: + """``√(δᵀ Σ⁻¹ δ)``: a displacement in units of the shape's extent in its direction.""" + return float(math.sqrt(delta @ np.linalg.solve(sigma, delta))) diff --git a/src/adapt/modules/tracking/matching/hungarian.py b/src/adapt/modules/tracking/matching/hungarian.py index 13a39ca..2432c03 100644 --- a/src/adapt/modules/tracking/matching/hungarian.py +++ b/src/adapt/modules/tracking/matching/hungarian.py @@ -1,13 +1,13 @@ # Copyright © 2026, UChicago Argonne, LLC # See LICENSE for terms and disclaimer. -"""Per-component Hungarian assignment (last resort). +"""Per-component Hungarian assignment with a "no link" option. -Runs ``scipy.optimize.linear_sum_assignment`` over the validated edges *inside one -connected component only* — never a global matrix. Non-edge cells in the padded -matrix are dropped, so a component with unequal sides naturally leaves the surplus -objects unmatched (they become births / dissipations upstream). This is the only -home for ``scipy`` optimisation in the tracker. +Runs ``scipy.optimize.linear_sum_assignment`` over the admissible links of one +group of competing cells. Every previous cell also has its own "no link" +column at the cost ceiling, so a link is never forced when every candidate is +poor (Jaqaman et al. 2008 structure). The only home for ``scipy`` optimisation +in the tracker. """ import numpy as np @@ -15,35 +15,37 @@ __all__ = ["HungarianMatcher"] -# Larger than any real geometric cost; padding cells are dropped after assignment. +# Larger than any admissible cost: marks a non-edge, never chosen over no-link. _SENTINEL = 1.0e9 class HungarianMatcher: - """Optimal assignment restricted to one ambiguous connected component.""" + """Optimal assignment restricted to one group of competing cells.""" @staticmethod def match( prev_indices: list[int], curr_indices: list[int], costs: dict[tuple[int, int], float], + no_link_cost: float, ) -> list[tuple[int, int]]: - """Return the minimum-cost ``(prev_idx, curr_idx)`` matches for the component. + """Return the minimum-cost ``(prev_idx, curr_idx)`` links of the group. - ``costs`` holds only validated edges; assignments landing on a non-edge - (padding) are discarded so unequal-sized components leave a surplus unmatched. + ``costs`` holds admissible links only. A previous cell left unlinked + costs ``no_link_cost``; a link at exactly that cost is still preferred. """ if not prev_indices or not curr_indices: return [] - matrix = np.full((len(prev_indices), len(curr_indices)), _SENTINEL, dtype=float) + n, m = len(prev_indices), len(curr_indices) + matrix = np.full((n, m + n), _SENTINEL, dtype=float) for a, i in enumerate(prev_indices): for b, j in enumerate(curr_indices): if (i, j) in costs: matrix[a, b] = costs[(i, j)] + matrix[a, m + a] = np.nextafter(no_link_cost, np.inf) rows, cols = linear_sum_assignment(matrix) - matched: list[tuple[int, int]] = [] - for a, b in zip(rows, cols, strict=False): - pair = (prev_indices[a], curr_indices[b]) - if pair in costs: # real validated edge only - matched.append(pair) - return matched + return sorted( + (prev_indices[a], curr_indices[b]) + for a, b in zip(rows, cols, strict=True) + if b < m and (prev_indices[a], curr_indices[b]) in costs + ) diff --git a/src/adapt/modules/tracking/matching/validation.py b/src/adapt/modules/tracking/matching/validation.py deleted file mode 100644 index 618a004..0000000 --- a/src/adapt/modules/tracking/matching/validation.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright © 2026, UChicago Argonne, LLC -# See LICENSE for terms and disclaimer. - -"""Hard geometric validation gate. - -Removes obviously impossible candidate pairs using *bidirectional* overlap: a pair -survives only when the candidate is sufficiently covered by the projected hull -(Opc) AND the hull is sufficiently covered by the candidate (Ocp). Bidirectionality -rejects tiny cells inside large hulls, merged cells engulfing a prediction, and -grazing contacts — cases a single IoU or one-sided fraction would let through. - -The kinematic gate (speed/acceleration) is applied by the orchestrator, which owns -per-track velocity; this stage is pure geometry and returns the overlaps so callers -can reuse them for the cost. -""" - -from dataclasses import dataclass - -import numpy as np - -from adapt.modules.tracking.matching.geometry import bidirectional_overlap - -__all__ = ["GeometricValidator", "ValidationResult"] - - -@dataclass(frozen=True) -class ValidationResult: - """Overlaps for a candidate pair plus whether it clears the bidirectional gate.""" - - opc: float - ocp: float - passed: bool - - -class GeometricValidator: - """Bidirectional-overlap hard gate.""" - - def __init__(self, min_candidate_overlap: float, min_projected_overlap: float): - self.min_candidate_overlap = min_candidate_overlap - self.min_projected_overlap = min_projected_overlap - - def validate(self, hull: np.ndarray, cell: np.ndarray) -> ValidationResult: - opc, ocp = bidirectional_overlap(hull, cell) - passed = opc >= self.min_candidate_overlap and ocp >= self.min_projected_overlap - return ValidationResult(opc=opc, ocp=ocp, passed=passed) diff --git a/src/adapt/modules/tracking/models.py b/src/adapt/modules/tracking/models.py index 51305c0..cda291e 100644 --- a/src/adapt/modules/tracking/models.py +++ b/src/adapt/modules/tracking/models.py @@ -1,75 +1,16 @@ # Copyright © 2026, UChicago Argonne, LLC # See LICENSE for terms and disclaimer. -"""Value types for the tracker: match methods, error/diagnostic codes, motion state. +"""Structured tracking diagnostic codes (logged; not pipeline-fatal).""" -Plain enums and frozen dataclasses — no logic, no I/O. Shared by the matching, -motion, and event layers so a decision and its explanation travel together. -""" - -from dataclasses import dataclass from enum import StrEnum -__all__ = [ - "MatchDiagnostics", - "MatchMethod", - "TrackMotionState", - "TrackingError", -] - - -class MatchMethod(StrEnum): - """How an accepted match was decided.""" - - PROPAGATED = "PROPAGATED" # deterministic constraint propagation (no optimisation) - HUNGARIAN = "HUNGARIAN" # optimal assignment inside an ambiguous component - SPLIT = "SPLIT" - MERGE = "MERGE" +__all__ = ["TrackingError"] class TrackingError(StrEnum): - """Structured tracking diagnostic codes (logged; not pipeline-fatal).""" + """Scan-cadence codes: why a scan pair restarted every track, or was flagged.""" - # Scan-cadence / gap problems NON_MONOTONIC_TIME = "NON_MONOTONIC_TIME" TRACK_GAP_EXCEEDED = "TRACK_GAP_EXCEEDED" IRREGULAR_SCAN_CADENCE = "IRREGULAR_SCAN_CADENCE" - # Candidate-pair rejections - OVERLAP_REJECTED = "OVERLAP_REJECTED" - VELOCITY_EXCEEDED = "VELOCITY_EXCEEDED" - ACCELERATION_EXCEEDED = "ACCELERATION_EXCEEDED" - - -@dataclass(frozen=True) -class MatchDiagnostics: - """Per-accepted-match explainability record (persisted with the event row). - - ``opc``/``ocp`` are the bidirectional overlap fractions; ``centroid_distance_m`` - is the projected-hull→candidate residual; ``final_cost`` is ``m + d/L`` (+ penalty). - """ - - opc: float | None = None - ocp: float | None = None - centroid_distance_m: float | None = None - speed_ms: float | None = None - heading_change_deg: float | None = None - area_ratio: float | None = None - final_cost: float | None = None - match_method: str | None = None - - -@dataclass(frozen=True) -class TrackMotionState: - """Per-track velocity carried forward for acceleration and heading checks. - - ``speed`` is the running mean of the track's step speeds (m/s) over - ``n_steps`` — the acceleration cap references the track's established - motion, not one jittery step; ``heading`` is the last step's direction in - radians measured as ``atan2(vy, vx)``. ``has_velocity`` is False until a - track has been observed across two scans. - """ - - speed: float = 0.0 - heading: float = 0.0 - has_velocity: bool = False - n_steps: int = 0 diff --git a/src/adapt/modules/tracking/module.py b/src/adapt/modules/tracking/module.py index 9ccca3f..e57aa79 100644 --- a/src/adapt/modules/tracking/module.py +++ b/src/adapt/modules/tracking/module.py @@ -1,240 +1,171 @@ # Copyright © 2026, UChicago Argonne, LLC # See LICENSE for terms and disclaimer. -"""Track segmented geophysical objects across consecutive scans (geometry-first). - -The tracker is **field-agnostic**: the pixel field (reflectivity, brightness temperature, -vertical wind, …) is used only as a centroid weight, never assumed to be reflectivity. -Association is resolved as progressively stronger constraints, with optimisation last: - -1. Registration projected hulls (minute nearest the real gap) -2. Dilate hulls by ``projected_hull_buffer_km`` → liberal candidate pairs (high recall) -3. Hard gate: bidirectional overlap (Opc, Ocp) + kinematic speed/acceleration caps -4. Deterministic constraint propagation for mutually-unique matches (no optimisation) -5. Connected components → Hungarian assignment inside ambiguous groups only -6. Split / merge on the leftover born / dissipated cells -7. Initiation / termination - -Each object gets a stable `cell_uid`; lineage is graph edges. State lives in -``graph.TrackingGraph``; matching in ``matching/``; uid generation in ``identity``; event -rows in ``events``. ``CellTracker`` here is orchestration only. - -Scan outputs: -1. **tracked_cells**: per-observation rows for the current scan -2. **cell_events**: explicit lineage/event rows (CONTINUE, SPLIT, MERGE, INITIATION, TERMINATION) - -Inspired by TINT (Raut et al., 2021) but mask/geometry-driven rather than centroid-only. - -Author: Bhupendra Raut, ANL. - -References: Raut, B. A., Jackson, R., Picel, M., Collis, S. M., Bergemann, M., & Jakob, C. -(2021). An adaptive tracking algorithm for convection in simulated and remote sensing data. -Journal of Applied Meteorology and Climatology, 60(4), 513-526. +"""Track segmented cells across consecutive scans (tracker v2). + +Per scan pair, in order: + +1. Predict: each previous cell's footprint is the cell advected by the flow. +2. Grow footprint and cell by r = r_max (1 − √A/ℓ0)₊; overlap fractions O^c, O^h + on the grown shapes; u = 1 − √(O^c O^h). +3. Gates (hard): u ≤ u_max, speed ≤ v_max, speed change ≤ additive limit. +4. Cost c = u + w_d d + h (shape-aware residual of the e²-weighted centre, + heading term); c ≤ c_max. +5. Joint assignment per group of competing cells, no-link at c_max, no crossing. +6. Lineage of what is left (fractions and crossing test; the speed gates bind + continuations and resumptions only): resumption of a latent track, split child, merge + source, initiation; unlinked tracks become latent for up to N scans. +7. Identity at splits and merges follows the larger score S = log A + γ log ē. + +Causal: a scan's decisions use only that scan and earlier ones, and are never +revised. Every decision is logged (``decisions()``). """ import logging import math -from dataclasses import dataclass +from dataclasses import dataclass, replace +from typing import cast import numpy as np import pandas as pd import xarray as xr -from adapt.contracts import TrackingDecisions, TrackingSplitMergeTest, stat_column -from adapt.modules.tracking.decision_log import FrameLog, LogCell +from adapt.contracts import ( + TrackingCell, + TrackingDecisions, + TrackingIdentity, + TrackingLatent, + TrackingLineage, +) +from adapt.modules.tracking.config import TrackingConfig +from adapt.modules.tracking.decision_log import ScanLog from adapt.modules.tracking.events import ( + EndPoint, + LinkDiagnostics, build_cell_events_dataframe, - event_continue, - event_initiation, - event_merge, - event_split, - event_termination, -) -from adapt.modules.tracking.graph import TrackingGraph -from adapt.modules.tracking.identity import ( - _cell_uid_from_signature, - track_signature_v2, + event_row, ) -from adapt.modules.tracking.matching.assignment import AssignmentGraph, ConstraintPropagator -from adapt.modules.tracking.matching.candidate import CandidateGenerator -from adapt.modules.tracking.matching.geometry import ( - buffer_pixels_from_km, - length_scale, - mask_centroid, - mass_weighted_centroid, - pair_cost, +from adapt.modules.tracking.identity import _cell_uid_from_signature, track_signature_v2 +from adapt.modules.tracking.latent import ( + ORIGIN_MERGED, + ORIGIN_TERMINATED, + LatentTrack, + carry_footprint, ) -from adapt.modules.tracking.matching.hungarian import HungarianMatcher -from adapt.modules.tracking.matching.validation import GeometricValidator -from adapt.modules.tracking.models import ( - MatchDiagnostics, - MatchMethod, - TrackingError, - TrackMotionState, +from adapt.modules.tracking.lineage import ( + IdentityCandidate, + best_partner, + choose_identity, ) -from adapt.modules.tracking.motion import ( - MotionValidator, - heading_change_degrees, - heading_change_radians, +from adapt.modules.tracking.matching.assignment import Link, components, solve_component +from adapt.modules.tracking.matching.gates import drop_crossing_links, segments_cross +from adapt.modules.tracking.matching.geometry import mahalanobis +from adapt.modules.tracking.models import TrackingError +from adapt.modules.tracking.observations import ( + GATE_PASS, + Cell, + Footprint, + Grid, + Pair, + Track, + evaluate_pairs, + extract_cells, + latent_footprint, + live_footprint, ) from adapt.utils.time import normalize_time_scalar +__all__ = ["CellTracker", "_cell_uid_from_signature", "track_signature_v2"] + +logger = logging.getLogger(__name__) + # Beyond this ratio between consecutive scan intervals the cadence is flagged # irregular (diagnostic only — no track reset). _CADENCE_IRREGULAR_RATIO = 2.0 -# Re-exported for the public import surface (tests + tooling import these from here). -__all__ = [ - "CellTracker", - "TrackingGraph", - "_cell_uid_from_signature", - "track_signature_v2", -] +Segment = tuple[str, tuple[float, float], tuple[float, float]] # (uid, start, end) -logger = logging.getLogger(__name__) + +@dataclass(frozen=True) +class _Interval: + """Accepted motion of every track, live or latent, over one scan interval.""" + + t0_s: float + t1_s: float + segments: tuple[Segment, ...] @dataclass(frozen=True) -class _EdgeCost: - """Pre-computed geometry for a surviving candidate edge (prev_idx ↔ curr_idx).""" +class _LastScan: + """The previous scan: the one every live track was last observed in.""" + + scan_id: str + time: object + time_s: float + + +@dataclass +class _Outcome: + """What one scan pair decided, cell by cell (indices into prev tracks / labels).""" + + continuing: dict[int, int] # prev idx → curr label + method: dict[int, str] # prev idx → ASSIGNED | IDENTITY_MERGE | IDENTITY_SPLIT + resumed: dict[int, Pair] # curr label → latent pair + split: dict[int, int] # curr label → parent prev idx + merged: dict[int, int] # prev idx → survivor curr label + + +def _epoch_s(time_val) -> float: + ts = pd.Timestamp(time_val) + return float((ts.tz_localize("UTC") if ts.tzinfo is None else ts).timestamp()) + - opc: float - ocp: float - displacement: float # metres, projected-hull centroid → candidate mass centroid - cost: float # m + d/L (+ heading penalty) +def _distance(fp: Footprint, cell: Cell, grid: Grid) -> float: + """Shape-aware distance of a cell's centre from a footprint's predicted centre.""" + if fp.sigma is None: # a footprint outside the domain is infinitely far + return math.inf + return mahalanobis(grid.delta_px(fp.predicted, cell.centre), fp.sigma) class CellTracker: - """Track segmented geophysical objects with a geometry-first pipeline. - - The tracker is field-agnostic (the field is used only as a centroid weight, - never assumed to be reflectivity) and resolves association as progressively - stronger constraints, using optimisation only as a last resort: - - 1. Registration projected hulls (minute nearest the real gap) - 2. Dilate hulls by ``projected_hull_buffer_km`` and generate candidates liberally - 3. Hard gate: bidirectional overlap (Opc, Ocp) + kinematic speed/acceleration caps - 4. Deterministic constraint propagation (mutually-unique matches) - 5. Connected components → Hungarian assignment inside ambiguous groups only - 6. Split / merge on the leftover born/dissipated cells - 7. Initiation / termination - - Cost is ``m + d/L`` where ``m = 1 − √Opc·√Ocp``, ``d`` is the mass-weighted - centroid's residual from the projected-hull centroid, and ``L`` is a configurable - characteristic length. An optional heading-change penalty is added on top. - """ - - def __init__(self, config): - self.split_overlap = config.split_overlap - self.merge_overlap = config.merge_overlap - self.core_threshold = config.core_field_threshold - self.field_var = config.field_var # generic field; not assumed to be reflectivity - self.labels_var = config.labels_var - self.uid_width = config.uid_width - - self.buffer_km = config.projected_hull_buffer_km - self.length_scale_name = config.length_scale - self.geometry_length_scale_km = config.geometry_length_scale_km - - self.max_tracking_gap_minutes: float = config.max_tracking_gap_minutes - self.max_tracking_gap_s: float = config.max_tracking_gap_minutes * 60.0 - - self.graph = TrackingGraph() - self.validator = GeometricValidator( - config.minimum_candidate_overlap, config.minimum_projected_overlap - ) - self.motion = MotionValidator( - config.max_speed_ms, config.max_speed_multiplier, config.acceleration_floor_ms - ) - self.heading_penalty_weight = config.heading_change_penalty_weight - self._previous_scan: tuple | None = None # (time, node_ids) - self._cell_identity: dict[int, tuple[str, str]] = {} - self._track_motion: dict[int, TrackMotionState] = {} # track_index → kinematics - self._prev_dt_s: float | None = None # last good scan interval (cadence check) - # Decision log of the scan being tracked; frozen into _decisions at the end. - self._log = FrameLog(dt_s=None, prev=[], curr_labels=[], reset_code="FIRST_SCAN") + """Track cells across scans; one ``track`` call per scan, in time order.""" + + def __init__(self, config: TrackingConfig): + self.cfg = config + self._tracks: list[Track] = [] + self._last: _LastScan | None = None + self._prev_dt_s: float | None = None + self._latent: list[LatentTrack] = [] + self._intervals: list[_Interval] = [] self._decisions: TrackingDecisions | None = None - logger.info( - "CellTracker initialized: buffer=%.1fkm min_opc=%.2f min_ocp=%.2f" - " length_scale=%s max_gap=%.1fmin max_speed=%.1fm/s", - self.buffer_km, - config.minimum_candidate_overlap, - config.minimum_projected_overlap, - self.length_scale_name, - self.max_tracking_gap_minutes, - config.max_speed_ms, - ) - - # ------------------------------------------------------------------ - # Public API - # ------------------------------------------------------------------ + # ── public API ───────────────────────────────────────────────────────── def track( - self, - ds_projected: xr.Dataset, - cell_stats_df: pd.DataFrame, - *, - scan_id: str, + self, ds_projected: xr.Dataset, cell_stats_df: pd.DataFrame, *, scan_id: str ) -> tuple[pd.DataFrame, pd.DataFrame]: - """Process one scan. - - ``scan_id`` (sha256 of the raw scan bytes) is the birth-identity - input for cell uids minted in this scan. - - Returns scan-local outputs: - - tracked_cells_df: one row per cell observation in this scan - - cell_events_df: explicit lineage/events (continue/split/merge/initiation/termination) - """ + """Process one scan; return its tracked cells and lineage events.""" if not scan_id: raise ValueError("CellTracker.track: scan_id is required") - self._scan_id = str(scan_id) - current_time = normalize_time_scalar(ds_projected.time.values) - cells_current = self._extract_cells_from_analyzer(ds_projected, cell_stats_df) - - events: list[dict] = [] - curr_labels = [int(cell["cell_id"]) for cell in cells_current] - if self._previous_scan is None: - self._log = FrameLog( - dt_s=None, prev=[], curr_labels=curr_labels, reset_code="FIRST_SCAN" + time = normalize_time_scalar(ds_projected.time.values) + time_s = _epoch_s(time) + grid = Grid.of(ds_projected) + cells = extract_cells(ds_projected, cell_stats_df, self.cfg, grid) + + reset_code, dt_s = self._reset_code(ds_projected, time_s) + log = ScanLog(dt_s=dt_s, reset_code=reset_code) + log.counts.update(prev=len(self._tracks), latent=len(self._latent)) + if reset_code is None: + uids, events, tracks = self._link( + self._last_scan(), ds_projected, cells, grid, time, time_s, scan_id, log ) - node_ids = self._initialize_tracks(current_time, cells_current) - self._previous_scan = (current_time, node_ids) - for j, node_id in enumerate(node_ids): - events.append( - event_initiation(self.graph, self._cell_identity, current_time, node_id) - ) - self._log.born(j, str(self.graph.get_node_attr(node_id, "cell_uid"))) - self._decisions = self._log.freeze() else: - prev_time, prev_node_ids = self._previous_scan - dt_s = self._to_epoch_seconds(current_time) - self._to_epoch_seconds(prev_time) - reset_code = self._gap_reset_code(dt_s, prev_time, current_time) - self._log = FrameLog( - dt_s=dt_s, - prev=self._log_cells(prev_node_ids), - curr_labels=curr_labels, - reset_code=reset_code, - ) - if reset_code is not None: - events = self._reset_tracks(prev_node_ids, current_time, cells_current) - else: - self._prev_dt_s = dt_s - events = self._track_frame_pair( - prev_node_ids, current_time, ds_projected, cells_current, dt_s - ) - current_node_ids = self.graph.get_nodes_at_time(current_time) - self._previous_scan = (current_time, current_node_ids) - - current_node_ids = self.graph.get_nodes_at_time(current_time) - tracked_cells_df = self._build_tracked_cells_current(current_time, current_node_ids) - cell_events_df = build_cell_events_dataframe(events) - return tracked_cells_df, cell_events_df + uids, events, tracks = self._restart(cells, time, time_s, scan_id, log) - def get_cell_identity(self, track_index: int) -> tuple[str, str]: - if track_index not in self._cell_identity: - raise ValueError(f"Missing cell identity for track_index={track_index}") - return self._cell_identity[track_index] + self._tracks = tracks + self._last = _LastScan(scan_id, time, time_s) + self._decisions = log.freeze() + return self._tracked_cells(time, cells, uids), build_cell_events_dataframe(events) def decisions(self) -> TrackingDecisions: """The decision record of the most recent ``track`` call (analysis only).""" @@ -242,729 +173,736 @@ def decisions(self) -> TrackingDecisions: raise ValueError("CellTracker.decisions: no scan has been tracked yet") return self._decisions - def _log_cells(self, node_ids: list[int]) -> list[LogCell]: - """Previous cells as the decision log identifies them.""" - cells = [] - for node_id in node_ids: - track_index = int(self.graph.get_node_attr(node_id, "track_index") or 0) - state = self._track_motion.get(track_index) - cells.append( - LogCell( - uid=str(self.graph.get_node_attr(node_id, "cell_uid")), - label=int(self.graph.get_node_attr(node_id, "cell_id")), - steps_before=state.n_steps if state is not None else 0, - ) - ) - return cells - - # ------------------------------------------------------------------ - # Scan-gap classification (physical time) - # ------------------------------------------------------------------ - - def _gap_reset_code(self, dt_s: float, prev_time, curr_time) -> str | None: - """Classify the inter-scan interval; log a structured code. + # ── scan classification and restart ──────────────────────────────────── - Returns the reset code when tracks must be terminated and restarted - (non-monotonic time or a gap above the hard limit) — never raises, never - matches across the gap. An irregular-but-monotonic cadence is a - diagnostic warning only and returns None. - """ + def _reset_code(self, ds: xr.Dataset, time_s: float) -> tuple[str | None, float | None]: + if self._last is None: + return "FIRST_SCAN", None + dt_s = time_s - self._last.time_s if dt_s <= 0: - logger.error( - "tracking_error code=%s prev=%s curr=%s dt_s=%.1f", - TrackingError.NON_MONOTONIC_TIME.value, - prev_time, - curr_time, - dt_s, - ) - return TrackingError.NON_MONOTONIC_TIME.value - if dt_s > self.max_tracking_gap_s: - logger.error( - "tracking_error code=%s dt_minutes=%.1f limit_minutes=%.1f", - TrackingError.TRACK_GAP_EXCEEDED.value, - dt_s / 60.0, - self.max_tracking_gap_minutes, - ) - return TrackingError.TRACK_GAP_EXCEEDED.value - if self._prev_dt_s is not None and ( - dt_s > _CADENCE_IRREGULAR_RATIO * self._prev_dt_s - or dt_s * _CADENCE_IRREGULAR_RATIO < self._prev_dt_s - ): - logger.warning( - "tracking_error code=%s dt_s=%.1f prev_dt_s=%.1f", - TrackingError.IRREGULAR_SCAN_CADENCE.value, - dt_s, - self._prev_dt_s, - ) - return None + code = TrackingError.NON_MONOTONIC_TIME.value + elif dt_s > self.cfg.max_tracking_gap_minutes * 60.0: + code = TrackingError.TRACK_GAP_EXCEEDED.value + elif "cell_projections" not in ds.data_vars: + code = "NO_PROJECTIONS" + else: + if self._prev_dt_s is not None and not ( + 1 / _CADENCE_IRREGULAR_RATIO <= dt_s / self._prev_dt_s <= _CADENCE_IRREGULAR_RATIO + ): + logger.warning( + "tracking_error code=%s dt_s=%.1f prev_dt_s=%.1f", + TrackingError.IRREGULAR_SCAN_CADENCE.value, + dt_s, + self._prev_dt_s, + ) + self._prev_dt_s = dt_s + return None, dt_s + logger.error("tracking_error code=%s dt_s=%.1f — tracks restart", code, dt_s) + return code, dt_s - def _reset_tracks( - self, prev_node_ids: list[int], curr_time, cells_current: list[dict] - ) -> list[dict]: - """Terminate every active track, then start fresh tracks for the current cells.""" + def _restart(self, cells, time, time_s, scan_id, log): + """End every live and latent track; every current cell starts a track.""" events = [ - event_termination(self.graph, self._cell_identity, curr_time, node_id, None) - for node_id in prev_node_ids + event_row( + time, + "TERMINATION", + EndPoint(t.uid, t.label), + None, + source_scan=(t.scan_id, t.scan_time), + ) + for t in self._tracks + ] + [ + event_row( + time, + "TERMINATION", + EndPoint(lt.uid, lt.label), + None, + source_scan=(lt.scan_id, lt.scan_time), + ) + for lt in self._latent ] - self._initialize_tracks(curr_time, cells_current) - for j, node_id in enumerate(self.graph.get_nodes_at_time(curr_time)): - events.append(event_initiation(self.graph, self._cell_identity, curr_time, node_id)) - self._log.born(j, str(self.graph.get_node_attr(node_id, "cell_uid"))) - self._decisions = self._log.freeze() - return events + log.counts["termination"] += len(events) + self._latent, self._intervals = [], [] + uids = {c.label: self._new_uid(scan_id, c.label) for c in cells} + for c in cells: + events.append(event_row(time, "INITIATION", None, EndPoint(uids[c.label], c.label))) + log.cells[c.label] = self._cell_record(c, uids[c.label], "INITIATION") + tracks = [self._new_track(c, uids[c.label], scan_id, time, time_s) for c in cells] + return uids, events, tracks + + # ── one scan pair ────────────────────────────────────────────────────── + + def _last_scan(self) -> _LastScan: + if self._last is None: + raise RuntimeError("CellTracker: a scan pair needs a previous scan") + return self._last + + def _link(self, last: _LastScan, ds, cells, grid, time, time_s, scan_id, log): + cfg = self.cfg + labels = ds[cfg.labels_var].values + by_label = {c.label: c for c in cells} + proj = np.asarray(ds["cell_projections"].values[0]) + live = [live_footprint(t, proj, grid, time_s - last.time_s, cfg) for t in self._tracks] + carried = self._carry_latent(ds, grid) + latent_fps = [latent_footprint(lt, time_s, grid, cfg) for lt in self._latent] + + live_pairs = { + (i, p.cell.label): p + for i, fp in enumerate(live) + for p in evaluate_pairs(fp, by_label, labels, grid, cfg) + } + latent_pairs = { + (k, p.cell.label): p + for k, fp in enumerate(latent_fps) + for p in evaluate_pairs(fp, by_label, labels, grid, cfg) + } + for p in [*live_pairs.values(), *latent_pairs.values()]: + log.pair(p.fp.uid, p.cell.label, **self._pair_record(p)) + + out = _Outcome({}, {}, {}, {}, {}) + self._assign(live_pairs, out, log) + current = tuple( + (live[i].uid, live[i].origin, by_label[j].centre) for i, j in out.continuing.items() + ) + self._resume(latent_pairs, cells, out, current, last.time_s, time_s, log) + self._split_and_merge(live, live_pairs, cells, out, current, log) + self._identity(live, live_pairs, by_label, grid, out, log) + return self._emit(last, live, live_pairs, cells, out, carried, time, time_s, scan_id, log) - # ------------------------------------------------------------------ - # Cell extraction - # ------------------------------------------------------------------ + def _carry_latent(self, ds: xr.Dataset, grid: Grid) -> dict[str, Segment]: + """Move every latent footprint with this scan's flow; one scan older. - @staticmethod - def _to_epoch_seconds(time_val) -> float: - ts = pd.Timestamp(time_val) - if ts.tzinfo is None: - ts = ts.tz_localize("UTC") - return float(ts.timestamp()) - - def _extract_cells_from_analyzer( - self, ds: xr.Dataset, cell_stats_df: pd.DataFrame - ) -> list[dict]: - """Merge per-cell stats with masks; compute mass-weighted metric centroids. - - The tracker's own centroid is the **field-weighted** centroid of each mask - (metres), so matching and motion are field-agnostic. Reflectivity-named stats - are carried through only for identity and output columns. + Returns each latent track's predicted step over this interval. """ - labels = ds[self.labels_var].values - field = ds[self.field_var].values - - # Stats for the CONFIGURED field, via the one authoritative minting - # convention. Guarded for the zero-cell case: the analyzer's empty - # structural stub carries no stat columns, and this runs every scan. - mean_col = stat_column(self.field_var, "mean") - max_col = stat_column(self.field_var, "max") - if not cell_stats_df.empty and ( - mean_col not in cell_stats_df.columns or max_col not in cell_stats_df.columns - ): - raise ValueError( - f"Tracking: cell_stats is missing {mean_col!r}/{max_col!r} for " - f"the configured tracking field {self.field_var!r}. The analyzer " - "whitelist (analyzer.radar_variables) must include it." - ) - - cell_props_map: dict[int, dict] = {} - for _, row in cell_stats_df.iterrows(): - lbl = int(row["cell_label"]) - cell_props_map[lbl] = { - "area": float(row["cell_area_sqkm"]), - # role keys: "…reflectivity" = the tracked field, whatever it is - "mean_reflectivity": float(row[mean_col]), - "max_reflectivity": float(row[max_col]), - "time_volume_start": row["time_volume_start"], - "centroid_mass_lat": float(row["cell_centroid_mass_lat"]), - "centroid_mass_lon": float(row["cell_centroid_mass_lon"]), - } + flow_x, flow_y = ds["heading_x"].values, ds["heading_y"].values + carried, steps = [], {} + for lt in self._latent: + mask, (d_row, d_col) = carry_footprint(lt.footprint, flow_x, flow_y) + predicted = (lt.predicted[0] + d_col * grid.sx, lt.predicted[1] + d_row * grid.sy) + carried.append(replace(lt, footprint=mask, predicted=predicted, age=lt.age + 1)) + steps[lt.uid] = (lt.uid, lt.predicted, predicted) + self._latent = carried + return steps + + def _assign(self, pairs: dict[tuple[int, int], Pair], out: _Outcome, log: ScanLog) -> None: + """Hungarian per component with no-link at c_max, then a scan-wide crossing check.""" + admissible = {e: p for e, p in pairs.items() if p.gate == GATE_PASS} + chosen: dict[tuple[int, int], Pair] = {} + groups = components(sorted(admissible)) + log.counts["components"] = len(groups) + for cid, (prevs, _) in enumerate(groups): + links = [ + Link(i, j, p.cost, p.fp.origin, p.cell.centre) + for (i, j), p in sorted(admissible.items()) + if i in prevs + ] + solution = solve_component(links, self.cfg.max_link_cost) + for i, j in (e for e in admissible if e[0] in prevs): + log.update_pair(pairs[(i, j)].fp.uid, j, component_id=cid, outcome="LOST") + for e, blocker in solution.crossing_excluded.items(): + self._mark_crossing(pairs, e, blocker, log) + for e in solution.chosen: + chosen[e] = admissible[e] + log.update_pair(admissible[e].fp.uid, e[1], margin=solution.margins[e]) + kept, dropped = drop_crossing_links( + {e: (p.fp.origin, p.cell.centre, p.cost) for e, p in chosen.items()} + ) + for e, blocker in dropped.items(): + self._mark_crossing(pairs, e, blocker, log) + for i, j in kept: + out.continuing[i] = j + out.method[i] = "ASSIGNED" + log.update_pair(pairs[(i, j)].fp.uid, j, outcome="CONTINUE") - x_coords = np.asarray(ds.x.values, dtype=float) - y_coords = np.asarray(ds.y.values, dtype=float) - sx = float(x_coords[1] - x_coords[0]) - sy = float(y_coords[1] - y_coords[0]) - x0, y0 = float(x_coords[0]), float(y_coords[0]) - pixel_area_km2 = abs(sx * sy) / 1e6 + @staticmethod + def _mark_crossing(pairs, edge, blocker, log: ScanLog) -> None: + log.counts["crossing_excluded"] += 1 + log.update_pair( + pairs[edge].fp.uid, edge[1], outcome="CROSSING", crossing_with_uid=pairs[blocker].fp.uid + ) - cells: list[dict] = [] - for cell_id in np.unique(labels): - if cell_id == 0: - continue - if cell_id not in cell_props_map: - logger.warning("Cell %d in labels but not in analyzer stats; skipping", cell_id) + def _resume(self, pairs, cells, out: _Outcome, current, prev_s, time_s, log: ScanLog) -> None: + """Offer latent tracks to orphans: same gates and cost over the elapsed time, + and no crossing in any intervening interval.""" + orphans = {c.label for c in cells} - set(out.continuing.values()) + admissible: dict[tuple[int, int], Pair] = {} + for (k, j), p in sorted(pairs.items()): + if j not in orphans: continue - mask = labels == cell_id - props = cell_props_map[cell_id] - core_area_km2 = float(np.sum(mask & (field > self.core_threshold)) * pixel_area_km2) - row_c, col_c = mass_weighted_centroid(field, mask) - cells.append( - { - "cell_id": int(cell_id), - "mask": mask, - "area": props["area"], - "area_px": float(np.count_nonzero(mask)), - "centroid_x": x0 + col_c * sx, # mass-weighted, metres - "centroid_y": y0 + row_c * sy, - "mean_reflectivity": props["mean_reflectivity"], - "max_reflectivity": props["max_reflectivity"], - "core_area": core_area_km2, - "time_volume_start": props["time_volume_start"], - "centroid_mass_lat": props["centroid_mass_lat"], - "centroid_mass_lon": props["centroid_mass_lon"], - } - ) - return cells - - def _new_cell_identity(self, cell: dict) -> tuple[str, str]: - signature = track_signature_v2(self._scan_id, cell["cell_id"]) - cell_uid = _cell_uid_from_signature(signature, width=self.uid_width) - return cell_uid, signature - - # ------------------------------------------------------------------ - # Track initialisation helpers - # ------------------------------------------------------------------ - - def _initialize_tracks(self, time, cells: list[dict]) -> list[int]: - node_ids = [] - for cell in cells: - track_index = self.graph.get_new_track_index() - cell_uid, track_signature = self._new_cell_identity(cell) - self._cell_identity[track_index] = (cell_uid, track_signature) - node_ids.append(self._add_cell_node(time, cell, track_index, cell_uid, track_signature)) - logger.debug("Initialized %d paths at time %s", len(cells), time) - return node_ids - - def _add_cell_node( - self, - time, - cell: dict, - track_index: int, - cell_uid: str | None = None, - track_signature: str | None = None, - ) -> int: - if cell_uid is None or track_signature is None: - cell_uid, track_signature = self.get_cell_identity(track_index) - return self.graph.add_observation( - time=time, - cell_id=cell["cell_id"], - track_index=track_index, - area=cell["area"], - centroid_x=cell["centroid_x"], - centroid_y=cell["centroid_y"], - mean_reflectivity=cell["mean_reflectivity"], - max_reflectivity=cell["max_reflectivity"], - core_area=cell["core_area"], - cell_uid=cell_uid, - track_signature=track_signature, + reason = None if p.gate == GATE_PASS else p.gate + if reason is None and self._resume_crosses(self._latent[k], p, current, prev_s, time_s): + reason = "CROSSING" + log.counts["crossing_excluded"] += 1 + log.update_pair(p.fp.uid, j, outcome="CROSSING") + if reason is None: + admissible[(k, j)] = p + log.lineage.append(self._hypothesis("RESUME", p, cost=p.cost, reason=reason)) + for prevs, _ in components(sorted(admissible)): + links = [ + Link(k, j, p.cost, p.fp.origin, p.cell.centre) + for (k, j), p in sorted(admissible.items()) + if k in prevs + ] + for k, j in solve_component(links, self.cfg.max_link_cost).chosen: + out.resumed[j] = admissible[(k, j)] + log.update_pair(admissible[(k, j)].fp.uid, j, outcome="RESUMED") + + def _resume_crosses(self, lt: LatentTrack, pair: Pair, current, prev_s, time_s) -> bool: + """Interpolate the resumed path to each intervening scan and test each sub-segment + against every other track's segment of that interval.""" + start, end, t_start = lt.centre, pair.cell.centre, lt.time_s + + def at(t: float) -> tuple[float, float]: + f = (t - t_start) / (time_s - t_start) + return start[0] + f * (end[0] - start[0]), start[1] + f * (end[1] - start[1]) + + intervals = [iv for iv in self._intervals if iv.t0_s >= t_start] + intervals.append(_Interval(prev_s, time_s, current)) + return any( + uid != lt.uid and segments_cross(at(iv.t0_s), at(iv.t1_s), a, b) + for iv in intervals + for uid, a, b in iv.segments ) - # ------------------------------------------------------------------ - # Frame-pair matching - # ------------------------------------------------------------------ + def _split_and_merge(self, live, pairs, cells, out: _Outcome, current, log: ScanLog) -> None: + """Split children among remaining orphans; merge sources among unlinked tracks.""" + survivors = {j: i for i, j in out.continuing.items()} + for j in ( + c.label for c in cells if c.label not in survivors and c.label not in out.resumed + ): + tested = { + i: p for (i, jj), p in sorted(pairs.items()) if jj == j and i in out.continuing + } + parent = self._lineage_partner(tested, "SPLIT", current, log) + if parent is not None: + out.split[j] = parent + for i in (i for i in range(len(live)) if i not in out.continuing): + tested = {j: p for (ii, j), p in sorted(pairs.items()) if ii == i and j in survivors} + target = self._lineage_partner(tested, "MERGE", current, log) + if target is not None: + out.merged[i] = target + log.lineage.append(self._latent_hypothesis(live[i])) + + def _lineage_partner(self, tested: dict[int, Pair], kind: str, current, log: ScanLog): + """Best partner by fraction among those passing the crossing test and the + threshold; every partner tested is logged. + + No speed gate: a fragment's centre moves from its parent's centre by up to + the parent's size, which is not motion (the speed limits apply to links of + one cell: continuations and resumptions). + """ + threshold = self.cfg.split_overlap if kind == "SPLIT" else self.cfg.merge_overlap + fractions = {} + for key, p in tested.items(): + fraction = p.overlap.o_c if kind == "SPLIT" else p.overlap.o_h + reason = None + if any( + uid != p.fp.uid and segments_cross(p.fp.origin, p.cell.centre, a, b) + for uid, a, b in current + ): + reason = "CROSSING" + if reason is None and fraction < threshold: + reason = "BELOW_THRESHOLD" + if reason is None: + fractions[key] = fraction + log.lineage.append( + self._hypothesis(kind, p, fraction=fraction, threshold=threshold, reason=reason) + ) + return best_partner(fractions, threshold) - def _update_motion( - self, prev_node: int, curr_cell: dict, track_index: int, dt_s: float - ) -> None: - """Record a continuing track's kinematics for acceleration/heading checks.""" - prev_cx = float(self.graph.get_node_attr(prev_node, "centroid_x")) - prev_cy = float(self.graph.get_node_attr(prev_node, "centroid_y")) - vx = (float(curr_cell["centroid_x"]) - prev_cx) / dt_s - vy = (float(curr_cell["centroid_y"]) - prev_cy) / dt_s - step_speed = math.hypot(vx, vy) - prior = self._track_motion.get(track_index) - n_steps = (prior.n_steps if prior is not None else 0) + 1 - mean_speed = ( - step_speed if prior is None else prior.speed + (step_speed - prior.speed) / n_steps - ) - self._track_motion[track_index] = TrackMotionState( - speed=mean_speed, - heading=math.atan2(vy, vx), - has_velocity=True, - n_steps=n_steps, + @staticmethod + def _latent_hypothesis(fp: Footprint) -> TrackingLineage: + """An unlinked track kept latent (chosen unless it merged).""" + return TrackingLineage( + "prev", fp.uid, fp.label, "LATENT", None, None, None, None, None, True, False, None ) - def _record_continue( - self, - i: int, - c: int, - all_prev_ids: list[int], - curr_cells: list[dict], - curr_time, - dt_s: float, - edge: "_EdgeCost", - method: MatchMethod, - matched_prev: dict[int, int], - matched_curr: dict[int, int], - ) -> dict: - """Create the CONTINUE node/edge for prev row ``i`` ↔ curr col ``c``. - - Shared by the constraint-propagation and Hungarian paths. Records diagnostics - from the pre-computed edge, updates the matched maps and the track's motion - state; returns the event row. - """ - prev_node = all_prev_ids[i] - track_index = int(self.graph.get_node_attr(prev_node, "track_index") or 0) - # Diagnostics use the track's prior motion — compute before _update_motion. - diagnostics = self._match_diagnostics(prev_node, curr_cells[c], dt_s, edge, method) - curr_node = self._add_cell_node(curr_time, curr_cells[c], track_index) - self.graph.add_edge(prev_node, curr_node, edge_type="CONTINUE", cost=edge.cost) - matched_prev[i] = curr_node - matched_curr[c] = curr_node - self._update_motion(prev_node, curr_cells[c], track_index, dt_s) - return event_continue( - self.graph, self._cell_identity, curr_time, prev_node, curr_node, edge.cost, diagnostics + @staticmethod + def _hypothesis( + kind, p: Pair, *, fraction=None, cost=None, threshold=None, reason=None + ) -> TrackingLineage: + """A lineage row. Uids that are final only at the end of the scan (the orphan's, + a merge survivor's) are filled in by ``_finish_lineage``.""" + if kind == "MERGE": + side, uid, label, partner_uid, partner_label = ( + "prev", + p.fp.uid, + p.fp.label, + None, + p.cell.label, + ) + else: + side, uid, label, partner_uid, partner_label = ( + "curr", + "", + p.cell.label, + p.fp.uid, + p.fp.label, + ) + return TrackingLineage( + side, + uid, + label, + kind, + partner_uid, + partner_label, + fraction, + cost, + threshold, + reason in (None, "IDENTITY"), # IDENTITY: made so by the identity rule + False, + reason, ) - def _match_diagnostics( - self, - prev_node: int, - curr_cell: dict, - dt_s: float, - edge: "_EdgeCost", - method: MatchMethod, - ) -> MatchDiagnostics: - """Assemble the per-match explainability record for an accepted CONTINUE.""" - prev_cx = float(self.graph.get_node_attr(prev_node, "centroid_x")) - prev_cy = float(self.graph.get_node_attr(prev_node, "centroid_y")) - prev_area = float(self.graph.get_node_attr(prev_node, "area")) - track_index = int(self.graph.get_node_attr(prev_node, "track_index") or 0) - - heading_change_deg = None - prev_state = self._track_motion.get(track_index) - if prev_state is not None and prev_state.has_velocity: - cand_heading = math.atan2( - float(curr_cell["centroid_y"]) - prev_cy, float(curr_cell["centroid_x"]) - prev_cx + def _identity( + self, live: list[Footprint], pairs, by_label, grid: Grid, out: _Outcome, log: ScanLog + ): + """The identity at a merge or split follows the larger score S; within δ_S the + candidate nearer the predicted position. Only candidates whose link would pass + the speed gates contend: a continuation always obeys the physical limits.""" + margin = self.cfg.identity_score_margin + survivors = {j: i for i, j in out.continuing.items()} + for s in sorted(set(out.merged.values())): + p = survivors[s] + contenders = [ + p, + *sorted( + i + for i, t in out.merged.items() + if t == s and pairs[(i, s)].speed_gate.code is None + ), + ] + if len(contenders) == 1: + continue + winner = self._decide( + "MERGE", + live[p].uid, + "prev", + [ + ( + i, + self._tracks[i].label, + self._tracks[i].score, + _distance(live[i], by_label[s], grid), + ) + for i in contenders + ], + p, + margin, + log, ) - heading_change_deg = heading_change_degrees(prev_state.heading, cand_heading) - - curr_area = float(curr_cell["area"]) - return MatchDiagnostics( - opc=edge.opc, - ocp=edge.ocp, - centroid_distance_m=edge.displacement, - speed_ms=edge.displacement / dt_s, - heading_change_deg=heading_change_deg, - area_ratio=(curr_area / prev_area if prev_area > 0 else None), - final_cost=edge.cost, - match_method=method, + if winner != p: + log.update_pair(live[p].uid, s, outcome="LOST") + del out.continuing[p], out.merged[winner] + out.continuing[winner], out.method[winner] = s, "IDENTITY_MERGE" + out.merged[p] = s + log.lineage.append( + self._hypothesis( + "MERGE", + pairs[(p, s)], + fraction=pairs[(p, s)].overlap.o_h, + threshold=self.cfg.merge_overlap, + reason="IDENTITY", + ) + ) + log.lineage.append(self._latent_hypothesis(live[p])) + children: dict[int, list[int]] = {} + for j, parent in out.split.items(): + children.setdefault(parent, []).append(j) + for parent, kids in sorted(children.items()): + if parent not in out.continuing: + continue # the parent itself merged away: its children stay split children + c = out.continuing[parent] + kids = [j for j in kids if pairs[(parent, j)].speed_gate.code is None] + if not kids: + continue + winner = self._decide( + "SPLIT", + live[parent].uid, + "curr", + [ + (j, j, by_label[j].score, _distance(live[parent], by_label[j], grid)) + for j in [c, *sorted(kids)] + ], + c, + margin, + log, + ) + if winner != c: + log.update_pair(live[parent].uid, c, outcome="LOST") + del out.split[winner] + out.continuing[parent], out.method[parent] = winner, "IDENTITY_SPLIT" + out.split[c] = parent + log.lineage.append( + self._hypothesis( + "SPLIT", + pairs[(parent, c)], + fraction=pairs[(parent, c)].overlap.o_c, + threshold=self.cfg.split_overlap, + reason="IDENTITY", + ) + ) + + @staticmethod + def _decide(kind, uid, side, contenders, assigned, margin, log: ScanLog): + winner, rule = choose_identity( + [IdentityCandidate(key, score, dist) for key, _, score, dist in contenders], margin ) + for key, label, score, dist in contenders: + log.identity.append( + TrackingIdentity( + kind, + uid, + side, + label, + score, + dist, + key == winner, + rule, + key == winner and winner != assigned, + ) + ) + if winner != assigned: + log.counts["identity_transfer"] += 1 + return winner + + # ── outputs of one scan pair ─────────────────────────────────────────── - def _build_edges( + def _emit( self, - all_prev_ids: list[int], - curr_cells: list[dict], - proj_labels: np.ndarray, - ds: xr.Dataset, - dt_s: float, - ) -> dict[tuple[int, int], _EdgeCost]: - """Generate candidates from buffered hulls and keep only validated edges. - - For every previous object the registration hull is dilated by the buffer, - every intersecting current cell becomes a candidate, and a pair survives - only if it clears the bidirectional-overlap gate and the kinematic gate. - Returns ``{(prev_idx, curr_idx): _EdgeCost}`` with the geometry-first cost. - """ - x_coords = np.asarray(ds.x.values, dtype=float) - y_coords = np.asarray(ds.y.values, dtype=float) - sx = float(x_coords[1] - x_coords[0]) - sy = float(y_coords[1] - y_coords[0]) - x0, y0 = float(x_coords[0]), float(y_coords[0]) - pixel_area_m2 = abs(sx * sy) - buffer_pixels = buffer_pixels_from_km(self.buffer_km, min(abs(sx), abs(sy))) - - prev_hulls = [ - proj_labels == self.graph.get_node_attr(node_id, "cell_id") for node_id in all_prev_ids - ] - curr_masks = [cell["mask"] for cell in curr_cells] - # Buffer only widens candidate recall; the overlap gate and cost are measured - # against the true (un-buffered) projected hull so the buffer never dilutes them. - _, pairs = CandidateGenerator(buffer_pixels).generate(prev_hulls, curr_masks) - - hull_area_px = [float(np.count_nonzero(h)) for h in prev_hulls] - hull_centroid_m: list[tuple[float, float] | None] = [] - for hull in prev_hulls: - if hull.any(): - row_c, col_c = mask_centroid(hull) - hull_centroid_m.append((x0 + col_c * sx, y0 + row_c * sy)) - else: - hull_centroid_m.append(None) - - edges: dict[tuple[int, int], _EdgeCost] = {} - for i, j in pairs: - prev_node = all_prev_ids[i] - track_index = int(self.graph.get_node_attr(prev_node, "track_index") or 0) - result = self.validator.validate(prev_hulls[i], curr_masks[j]) - hull_c = hull_centroid_m[i] or (float("nan"), float("nan")) - self._log.candidate( - i, - j, - hull_area_px=int(hull_area_px[i]), - hull_centroid_x=hull_c[0], - hull_centroid_y=hull_c[1], - curr_area_px=int(curr_cells[j]["area_px"]), - curr_centroid_x=float(curr_cells[j]["centroid_x"]), - curr_centroid_y=float(curr_cells[j]["centroid_y"]), - intersection_px=int(np.count_nonzero(prev_hulls[i] & curr_masks[j])), - opc=result.opc, - ocp=result.ocp, - overlap_passed=result.passed, + last: _LastScan, + live, + pairs, + cells, + out: _Outcome, + carried, + time, + time_s, + scan_id, + log, + ): + cfg = self.cfg + by_label = {c.label: c for c in cells} + uids: dict[int, str] = {} + events: list[dict] = [] + tracks: list[Track] = [] + segments: list[Segment] = [] + prev_scan = (last.scan_id, last.time) + + for i, j in sorted(out.continuing.items(), key=lambda e: e[1]): + track, p = self._tracks[i], pairs[(i, j)] + uids[j] = track.uid + events.append( + event_row( + time, + "CONTINUE", + EndPoint(track.uid, track.label), + EndPoint(track.uid, j), + source_scan=prev_scan, + diagnostics=self._diagnostics(p, out.method[i]), + is_dominant=True, + ) ) - if not result.passed: - # Same "tracking_error code=" grammar as the gap and kinematic - # rejections, so one grep explains why a track ended. - logger.debug( - "tracking_error code=%s track=%d curr=%d opc=%.2f ocp=%.2f", - TrackingError.OVERLAP_REJECTED.value, - track_index, - j, - result.opc, - result.ocp, + log.update_pair(track.uid, j, outcome="CONTINUE") + log.cells[j] = self._cell_record(by_label[j], track.uid, "CONTINUE") + tracks.append(self._moved(by_label[j], track.uid, p, scan_id, time, time_s)) + segments.append((track.uid, track.centre, by_label[j].centre)) + + resumed_uids = set() + for j, p in sorted(out.resumed.items()): + lt = next(lt for lt in self._latent if lt.uid == p.fp.uid) + resumed_uids.add(lt.uid) + uids[j] = lt.uid + events.append( + event_row( + time, + "RESUMED", + EndPoint(lt.uid, lt.label), + EndPoint(lt.uid, j), + source_scan=(lt.scan_id, lt.scan_time), + diagnostics=self._diagnostics(p, "RESUMED"), + is_dominant=True, ) - continue - - prev_cx = float(self.graph.get_node_attr(prev_node, "centroid_x")) - prev_cy = float(self.graph.get_node_attr(prev_node, "centroid_y")) - curr_cx = float(curr_cells[j]["centroid_x"]) - curr_cy = float(curr_cells[j]["centroid_y"]) - state = self._track_motion.get(track_index) - previous_speed = state.speed if state and state.has_velocity else None - - decision = self.motion.check(prev_cx, prev_cy, curr_cx, curr_cy, dt_s, previous_speed) - self._log.kinematic( - i, - j, - speed_ms=decision.speed_ms, - previous_speed_ms=previous_speed, - accel_cap_ms=decision.cap_ms, - kinematic_code=decision.code.value if decision.code is not None else None, ) - if not decision.ok: # B3 kinematic gate - logger.debug( - "tracking_error code=%s track=%d curr=%d speed=%.1fm/s prev_speed=%s", - decision.code.value, - track_index, - j, - decision.speed_ms, - f"{previous_speed:.1f}" if previous_speed is not None else "-", + log.cells[j] = self._cell_record(by_label[j], lt.uid, "RESUMED") + log.latent.append(self._latent_record(lt, "RESUMED")) + tracks.append(self._moved(by_label[j], lt.uid, p, scan_id, time, time_s)) + f = (last.time_s - lt.time_s) / (time_s - lt.time_s) + mid = ( + lt.centre[0] + f * (by_label[j].centre[0] - lt.centre[0]), + lt.centre[1] + f * (by_label[j].centre[1] - lt.centre[1]), + ) + segments.append((lt.uid, mid, by_label[j].centre)) + + for j in sorted(c.label for c in cells if c.label not in uids): + uids[j] = self._new_uid(scan_id, j) + if j in out.split: + parent = self._tracks[out.split[j]] + events.append( + event_row( + time, + "SPLIT", + EndPoint(parent.uid, parent.label), + EndPoint(uids[j], j), + source_scan=prev_scan, + ) + ) + log.cells[j] = self._cell_record(by_label[j], uids[j], "SPLIT_CHILD") + else: + events.append(event_row(time, "INITIATION", None, EndPoint(uids[j], j))) + log.cells[j] = self._cell_record(by_label[j], uids[j], "INITIATION") + tracks.append(self._new_track(by_label[j], uids[j], scan_id, time, time_s)) + + new_latent: list[LatentTrack] = [] + for i in (i for i in range(len(live)) if i not in out.continuing): + track, fp = self._tracks[i], live[i] + survivor = EndPoint(uids[out.merged[i]], out.merged[i]) if i in out.merged else None + into = survivor.uid if survivor else None + if survivor: + log.counts["merge"] += 1 + events.append( + event_row( + time, + "MERGE", + EndPoint(track.uid, track.label), + survivor, + source_scan=prev_scan, + ) + ) + if cfg.latent_scans <= 1: + log.counts["termination"] += 1 + events.append( + event_row( + time, + "TERMINATION", + EndPoint(track.uid, track.label), + survivor, + source_scan=prev_scan, + ) ) continue - - hull_xy = hull_centroid_m[i] - displacement = ( - math.hypot(curr_cx - hull_xy[0], curr_cy - hull_xy[1]) if hull_xy else 0.0 + lt = LatentTrack( + uid=track.uid, + label=track.label, + scan_id=track.scan_id, + scan_time=track.scan_time, + time_s=track.time_s, + centre=track.centre, + step=track.step, + speed=track.speed, + score=track.score, + footprint=fp.mask, + predicted=fp.predicted, + age=1, + origin=ORIGIN_MERGED if into else ORIGIN_TERMINATED, + merged_into_uid=into, ) - length_m = length_scale( - self.length_scale_name, - hull_area_px[i], - curr_cells[j]["area_px"], - pixel_area_m2, - self.geometry_length_scale_km, + new_latent.append(lt) + log.counts["latent_created"] += 1 + log.latent.append(self._latent_record(lt, "CREATED")) + events.append( + event_row( + time, "LATENT", EndPoint(track.uid, track.label), None, source_scan=prev_scan + ) ) - cost = pair_cost(result.opc, result.ocp, displacement, length_m) - heading_change_deg = None - if state is not None and state.has_velocity: - cand_heading = math.atan2(curr_cy - prev_cy, curr_cx - prev_cx) # B5 - heading_change_deg = heading_change_degrees(state.heading, cand_heading) - if self.heading_penalty_weight > 0.0: - cost += self.heading_penalty_weight * heading_change_radians( - state.heading, cand_heading + + for lt in self._latent: + if lt.uid in resumed_uids: + continue + if lt.age >= cfg.latent_scans: + log.counts["termination"] += 1 + log.latent.append(self._latent_record(lt, "EXPIRED")) + target = EndPoint(lt.merged_into_uid, None) if lt.merged_into_uid else None + events.append( + event_row( + time, + "TERMINATION", + EndPoint(lt.uid, lt.label), + target, + source_scan=(lt.scan_id, lt.scan_time), ) - self._log.costed( - i, - j, - displacement_m=displacement, - length_scale_m=length_m, - heading_change_deg=heading_change_deg, - cost=cost, - ) - edges[(i, j)] = _EdgeCost(result.opc, result.ocp, displacement, cost) - return edges + ) + else: + log.latent.append(self._latent_record(lt, "CARRIED")) + new_latent.append(lt) + segments.append(carried[lt.uid]) + + segments += [(lt.uid, lt.centre, lt.predicted) for lt in new_latent if lt.age == 1] + self._record_interval(tuple(segments), last.time_s, time_s) + self._latent = new_latent + merged_uids = {self._tracks[i].uid: uids[j] for i, j in out.merged.items()} + split_parents = {j: self._tracks[i].uid for j, i in out.split.items()} + self._finish_lineage(log, uids, merged_uids, split_parents) + return uids, events, tracks + + def _record_interval(self, segments: tuple[Segment, ...], prev_s: float, time_s: float) -> None: + """Keep the last N−1 intervals' motion for the crossing test of resumed paths.""" + keep = max(self.cfg.latent_scans - 1, 0) + intervals = [*self._intervals, _Interval(prev_s, time_s, segments)] + self._intervals = intervals[-keep:] if keep else [] - def _track_frame_pair( - self, - prev_node_ids: list[int], - curr_time, - ds_curr: xr.Dataset, - curr_cells: list[dict], - dt_s: float, - ) -> list[dict]: - # Without a registration frame we cannot match — reset (terminate + reinit). - projections_missing = ( - "cell_projections" not in ds_curr.data_vars - or ds_curr["cell_projections"].values.shape[0] < 1 - ) - if projections_missing: - logger.warning("No cell_projections — resetting %d tracks", len(prev_node_ids)) - self._log.reset_code = "NO_PROJECTIONS" - return self._reset_tracks(prev_node_ids, curr_time, curr_cells) + @staticmethod + def _finish_lineage( + log: ScanLog, uids: dict[int, str], merged: dict[str, str], split: dict[int, str] + ) -> None: + """Fill the final uids, add each orphan's INITIATION hypothesis, mark what was chosen. - # Registration hull: the previous labels advected by the full flow step - # (fraction 1.0 of the scan gap). The minute frames are registered to - # whole minutes short of the current scan, so they under-advect. - proj_labels = np.asarray(ds_curr["cell_projections"].values[0]) + ``merged``: merge source uid → survivor uid; ``split``: child label → parent uid. + """ + fates = {label: rec.fate for label, rec in log.cells.items()} + rows = [] + for r in log.lineage: + if r.side == "curr": + chosen = { + "RESUME": fates[r.cell_label] == "RESUMED" + and uids[r.cell_label] == r.partner_uid, + "SPLIT": split.get(r.cell_label) == r.partner_uid, + }[r.hypothesis] + rows.append(replace(r, cell_uid=uids[r.cell_label], chosen=chosen)) + elif r.hypothesis == "MERGE": + into = uids[cast(int, r.partner_label)] + rows.append(replace(r, partner_uid=into, chosen=merged.get(r.cell_uid) == into)) + else: + continuing = r.cell_uid in {rec.cell_uid for rec in log.cells.values()} + rows.append(replace(r, chosen=r.cell_uid not in merged and not continuing)) + for j in sorted(j for j, f in fates.items() if f != "CONTINUE"): + rows.append( + TrackingLineage( + "curr", + uids[j], + j, + "INITIATION", + None, + None, + None, + None, + None, + True, + fates[j] == "INITIATION", + None, + ) + ) + log.lineage[:] = rows - matched_prev: dict[int, int] = {} # prev_idx → new curr node_id - matched_curr: dict[int, int] = {} # curr_idx → new curr node_id + # ── records ──────────────────────────────────────────────────────────── - # 1. Candidate generation + hard gate → validated, costed edges. - edge_costs = self._build_edges(prev_node_ids, curr_cells, proj_labels, ds_curr, dt_s) + def _new_uid(self, scan_id: str, label: int) -> str: + return _cell_uid_from_signature(track_signature_v2(scan_id, label), self.cfg.uid_width) - # 2. Deterministic propagation first; Hungarian only for ambiguous components. - events = self._resolve_matches( - edge_costs, prev_node_ids, curr_cells, curr_time, dt_s, matched_prev, matched_curr + @staticmethod + def _new_track(cell: Cell, uid: str, scan_id: str, time, time_s: float) -> Track: + return Track( + uid, cell.label, scan_id, time, time_s, cell.score, cell.centre, cell.geom, None, None ) - dissipated = [prev_node_ids[i] for i in range(len(prev_node_ids)) if i not in matched_prev] - born = [j for j in range(len(curr_cells)) if j not in matched_curr] - - # 3. Split / merge on the leftovers (behaviour preserved). - split_born, split_events = self._detect_splits( - prev_node_ids, curr_cells, proj_labels, matched_prev, born, curr_time - ) - merged, merge_events = self._detect_merges( - dissipated, curr_cells, proj_labels, matched_curr, curr_time - ) - events += split_events + merge_events - - # 4. Births and terminations for everything still unmatched. - events += self._emit_births(curr_cells, born, split_born, curr_time) - events += self._emit_terminations(dissipated, merged, curr_time) - - logger.info( - "Frame pair: prev=%d → continue=%d merged=%d dissipated=%d | born=%d split=%d", - len(prev_node_ids), - len(matched_prev), - len(merged), - len(dissipated) - len(merged), - len(born) - len(split_born), - len(split_born), + @staticmethod + def _moved(cell: Cell, uid: str, p: Pair, scan_id, time, time_s) -> Track: + """The track after its link: last step per interval and speed over the link.""" + disp = p.displacement + step = (disp[0] / p.fp.steps, disp[1] / p.fp.steps) + return Track( + uid, + cell.label, + scan_id, + time, + time_s, + cell.score, + cell.centre, + cell.geom, + step, + p.speed, ) - self._decisions = self._log.freeze() - return events - - # ------------------------------------------------------------------ - # Matching, split/merge, births/terminations - # ------------------------------------------------------------------ - - def _resolve_matches( - self, - edge_costs: dict[tuple[int, int], _EdgeCost], - prev_node_ids: list[int], - curr_cells: list[dict], - curr_time, - dt_s: float, - matched_prev: dict[int, int], - matched_curr: dict[int, int], - ) -> list[dict]: - """Constraint propagation (PROPAGATED), then Hungarian per ambiguous component.""" - - def record(i: int, c: int, method: MatchMethod) -> dict: - return self._record_continue( - i, - c, - prev_node_ids, - curr_cells, - curr_time, - dt_s, - edge_costs[(i, c)], - method, - matched_prev, - matched_curr, - ) - - forced, remaining = ConstraintPropagator.resolve(list(edge_costs.keys())) - events = [] - for i, c in forced: - self._log.component([i], [c]) - self._log.matched(i, c, MatchMethod.PROPAGATED.value) - events.append(record(i, c, MatchMethod.PROPAGATED)) - - component_costs = {edge: edge_costs[edge].cost for edge in remaining} - for prevs, currs in AssignmentGraph(remaining).components(): - self._log.component(prevs, currs) - for i, c in HungarianMatcher.match(prevs, currs, component_costs): - self._log.matched(i, c, MatchMethod.HUNGARIAN.value) - events.append(record(i, c, MatchMethod.HUNGARIAN)) - return events @staticmethod - def _overlap_areas( - proj_labels: np.ndarray, cell_id: int, mask: np.ndarray - ) -> tuple[int, int, int]: - """Intersection and both areas (px) for a projected hull against a cell mask.""" - proj_mask = proj_labels == cell_id - return ( - int(np.sum(mask & proj_mask)), - int(np.sum(proj_mask)), - int(np.sum(mask)), + def _diagnostics(p: Pair, method: str) -> LinkDiagnostics: + return LinkDiagnostics( + o_c=p.overlap.o_c, + o_h=p.overlap.o_h, + residual_m=p.residual_m, + speed_ms=p.speed, + heading_change_deg=p.turn_deg, + area_ratio=np.count_nonzero(p.cell.mask) / max(1, int(np.count_nonzero(p.fp.mask))), + cost=p.cost, + method=method, ) @staticmethod - def _hull_overlap_fraction(proj_labels: np.ndarray, cell_id: int, mask: np.ndarray) -> float: - """Fraction of a previous cell's projected hull covered by ``mask``. - - The MERGE criterion: the hull belongs to the *dissipating* cell, so this - asks how completely that vanishing cell went into ``mask``. It reaches - 1.0 when a small cell is wholly absorbed by a larger one. - """ - inter, hull_px, _ = CellTracker._overlap_areas(proj_labels, cell_id, mask) - return inter / hull_px if hull_px else 0.0 + def _pair_record(p: Pair) -> dict: + o = p.overlap + return { + "prev_cell_label": p.fp.label, + "source": p.fp.source, + "steps": p.fp.steps, + "growth_radius_km": p.fp.radius_km, + "footprint_area_px": int(np.count_nonzero(p.fp.mask)), + "footprint_grown_px": o.footprint_grown_px, + "cell_grown_px": o.cell_grown_px, + "intersection_px": o.intersection_px, + "o_c": o.o_c, + "o_h": o.o_h, + "u": o.u, + "predicted_x": p.fp.predicted[0], + "predicted_y": p.fp.predicted[1], + "curr_centre_x": p.cell.centre[0], + "curr_centre_y": p.cell.centre[1], + "d": p.d, + "speed_ms": p.speed, + "prev_speed_ms": p.fp.speed, + "speed_limit_ms": p.speed_gate.limit_ms, + "heading_change_deg": p.turn_deg, + "h": p.h, + "cost": p.cost, + "gate": p.gate, + "outcome": "REJECTED" if p.gate != GATE_PASS else "LOST", + } @staticmethod - def _cell_overlap_fraction(proj_labels: np.ndarray, cell_id: int, mask: np.ndarray) -> float: - """Fraction of ``mask`` explained by a previous cell's projected hull (Opc). - - The SPLIT criterion, and the mirror of the merge one: here the hull - belongs to the *continuing parent* and ``mask`` is a born fragment, so - normalising by the hull would ask the fragment to account for the whole - parent — impossible for a split child by construction (its ceiling is - child area / parent hull area, observed at 0.72 on KHTX 2025-06-17 with - a median parent/child area ratio of 3.3, so no split could ever pass a - 0.8 threshold). Normalising by the born cell's area instead asks the - question that matters: is this new cell explained by the parent? It is - the same quantity as ``minimum_candidate_overlap`` in the main gate. - """ - inter, _, mask_px = CellTracker._overlap_areas(proj_labels, cell_id, mask) - return inter / mask_px if mask_px else 0.0 - - def _new_track_node(self, cell: dict, curr_time) -> int: - """Allocate a new track index + identity and add the observation node.""" - new_index = self.graph.get_new_track_index() - cell_uid, track_signature = self._new_cell_identity(cell) - self._cell_identity[new_index] = (cell_uid, track_signature) - return self._add_cell_node(curr_time, cell, new_index, cell_uid, track_signature) - - def _detect_splits( - self, - prev_node_ids: list[int], - curr_cells: list[dict], - proj_labels: np.ndarray, - matched_prev: dict[int, int], - born: list[int], - curr_time, - ) -> tuple[set[int], list[dict]]: - """A born cell explained by a continuing parent's projected hull is a SPLIT child. - - The test is ``Opc`` — intersection over the *born* cell's area — not the - hull-normalised fraction the merge test uses. See - ``_cell_overlap_fraction`` for why the hull denominator makes a split - structurally undetectable. - """ - split_born: set[int] = set() - events: list[dict] = [] - for b_idx in born: - b_mask = curr_cells[b_idx]["mask"] - best_parent, best_overlap = None, 0.0 - for prev_idx, curr_node in matched_prev.items(): - cell_id = self.graph.get_node_attr(prev_node_ids[prev_idx], "cell_id") - inter, hull_px, tested_px = self._overlap_areas(proj_labels, cell_id, b_mask) - overlap = inter / tested_px if tested_px else 0.0 - self._log.split_merge_test( - TrackingSplitMergeTest( - "SPLIT", - str(self.graph.get_node_attr(curr_node, "cell_uid")), - int(curr_cells[b_idx]["cell_id"]), - float(overlap), - self.split_overlap, - overlap >= self.split_overlap, - intersection_px=inter, - hull_px=hull_px, - tested_px=tested_px, - hull_fraction=(inter / hull_px if hull_px else 0.0), - cell_fraction=float(overlap), - ) - ) - if overlap >= self.split_overlap and overlap > best_overlap: - best_parent, best_overlap = curr_node, overlap - if best_parent is not None: - child_node = self._new_track_node(curr_cells[b_idx], curr_time) - self.graph.add_edge(best_parent, child_node, edge_type="SPLIT", cost=0.0) - split_born.add(b_idx) - self._log.born(b_idx, str(self.graph.get_node_attr(child_node, "cell_uid"))) - self._log.split_child(b_idx) - events.append( - event_split(self.graph, self._cell_identity, curr_time, best_parent, child_node) - ) - return split_born, events + def _cell_record(c: Cell, uid: str, fate: str) -> TrackingCell: + return TrackingCell( + c.label, + uid, + c.area_km2, + c.mass, + c.mean_excess, + c.centre[0], + c.centre[1], + c.score, + c.core_area_km2, + fate, + ) - def _detect_merges( - self, - dissipated: list[int], - curr_cells: list[dict], - proj_labels: np.ndarray, - matched_curr: dict[int, int], - curr_time, - ) -> tuple[dict[int, int], list[dict]]: - """A dissipated cell whose projected hull overlaps a continuing cell is a MERGE.""" - merged: dict[int, int] = {} - events: list[dict] = [] - for d_node in dissipated: - cell_id = self.graph.get_node_attr(d_node, "cell_id") - best_target, best_overlap = None, 0.0 - for c_idx, curr_node in matched_curr.items(): - c_mask = curr_cells[c_idx]["mask"] - inter, hull_px, tested_px = self._overlap_areas(proj_labels, cell_id, c_mask) - overlap = inter / hull_px if hull_px else 0.0 - self._log.split_merge_test( - TrackingSplitMergeTest( - "MERGE", - str(self.graph.get_node_attr(curr_node, "cell_uid")), - int(cell_id), - float(overlap), - self.merge_overlap, - overlap >= self.merge_overlap, - intersection_px=inter, - hull_px=hull_px, - tested_px=tested_px, - hull_fraction=float(overlap), - cell_fraction=(inter / tested_px if tested_px else 0.0), - ) - ) - if overlap >= self.merge_overlap and overlap > best_overlap: - best_target, best_overlap = curr_node, overlap - if best_target is not None: - self.graph.add_edge(d_node, best_target, edge_type="MERGE", cost=0.0) - merged[d_node] = best_target - self._log.merge_source(str(self.graph.get_node_attr(d_node, "cell_uid"))) - events.append( - event_merge(self.graph, self._cell_identity, curr_time, d_node, best_target) - ) - return merged, events + @staticmethod + def _latent_record(lt: LatentTrack, status: str) -> TrackingLatent: + return TrackingLatent( + lt.uid, + lt.label, + lt.scan_id, + lt.age, + lt.origin, + lt.merged_into_uid, + status, + int(np.count_nonzero(lt.footprint)), + lt.predicted[0], + lt.predicted[1], + ) - def _emit_births( - self, curr_cells: list[dict], born: list[int], split_born: set[int], curr_time - ) -> list[dict]: - """Initiate fresh tracks for born cells that were not claimed as split children.""" - events: list[dict] = [] - for b_idx in born: - if b_idx in split_born: - continue - node_id = self._new_track_node(curr_cells[b_idx], curr_time) - events.append(event_initiation(self.graph, self._cell_identity, curr_time, node_id)) - self._log.born(b_idx, str(self.graph.get_node_attr(node_id, "cell_uid"))) - return events - - def _emit_terminations( - self, dissipated: list[int], merged: dict[int, int], curr_time - ) -> list[dict]: - """Terminate every unmatched previous node (target = merge sink when merged).""" - return [ - event_termination( - self.graph, self._cell_identity, curr_time, d_node, merged.get(d_node) - ) - for d_node in dissipated + @staticmethod + def _tracked_cells(time, cells: list[Cell], uids: dict[int, str]) -> pd.DataFrame: + rows = [ + { + "time": pd.Timestamp(time).to_datetime64(), + "cell_label": c.label, + "cell_uid": uids[c.label], + "area": c.area_km2, + "centroid_x": c.centre[0], + "centroid_y": c.centre[1], + "mean_reflectivity": c.mean_field, + "max_reflectivity": c.max_field, + "core_area": c.core_area_km2, + } + for c in cells ] - - # ------------------------------------------------------------------ - # Scan-local builders (no per-track analytics) - # ------------------------------------------------------------------ - - def _build_tracked_cells_current(self, time, node_ids: list[int]) -> pd.DataFrame: - rows: list[dict] = [] - for node_id in node_ids: - node = self.graph.graph.nodes[node_id] - time_val = normalize_time_scalar(node["time"]) - time_val = pd.Timestamp(time_val).to_datetime64() - cell_uid = str(node["cell_uid"]) - rows.append( - { - "time": time_val, - "cell_label": int(node["cell_id"]), - "cell_uid": cell_uid, - "area": float(node["area"]), - "centroid_x": float(node["centroid_x"]), - "centroid_y": float(node["centroid_y"]), - "mean_reflectivity": float(node["mean_reflectivity"]), - "max_reflectivity": float(node["max_reflectivity"]), - "core_area": float(node["core_area"]), - } - ) df = pd.DataFrame(rows) if not df.empty: - df["time"] = pd.to_datetime(df["time"]) df = df.sort_values(["cell_uid", "cell_label"]).reset_index(drop=True) return df diff --git a/src/adapt/modules/tracking/motion.py b/src/adapt/modules/tracking/motion.py deleted file mode 100644 index 8032b3b..0000000 --- a/src/adapt/modules/tracking/motion.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright © 2026, UChicago Argonne, LLC -# See LICENSE for terms and disclaimer. - -"""Physical motion checks for candidate matching. - -``MotionValidator`` applies hard kinematic limits that reject impossible -candidate pairs *before* matching; the heading helpers quantify direction change -for the soft crossing-prevention penalty. Pure geometry — no graph, no I/O. -""" - -import math -from dataclasses import dataclass - -from adapt.modules.tracking.models import TrackingError - -__all__ = [ - "MotionDecision", - "MotionValidator", - "heading_change_degrees", - "heading_change_radians", -] - - -def heading_change_radians(prev_heading: float, curr_heading: float) -> float: - """Smallest absolute heading change (radians, in [0, π]) between two directions.""" - delta = curr_heading - prev_heading - return abs(math.atan2(math.sin(delta), math.cos(delta))) - - -def heading_change_degrees(prev_heading: float, curr_heading: float) -> float: - """Smallest absolute heading change (degrees) between two directions (radians).""" - return math.degrees(heading_change_radians(prev_heading, curr_heading)) - - -@dataclass(frozen=True) -class MotionDecision: - """Outcome of a physical-motion check for one candidate pair.""" - - ok: bool - speed_ms: float - code: TrackingError | None = None - cap_ms: float | None = None # the acceleration cap applied, None when no prior speed - - -class MotionValidator: - """Reject candidate pairs that violate hard kinematic limits. - - A pair is rejected when its implied speed exceeds ``max_speed_ms`` (absolute - cap) or ``max(max_speed_multiplier * previous_speed, acceleration_floor_ms)`` - (acceleration cap). The floor keeps a slow prior step — centroid jitter of a - px or two — from vetoing ordinary storm motion. Rejected pairs never reach - overlap or Hungarian matching. - """ - - def __init__( - self, max_speed_ms: float, max_speed_multiplier: float, acceleration_floor_ms: float - ): - self.max_speed_ms = max_speed_ms - self.max_speed_multiplier = max_speed_multiplier - self.acceleration_floor_ms = acceleration_floor_ms - - @staticmethod - def speed_ms(prev_x: float, prev_y: float, curr_x: float, curr_y: float, dt_s: float) -> float: - if dt_s <= 0: - raise ValueError("dt_s must be positive for a speed estimate") - return math.hypot(curr_x - prev_x, curr_y - prev_y) / dt_s - - def check( - self, - prev_x: float, - prev_y: float, - curr_x: float, - curr_y: float, - dt_s: float, - previous_speed: float | None = None, - ) -> MotionDecision: - speed = self.speed_ms(prev_x, prev_y, curr_x, curr_y, dt_s) - if speed > self.max_speed_ms: - return MotionDecision(False, speed, TrackingError.VELOCITY_EXCEEDED) - cap = None - if previous_speed is not None and previous_speed > 0.0: - cap = max(self.max_speed_multiplier * previous_speed, self.acceleration_floor_ms) - if speed > cap: - return MotionDecision(False, speed, TrackingError.ACCELERATION_EXCEEDED, cap) - return MotionDecision(True, speed, None, cap) diff --git a/src/adapt/modules/tracking/observations.py b/src/adapt/modules/tracking/observations.py new file mode 100644 index 0000000..df33a5e --- /dev/null +++ b/src/adapt/modules/tracking/observations.py @@ -0,0 +1,318 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""What the tracker observes and predicts, and the evidence of each pair. + +Cells of the current scan, tracks of the previous one, their footprints at the +current scan (live: the advected cell; latent: carried by the flow), and every +candidate pair's overlap, residual, speed, heading and cost with the first gate +it failed. Positions are metres ``(x, y)``; shapes are grid-index masks. +""" + +import math +from dataclasses import dataclass + +import numpy as np +import pandas as pd +import xarray as xr + +from adapt.contracts import stat_column +from adapt.modules.tracking.config import TrackingConfig +from adapt.modules.tracking.latent import LatentTrack +from adapt.modules.tracking.lineage import identity_score +from adapt.modules.tracking.matching.gates import SpeedGate, heading_term, speed_gate +from adapt.modules.tracking.matching.geometry import ( + PairOverlap, + cell_intensity, + core_centre, + footprint_pairs, + growth_radius_km, + mahalanobis, + second_moment, +) + +__all__ = [ + "GATE_COST", + "GATE_OVERLAP", + "GATE_PASS", + "Cell", + "Footprint", + "Grid", + "Pair", + "Track", + "evaluate_pairs", + "extract_cells", + "latent_footprint", + "live_footprint", +] + +GATE_PASS = "PASS" +GATE_OVERLAP = "OVERLAP" +GATE_COST = "COST" + +Point = tuple[float, float] + + +@dataclass(frozen=True) +class Grid: + """Regular grid: origin and spacing (metres) of columns (x) and rows (y).""" + + x0: float + y0: float + sx: float + sy: float + + @classmethod + def of(cls, ds: xr.Dataset) -> "Grid": + x = np.asarray(ds.x.values, dtype=float) + y = np.asarray(ds.y.values, dtype=float) + return cls(float(x[0]), float(y[0]), float(x[1] - x[0]), float(y[1] - y[0])) + + @property + def spacing_m(self) -> float: + """One grid length σ_x: the position error the gates allow for.""" + return min(abs(self.sx), abs(self.sy)) + + @property + def pixel_km2(self) -> float: + return abs(self.sx * self.sy) / 1e6 + + def to_m(self, row: float, col: float) -> Point: + return self.x0 + col * self.sx, self.y0 + row * self.sy + + def delta_px(self, start: Point, end: Point) -> np.ndarray: + """``end − start`` in (row, col) pixels, the frame of a shape's moment.""" + return np.array([(end[1] - start[1]) / self.sy, (end[0] - start[0]) / self.sx]) + + +def _geometric_centre(mask: np.ndarray, grid: Grid) -> Point: + rows, cols = np.nonzero(mask) + return grid.to_m(float(rows.mean()), float(cols.mean())) + + +@dataclass(frozen=True, eq=False) +class Cell: + """A current cell and the quantities the decisions use.""" + + label: int + mask: np.ndarray + area_km2: float + mass: float + mean_excess: float + score: float + centre: Point # e²-weighted + geom: Point # geometric + core_area_km2: float + mean_field: float + max_field: float + + +def extract_cells( + ds: xr.Dataset, stats: pd.DataFrame, cfg: TrackingConfig, grid: Grid +) -> list[Cell]: + """Cells of one scan, in label order, with the configured field's stats.""" + labels = ds[cfg.labels_var].values + field = ds[cfg.field_var].values + mean_col = stat_column(cfg.field_var, "mean") + max_col = stat_column(cfg.field_var, "max") + if not stats.empty and (mean_col not in stats.columns or max_col not in stats.columns): + raise ValueError( + f"Tracking: cell_stats is missing {mean_col!r}/{max_col!r} for the configured " + f"tracking field {cfg.field_var!r}. The analyzer whitelist " + "(analyzer.radar_variables) must include it." + ) + by_label = {int(r["cell_label"]): r for _, r in stats.iterrows()} + cells = [] + for label in np.unique(labels[labels > 0]): + if int(label) not in by_label: + raise ValueError(f"Tracking: cell {int(label)} is in the labels but not in cell_stats") + mask = labels == label + area = float(np.count_nonzero(mask)) * grid.pixel_km2 + mass, mean_excess = cell_intensity(field, mask, cfg.cell_threshold, cfg.polarity) + row, col = core_centre(field, mask, cfg.cell_threshold, cfg.polarity) + core = ( + cfg.polarity + * (np.nan_to_num(field[mask], nan=cfg.core_field_threshold) - cfg.core_field_threshold) + > 0 + ) + cells.append( + Cell( + label=int(label), + mask=mask, + area_km2=area, + mass=mass, + mean_excess=mean_excess, + score=identity_score(area, mean_excess, cfg.identity_intensity_weight), + centre=grid.to_m(row, col), + geom=_geometric_centre(mask, grid), + core_area_km2=float(np.count_nonzero(core)) * grid.pixel_km2, + mean_field=float(by_label[int(label)][mean_col]), + max_field=float(by_label[int(label)][max_col]), + ) + ) + return cells + + +@dataclass(frozen=True) +class Track: + """A live track as last observed (the previous scan).""" + + uid: str + label: int + scan_id: str + scan_time: object + time_s: float + score: float + centre: Point + geom: Point + step: Point | None # last displacement per scan interval + speed: float | None # last speed + + +@dataclass(frozen=True, eq=False) +class Footprint: + """Where a track is expected at the current scan, and its motion so far. + + ``origin`` is the last observed centre, ``predicted`` the expected one; + ``sigma`` is the footprint's second moment (px²); ``steps``/``dt_s`` span + the link from the last observation. + """ + + uid: str + label: int + source: str # LIVE | LATENT + steps: int + dt_s: float + mask: np.ndarray + radius_km: float + sigma: np.ndarray | None + predicted: Point + origin: Point + step: Point | None + speed: float | None + + +def _footprint(mask, grid, cfg, **kw) -> Footprint: + area = float(np.count_nonzero(mask)) * grid.pixel_km2 + return Footprint( + mask=mask, + radius_km=growth_radius_km(area, cfg.growth_max_km, cfg.growth_size_km), + sigma=second_moment(mask) if mask.any() else None, + **kw, + ) + + +def live_footprint( + track: Track, proj_labels: np.ndarray, grid: Grid, dt_s: float, cfg: TrackingConfig +) -> Footprint: + """The previous cell advected by the flow; its centre moves as the shape did.""" + mask = proj_labels == track.label + predicted = track.centre + if mask.any(): + gx, gy = _geometric_centre(mask, grid) + predicted = (track.centre[0] + gx - track.geom[0], track.centre[1] + gy - track.geom[1]) + return _footprint( + mask, + grid, + cfg, + uid=track.uid, + label=track.label, + source="LIVE", + steps=1, + dt_s=dt_s, + predicted=predicted, + origin=track.centre, + step=track.step, + speed=track.speed, + ) + + +def latent_footprint( + latent: LatentTrack, time_s: float, grid: Grid, cfg: TrackingConfig +) -> Footprint: + return _footprint( + latent.footprint, + grid, + cfg, + uid=latent.uid, + label=latent.label, + source="LATENT", + steps=latent.age, + dt_s=time_s - latent.time_s, + predicted=latent.predicted, + origin=latent.centre, + step=latent.step, + speed=latent.speed, + ) + + +@dataclass(frozen=True) +class Pair: + """Evidence for one footprint–cell pair and the first gate it failed.""" + + fp: Footprint + cell: Cell + overlap: PairOverlap + residual_m: float + d: float + speed: float + speed_gate: SpeedGate + h: float + turn_deg: float | None + cost: float + gate: str + + @property + def displacement(self) -> Point: + return (self.cell.centre[0] - self.fp.origin[0], self.cell.centre[1] - self.fp.origin[1]) + + +def evaluate_pairs( + fp: Footprint, cells: dict[int, Cell], labels: np.ndarray, grid: Grid, cfg: TrackingConfig +) -> list[Pair]: + """Every cell whose grown shape meets the grown footprint, with its evidence. + + Gates in order: OVERLAP (u > u_max), SPEED / SPEED_CHANGE, COST (c > c_max). + """ + if fp.sigma is None: # the footprint left the domain: nothing to compare + return [] + pairs = [] + radius_px = fp.radius_km * 1000.0 / grid.spacing_m + for overlap in footprint_pairs(fp.mask, labels, radius_px): + cell = cells[overlap.label] + residual = grid.delta_px(fp.predicted, cell.centre) + d = mahalanobis(residual, fp.sigma) + disp = (cell.centre[0] - fp.origin[0], cell.centre[1] - fp.origin[1]) + speed = math.hypot(*disp) / fp.dt_s + gate = speed_gate( + speed, fp.speed, fp.dt_s, cfg.max_speed_ms, cfg.max_acceleration_ms2, grid.spacing_m + ) + per_step = (disp[0] / fp.steps, disp[1] / fp.steps) + h, turn = heading_term(fp.step, per_step, cfg.heading_weight, 2.0 * grid.spacing_m) + cost = overlap.u + cfg.residual_weight * d + h + if overlap.u > cfg.max_overlap_mismatch: + code = GATE_OVERLAP + elif gate.code is not None: + code = gate.code + elif cost > cfg.max_link_cost: + code = GATE_COST + else: + code = GATE_PASS + pairs.append( + Pair( + fp=fp, + cell=cell, + overlap=overlap, + residual_m=math.hypot( + cell.centre[0] - fp.predicted[0], cell.centre[1] - fp.predicted[1] + ), + d=d, + speed=speed, + speed_gate=gate, + h=h, + turn_deg=turn, + cost=cost, + gate=code, + ) + ) + return pairs diff --git a/src/adapt/persistence/decision_store.py b/src/adapt/persistence/decision_store.py index c42b77e..0bdd214 100644 --- a/src/adapt/persistence/decision_store.py +++ b/src/adapt/persistence/decision_store.py @@ -19,6 +19,9 @@ __all__ = ["DecisionStore"] +# Tables whose rows have no natural key: keyed by their order within the scan. +_SEQUENCED = frozenset({"segmentation_seeds", "tracking_lineage"}) + class DecisionStore(SqliteStore): """One-scan writer for ``decisions.db``.""" @@ -47,10 +50,12 @@ def write( match decisions: case TrackingDecisions(): tables = ( - ("tracking_frames", (decisions.frame,)), - ("tracking_candidates", decisions.candidates), - ("tracking_unmatched", decisions.unmatched), - ("tracking_split_merge_tests", decisions.split_merge_tests), + ("tracking_scans", (decisions.scan,)), + ("tracking_cells", decisions.cells), + ("tracking_pairs", decisions.pairs), + ("tracking_lineage", decisions.lineage), + ("tracking_identity", decisions.identity), + ("tracking_latent", decisions.latent), ) case SegmentationDecisions(): tables = ( @@ -64,7 +69,7 @@ def write( conn = self._get_connection() for table, records in tables: rows = [{**stamp, **asdict(record)} for record in records] - if table == "segmentation_seeds": + if table in _SEQUENCED: for seq, row in enumerate(rows): row["seq"] = seq self._replace(conn, table, rows) diff --git a/src/adapt/persistence/track_store.py b/src/adapt/persistence/track_store.py index a2b6307..381dab2 100644 --- a/src/adapt/persistence/track_store.py +++ b/src/adapt/persistence/track_store.py @@ -204,6 +204,18 @@ def _target_uid(ev: pd.Series): return ev.get("target_cell_uid") +def _event_source_scan(ev: pd.Series, prev: tuple[str, str] | None) -> tuple[str, str] | None: + """(scan_id, scan_time) the source cell was last observed in. + + The previous scan, unless the event names its own — a resumed or an + expiring latent track refers back to the scan it vanished after. + """ + scan_id = ev.get("source_scan_id") + if scan_id is None or pd.isna(scan_id) or scan_id == (prev[0] if prev else None): + return prev + return str(scan_id), _to_iso(pd.Timestamp(ev["source_scan_time"]).to_pydatetime()) + + class TrackStore: """Read/write track persistence tables in a collection's products.db. @@ -331,7 +343,7 @@ def write_scan( # 4. Retroactively update previous scan's cells_by_scan flags prev = self._prev_scan(conn, run_id, scan_iso) if prev and not cell_events_df.empty: - self._update_retroactive_flags(conn, run_id, prev[0], cell_events_df) + self._update_retroactive_flags(conn, run_id, prev, cell_events_df) # 5. Insert cell_events if not cell_events_df.empty: @@ -677,34 +689,26 @@ def _update_retroactive_flags( self, conn: sqlite3.Connection, run_id: str, - prev_scan_id: str, + prev: tuple[str, str], cell_events_df: pd.DataFrame, ) -> None: - """Set is_split_source, is_merge_source, is_terminated_after on prev scan rows.""" - term_tracks, split_tracks, merge_tracks = set(), set(), set() + """Set is_split_source, is_merge_source, is_terminated_after on the source scan's rows.""" + flags = { + "TERMINATION": "is_terminated_after_here", + "SPLIT": "is_split_source_here", + "MERGE": "is_merge_source_here", + } for _, ev in cell_events_df.iterrows(): - etype = ev["event_type"] - stid = _source_uid(ev) - if pd.isna(stid): + flag, stid = flags.get(str(ev["event_type"])), _source_uid(ev) + if flag is None or pd.isna(stid): continue - if etype == "TERMINATION": - term_tracks.add(str(stid)) - elif etype == "SPLIT": - split_tracks.add(str(stid)) - elif etype == "MERGE": - merge_tracks.add(str(stid)) - - def _update(flag: str, cell_uids: set) -> None: - for tid in cell_uids: - conn.execute( - f"UPDATE cells_by_scan SET {flag}=1 " - "WHERE run_id=? AND scan_id=? AND cell_uid=?", - (run_id, prev_scan_id, tid), - ) - - _update("is_terminated_after_here", term_tracks) - _update("is_split_source_here", split_tracks) - _update("is_merge_source_here", merge_tracks) + source = _event_source_scan(ev, prev) + if source is None: + continue + conn.execute( + f"UPDATE cells_by_scan SET {flag}=1 WHERE run_id=? AND scan_id=? AND cell_uid=?", + (run_id, source[0], str(stid)), + ) def _insert_cell_events( self, @@ -715,7 +719,6 @@ def _insert_cell_events( prev: tuple[str, str] | None, cell_events_df: pd.DataFrame, ) -> None: - source_scan_id, source_iso = prev if prev else (None, None) cols = [ "run_id", "source_scan_id", @@ -746,17 +749,8 @@ def _insert_cell_events( placeholders = ", ".join("?" * len(all_cols)) sql = f"INSERT INTO cell_events ({', '.join(all_cols)}) VALUES ({placeholders})" - def _src_id(etype: str) -> str | None: - return None if etype == "INITIATION" else source_scan_id - - def _tgt_id(etype: str) -> str | None: - return None if etype == "TERMINATION" else scan_id - - def _src_time(etype: str) -> str | None: - return None if etype == "INITIATION" else source_iso - - def _tgt_time(etype: str) -> str | None: - return None if etype == "TERMINATION" else target_iso + # No target scan for an event that ends or suspends a track. + untargeted = {"TERMINATION", "LATENT"} def _num(ev: pd.Series, col: str) -> float | None: val = ev.get(col) @@ -768,13 +762,14 @@ def _num(ev: pd.Series, col: str) -> float | None: source_uid = _source_uid(ev) target_uid = _target_uid(ev) method = ev.get("match_method") + source = None if etype == "INITIATION" else _event_source_scan(ev, prev) rows.append( ( run_id, - _src_id(etype), - _tgt_id(etype), - _src_time(etype), - _tgt_time(etype), + source[0] if source else None, + None if etype in untargeted else scan_id, + source[1] if source else None, + None if etype in untargeted else target_iso, etype, source_uid if pd.notna(source_uid) else None, target_uid if pd.notna(target_uid) else None, @@ -842,6 +837,8 @@ def _upsert_cell_tracks( split_children[str(ttid)] = (str(stid), gid) elif etype == "TERMINATION" and pd.notna(stid): terminated[str(stid)] = gid + if pd.notna(ttid): # a merged-away track ending: target is the survivor + merged_into[str(stid)] = (str(ttid), gid) elif etype == "MERGE" and pd.notna(stid) and pd.notna(ttid): merged_into[str(stid)] = (str(ttid), gid) diff --git a/src/adapt/runtime/run_reporter.py b/src/adapt/runtime/run_reporter.py index 2c63a57..343cb1f 100644 --- a/src/adapt/runtime/run_reporter.py +++ b/src/adapt/runtime/run_reporter.py @@ -103,7 +103,8 @@ def format_header(start: RunStart) -> str: ), "tracking": lambda c: ( f"gap {c.global_.max_scan_gap_minutes:g}min " - f"· overlap≥{c.tracker.minimum_candidate_overlap:g} · vmax {c.tracker.max_speed_ms:g}m/s" + f"· u≤{c.tracker.max_overlap_mismatch:g} · c≤{c.tracker.max_link_cost:g} " + f"· vmax {c.tracker.max_speed_ms:g}m/s" ), } diff --git a/tests/configuration/test_full_config_roundtrip.py b/tests/configuration/test_full_config_roundtrip.py index 9cdc96a..2f3995a 100644 --- a/tests/configuration/test_full_config_roundtrip.py +++ b/tests/configuration/test_full_config_roundtrip.py @@ -47,9 +47,7 @@ def test_defaults_resolve_unchanged(self): internal.segmenter.threshold_params.threshold == defaults.segmenter.threshold_params.threshold ) - assert ( - internal.tracker.minimum_candidate_overlap == defaults.tracker.minimum_candidate_overlap - ) + assert internal.tracker.max_overlap_mismatch == defaults.tracker.max_overlap_mismatch assert ( internal.analyzer.adjacency_min_touching_boundary_pixels == defaults.analyzer.adjacency_min_touching_boundary_pixels diff --git a/tests/configuration/test_scan_gap_and_thresholds.py b/tests/configuration/test_scan_gap_and_thresholds.py index 343faa8..846b1a1 100644 --- a/tests/configuration/test_scan_gap_and_thresholds.py +++ b/tests/configuration/test_scan_gap_and_thresholds.py @@ -29,7 +29,7 @@ def test_every_stage_reads_the_same_gap(make_config): def test_split_and_merge_thresholds_are_independent(make_config): param = ParamConfig() - assert param.tracker.split_overlap_threshold == 0.6 + assert param.tracker.split_overlap_threshold == 0.65 assert param.tracker.merge_overlap_threshold == 0.7 cfg = make_config(tracker={"split_overlap_threshold": 0.5}) diff --git a/tests/configuration/test_tracker_config.py b/tests/configuration/test_tracker_config.py new file mode 100644 index 0000000..1450451 --- /dev/null +++ b/tests/configuration/test_tracker_config.py @@ -0,0 +1,61 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Tracker v2 parameters: defaults, overrides reach the tracker, legacy keys fail loudly.""" + +import pytest + +from adapt.configuration.schemas.errors import ConfigError +from adapt.configuration.schemas.param import ParamConfig +from adapt.execution.nodes.tracking import TrackingModule + +pytestmark = pytest.mark.unit + + +def test_physical_limits_and_rule_defaults(): + t = ParamConfig().tracker + assert (t.max_speed_ms, t.max_acceleration_ms2) == (40.0, 0.04) + assert (t.growth_max_km, t.growth_size_km) == (2.0, 12.0) + assert (t.residual_weight, t.heading_weight) == (0.25, 1.0) + assert (t.latent_scans, t.identity_intensity_weight, t.identity_score_margin) == (2, 1.0, 0.33) + assert (t.cell_threshold, t.polarity) == (30.0, 1) + + +def test_rule_thresholds_are_in_their_valid_ranges(): + t = ParamConfig().tracker + assert 0.0 < t.max_overlap_mismatch <= 1.0 + assert t.max_link_cost > 0.0 + assert 0.0 < t.split_overlap_threshold <= 1.0 + assert 0.0 < t.merge_overlap_threshold <= 1.0 + + +def test_override_reaches_the_tracker(make_config): + cfg = TrackingModule.build_config( + make_config(tracker={"max_link_cost": 1.5, "latent_scans": 3, "polarity": -1}) + ) + assert (cfg.max_link_cost, cfg.latent_scans, cfg.polarity) == (1.5, 3, -1) + + +@pytest.mark.parametrize( + ("legacy", "replacement"), + [ + ("minimum_candidate_overlap", "max_overlap_mismatch"), + ("projected_hull_buffer_km", "growth_max_km"), + ("max_speed_multiplier", "max_acceleration_ms2"), + ("heading_change_penalty_weight", "heading_weight"), + ("length_scale", "residual_weight"), + ], +) +def test_legacy_key_fails_naming_its_replacement(make_config, legacy, replacement): + with pytest.raises(ConfigError, match=replacement): + make_config(tracker={legacy: 1.0}) + + +def test_polarity_must_be_a_sign(make_config): + with pytest.raises(ConfigError): + make_config(tracker={"polarity": 2}) + + +def test_negative_latency_is_rejected(make_config): + with pytest.raises(ConfigError): + make_config(tracker={"latent_scans": -1}) diff --git a/tests/configuration/test_user_passthrough.py b/tests/configuration/test_user_passthrough.py index 4b7e3b6..0cdcb13 100644 --- a/tests/configuration/test_user_passthrough.py +++ b/tests/configuration/test_user_passthrough.py @@ -25,8 +25,8 @@ def _resolve(user_dict): class TestPassthroughSections: def test_tracker_override_reaches_internal(self): - internal = _resolve({"radar": "KLOT", "tracker": {"minimum_candidate_overlap": 0.5}}) - assert internal.tracker.minimum_candidate_overlap == 0.5 + internal = _resolve({"radar": "KLOT", "tracker": {"max_link_cost": 1.5}}) + assert internal.tracker.max_link_cost == 1.5 def test_visualization_override_reaches_internal(self): internal = _resolve({"radar": "KLOT", "visualization": {"dpi": 123}}) diff --git a/tests/contracts/test_decisions_contract.py b/tests/contracts/test_decisions_contract.py index a5db663..73cdcaf 100644 --- a/tests/contracts/test_decisions_contract.py +++ b/tests/contracts/test_decisions_contract.py @@ -12,7 +12,7 @@ SegmentationDecisions, SegmentationFrame, TrackingDecisions, - TrackingFrame, + TrackingScan, check_segmentation_decisions, check_tracking_decisions, ) @@ -20,33 +20,33 @@ pytestmark = pytest.mark.unit -def _tracking_frame(**kw) -> TrackingFrame: - fields = dict.fromkeys( - ( - "n_prev", - "n_curr", - "n_pairs", - "n_pass_overlap", - "n_pass_kinematic", - "n_propagated", - "n_hungarian", - "n_split", - "n_merge", - "n_initiation", - "n_termination", - ), - 0, - ) - return TrackingFrame(dt_s=None, reset_code="FIRST_SCAN", **{**fields, **kw}) +def _tracking_scan(**kw) -> TrackingScan: + counts = dict.fromkeys((f.name for f in dataclasses.fields(TrackingScan)), 0) + counts.update(dt_s=None, reset_code="FIRST_SCAN") + return TrackingScan(**{**counts, **kw}) + + +def _bundle(scan: TrackingScan) -> TrackingDecisions: + return TrackingDecisions(scan, (), (), (), (), ()) def test_tracking_bundle_with_no_pairs_is_valid(): - check_tracking_decisions(TrackingDecisions(_tracking_frame(), (), (), ())) + check_tracking_decisions(_bundle(_tracking_scan())) + + +def test_tracking_pair_rows_must_match_pair_count(): + with pytest.raises(ContractViolation): + check_tracking_decisions(_bundle(_tracking_scan(n_pairs=1))) + + +def test_tracking_cell_rows_must_match_current_cell_count(): + with pytest.raises(ContractViolation): + check_tracking_decisions(_bundle(_tracking_scan(n_curr=2))) -def test_tracking_candidate_rows_must_match_generated_pair_count(): +def test_terminations_cannot_exceed_previous_and_latent_tracks(): with pytest.raises(ContractViolation): - check_tracking_decisions(TrackingDecisions(_tracking_frame(n_pairs=1), (), (), ())) + check_tracking_decisions(_bundle(_tracking_scan(n_prev=1, n_latent=1, n_termination=3))) def test_segmentation_component_rows_must_match_component_count(): @@ -56,6 +56,6 @@ def test_segmentation_component_rows_must_match_component_count(): def test_records_are_frozen(): - frame = _tracking_frame() + frame = _tracking_scan() with pytest.raises(dataclasses.FrozenInstanceError): frame.n_prev = 3 # type: ignore[misc] diff --git a/tests/execution/nodes/test_tracking_node.py b/tests/execution/nodes/test_tracking_node.py index 57a42be..80a3d65 100644 --- a/tests/execution/nodes/test_tracking_node.py +++ b/tests/execution/nodes/test_tracking_node.py @@ -50,10 +50,10 @@ def track(self, ds_projected, cell_stats_df, *, scan_id): @staticmethod def decisions(): - from adapt.contracts import TrackingDecisions, TrackingFrame + from adapt.contracts import TrackingDecisions, TrackingScan - frame = TrackingFrame(None, "FIRST_SCAN", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) - return TrackingDecisions(frame, (), (), ()) + scan = TrackingScan(None, "FIRST_SCAN", *([0] * 17)) + return TrackingDecisions(scan, (), (), (), (), ()) def _run_module(module, tracked_per_scan): diff --git a/tests/modules/tracking/test_assignment.py b/tests/modules/tracking/test_assignment.py index 603f7e8..6bb3c1a 100644 --- a/tests/modules/tracking/test_assignment.py +++ b/tests/modules/tracking/test_assignment.py @@ -1,67 +1,77 @@ # Copyright © 2026, UChicago Argonne, LLC # See LICENSE for terms and disclaimer. -"""Unit tests for ConstraintPropagator and AssignmentGraph.""" +"""Joint assignment per group of competing cells: no-link, crossing, margin.""" import pytest -from adapt.modules.tracking.matching.assignment import AssignmentGraph, ConstraintPropagator +from adapt.modules.tracking.matching.assignment import Link, components, solve_component pytestmark = pytest.mark.unit +C_MAX = 1.8 -def test_unique_pair_is_forced(): - forced, remaining = ConstraintPropagator.resolve([(0, 0)]) - assert forced == [(0, 0)] - assert remaining == [] +def _link(i, j, cost, start=(0.0, 0.0), end=(0.0, 0.0)): + return Link(i, j, cost, start, end) -def test_disjoint_unique_pairs_all_forced(): - forced, remaining = ConstraintPropagator.resolve([(0, 0), (1, 1), (2, 2)]) - assert forced == [(0, 0), (1, 1), (2, 2)] - assert remaining == [] +def test_components_group_cells_connected_through_candidates(): + groups = components([(0, 0), (1, 0), (2, 1), (3, 2), (3, 3)]) + assert sorted(groups) == [([0, 1], [0]), ([2], [1]), ([3], [2, 3])] -def test_full_two_by_two_is_ambiguous(): - edges = [(0, 0), (0, 1), (1, 0), (1, 1)] - forced, remaining = ConstraintPropagator.resolve(edges) - assert forced == [] - assert remaining == sorted(edges) +def test_single_candidate_is_accepted_below_the_ceiling(): + solution = solve_component([_link(0, 0, 1.2)], C_MAX) + assert solution.chosen == [(0, 0)] -def test_partial_resolves_unique_leaves_ambiguous_remainder(): - # prev0↔curr0 unique; prev1 has two candidates → ambiguous - forced, remaining = ConstraintPropagator.resolve([(0, 0), (1, 1), (1, 2)]) - assert forced == [(0, 0)] - assert remaining == [(1, 1), (1, 2)] +def test_disjoint_pairs_resolve_independently(): + links = [_link(0, 0, 0.3), _link(1, 1, 0.4)] + chosen = [ + p + for prevs, currs in components([(0, 0), (1, 1)]) + for p in solve_component([lk for lk in links if lk.prev in prevs], C_MAX).chosen + ] + assert sorted(chosen) == [(0, 0), (1, 1)] -def test_cascade_peeling_converges(): - # Removing the unique (0,0) does NOT free curr1 (still deg 2) → stays ambiguous. - forced, remaining = ConstraintPropagator.resolve([(0, 0), (1, 1), (1, 2), (2, 2)]) - assert forced == [(0, 0)] - assert remaining == [(1, 1), (1, 2), (2, 2)] +def test_cheaper_predecessor_wins_a_contested_cell(): + solution = solve_component([_link(0, 0, 0.6), _link(1, 0, 0.2)], C_MAX) + assert solution.chosen == [(1, 0)] -def test_contested_candidate_not_forced(): - # prev0 has a single candidate curr0, but curr0 is also claimed by prev1 → not unique - forced, remaining = ConstraintPropagator.resolve([(0, 0), (1, 0), (1, 1)]) - assert forced == [] - assert remaining == sorted([(0, 0), (1, 0), (1, 1)]) +def test_an_assignment_whose_links_cross_is_not_admissible(): + # Two cells side by side exchanging places is the cheapest pairing but it + # crosses; the parallel pairing is chosen instead. + a0, b0 = (0.0, 0.0), (0.0, 10.0) + x, y = (10.0, 10.0), (10.0, 0.0) + links = [ + _link(0, 0, 0.1, a0, x), + _link(1, 1, 0.1, b0, y), + _link(0, 1, 0.5, a0, y), + _link(1, 0, 0.5, b0, x), + ] + solution = solve_component(links, C_MAX) + assert sorted(solution.chosen) == [(0, 1), (1, 0)] + assert set(solution.crossing_excluded) & {(0, 0), (1, 1)} -def test_components_single_group(): - comp = AssignmentGraph([(0, 0), (0, 1), (1, 0), (1, 1)]).components() - assert len(comp) == 1 - prevs, currs = comp[0] - assert prevs == [0, 1] and currs == [0, 1] +def test_when_every_alternative_crosses_the_costlier_link_is_dropped(): + a0, b0 = (0.0, 0.0), (0.0, 10.0) + links = [_link(0, 0, 0.1, a0, (10.0, 10.0)), _link(1, 1, 0.3, b0, (10.0, 0.0))] + solution = solve_component(links, C_MAX) + assert solution.chosen == [(0, 0)] + assert solution.crossing_excluded == {(1, 1): (0, 0)} -def test_components_disconnected_ambiguous_groups_independent(): - # Two separate 2x2 ambiguous groups with no cross edges → two components. - edges = [(0, 0), (0, 1), (1, 0), (1, 1), (2, 2), (2, 3), (3, 2), (3, 3)] - forced, remaining = ConstraintPropagator.resolve(edges) - assert forced == [] - comps = AssignmentGraph(remaining).components() - comps_sorted = sorted(comps) - assert comps_sorted == [([0, 1], [0, 1]), ([2, 3], [2, 3])] + +def test_margin_is_the_extra_cost_of_the_best_assignment_without_the_link(): + solution = solve_component([_link(0, 0, 0.4)], C_MAX) + assert solution.margins[(0, 0)] == pytest.approx(C_MAX - 0.4) + + +def test_margin_to_a_runner_up(): + solution = solve_component([_link(0, 0, 0.4), _link(1, 0, 0.5)], C_MAX) + # With (0,0): 0.4 + 1.8 (prev 1 unlinked). Without it: 1.8 + 0.5. + assert solution.chosen == [(0, 0)] + assert solution.margins[(0, 0)] == pytest.approx(0.1) diff --git a/tests/modules/tracking/test_candidate_generator.py b/tests/modules/tracking/test_candidate_generator.py deleted file mode 100644 index 2b934cc..0000000 --- a/tests/modules/tracking/test_candidate_generator.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright © 2026, UChicago Argonne, LLC -# See LICENSE for terms and disclaimer. - -"""Unit tests for CandidateGenerator (buffered high-recall candidates).""" - -import numpy as np -import pytest - -from adapt.modules.tracking.matching.candidate import CandidateGenerator - -pytestmark = pytest.mark.unit - - -def _mask(shape, cells): - m = np.zeros(shape, dtype=bool) - for r0, r1, c0, c1 in cells: - m[r0:r1, c0:c1] = True - return m - - -def test_direct_intersection_is_a_candidate_without_buffer(): - hull = _mask((6, 6), [(1, 3, 1, 3)]) - cell = _mask((6, 6), [(1, 3, 1, 3)]) # overlaps hull - gen = CandidateGenerator(buffer_pixels=0) - buffered, pairs = gen.generate([hull], [cell]) - assert pairs == [(0, 0)] - assert np.array_equal(buffered[0], hull) - - -def test_buffer_extends_recall_to_a_near_miss(): - hull = _mask((6, 6), [(2, 4, 2, 4)]) - near = _mask((6, 6), [(2, 4, 4, 5)]) # touches column 4, one pixel outside hull - gen0 = CandidateGenerator(buffer_pixels=0) - _, pairs0 = gen0.generate([hull], [near]) - # column 3 (hull) and column 4 (near) are adjacent but disjoint → no candidate - assert pairs0 == [] - gen1 = CandidateGenerator(buffer_pixels=1) - _, pairs1 = gen1.generate([hull], [near]) - assert pairs1 == [(0, 0)] # dilation makes them intersect - - -def test_high_recall_multiple_candidates_per_hull(): - hull = _mask((6, 8), [(1, 5, 1, 6)]) # broad hull - a = _mask((6, 8), [(1, 3, 1, 3)]) - b = _mask((6, 8), [(3, 5, 4, 6)]) - far = _mask((6, 8), [(0, 1, 7, 8)]) # outside even with 1 px buffer - gen = CandidateGenerator(buffer_pixels=1) - _, pairs = gen.generate([hull], [a, b, far]) - assert (0, 0) in pairs and (0, 1) in pairs - assert (0, 2) not in pairs - - -def test_empty_hull_yields_no_candidates(): - empty = np.zeros((4, 4), dtype=bool) - cell = _mask((4, 4), [(0, 2, 0, 2)]) - gen = CandidateGenerator(buffer_pixels=2) - _, pairs = gen.generate([empty], [cell]) - assert pairs == [] diff --git a/tests/modules/tracking/test_decision_log.py b/tests/modules/tracking/test_decision_log.py index b5c5462..16078df 100644 --- a/tests/modules/tracking/test_decision_log.py +++ b/tests/modules/tracking/test_decision_log.py @@ -1,9 +1,9 @@ # Copyright © 2026, UChicago Argonne, LLC # See LICENSE for terms and disclaimer. -"""The tracker records every candidate it considered and why each cell ended. +"""The tracker records every pair it considered, every lineage hypothesis, and why. -Synthetic 8x8 scans (1000 m pixels) with analytically known gate outcomes. +Synthetic 8x8 scans (1000 m pixels) with analytically known outcomes. """ import numpy as np @@ -21,6 +21,7 @@ T0 = np.datetime64("2024-01-01T12:00:00") T1 = np.datetime64("2024-01-01T12:05:00") +T2 = np.datetime64("2024-01-01T12:10:00") def _cells_scan(time, spans: dict[int, tuple[int, int]], proj_labels=None): @@ -42,113 +43,119 @@ def _cells_scan(time, spans: dict[int, tuple[int, int]], proj_labels=None): return _synthetic_ds(time, labels, proj_labels=proj_labels), _cell_stats(time, rows) -def test_first_scan_is_a_frame_with_no_pairs_and_born_cells(): +def test_first_scan_has_no_pairs_and_every_cell_initiates(): tracker = CellTracker(_make_config()) tracker.track(*_one_cell_scan(T0, 2), scan_id="scan0") d = tracker.decisions() - assert d.frame.reset_code == "FIRST_SCAN" - assert (d.frame.n_prev, d.frame.n_curr, d.frame.n_pairs) == (0, 1, 0) - assert [u.reason for u in d.unmatched] == ["FIRST_SCAN"] + assert d.scan.reset_code == "FIRST_SCAN" + assert (d.scan.n_prev, d.scan.n_curr, d.scan.n_pairs) == (0, 1, 0) + assert [c.fate for c in d.cells] == ["INITIATION"] -def test_perfect_continuation_is_one_propagated_pair(): +def test_perfect_continuation_is_one_assigned_pair_with_its_evidence(): tracker = CellTracker(_make_config()) tracker.track(*_one_cell_scan(T0, 2), scan_id="scan0") tracker.track(*_one_cell_scan(T1, 3), scan_id="scan1") - d = tracker.decisions() - (pair,) = d.candidates - assert pair.outcome == "CONTINUE" - assert pair.match_method == "PROPAGATED" - assert pair.last_stage == "ASSIGNMENT" - assert pair.opc == pair.ocp == 1.0 + (pair,) = tracker.decisions().pairs + assert (pair.outcome, pair.gate, pair.source, pair.steps) == ("CONTINUE", "PASS", "LIVE", 1) + assert pair.u == pytest.approx(0.0) + assert pair.cost == pytest.approx(0.0) + assert pair.speed_ms == pytest.approx(1000.0 / 300.0) assert pair.cost_rank == 1 + assert pair.margin == pytest.approx(_make_config().max_link_cost) + + +def test_overlap_rejection_is_recorded_with_its_numbers(make_tracking_config): + tracker = CellTracker(make_tracking_config(tracker={"max_overlap_mismatch": 0.1})) + tracker.track(*_cells_scan(T0, {1: (0, 4)}), scan_id="scan0") + proj = np.zeros((8, 8), dtype=np.int32) + proj[2:4, 0:4] = 1 + tracker.track(*_cells_scan(T1, {1: (2, 6)}, proj_labels=proj), scan_id="scan1") + (pair,) = tracker.decisions().pairs + assert (pair.gate, pair.outcome) == ("OVERLAP", "REJECTED") + assert pair.u > 0.1 + assert 0.0 < pair.o_c < 1.0 and 0.0 < pair.o_h < 1.0 + + +def test_speed_rejection_records_speed_and_code(): + tracker = CellTracker(_make_config(max_speed_ms=2.0)) + tracker.track(*_one_cell_scan(T0, 2), scan_id="scan0") + tracker.track(*_one_cell_scan(T1, 3), scan_id="scan1") + (pair,) = tracker.decisions().pairs + assert pair.gate == "SPEED" assert pair.speed_ms == pytest.approx(1000.0 / 300.0) - assert d.frame.n_propagated == 1 and d.frame.n_termination == 0 - assert d.unmatched == () -def test_overlap_rejection_is_recorded_with_its_numbers(): - """Hull left at x=2, cell at x=4: the 1 km buffer generates the pair, the - un-buffered hull has zero intersection, so the overlap gate rejects it.""" +def test_contested_cell_records_ranks_margin_and_the_loser(): + # Two previous cells whose footprints both reach one current cell. tracker = CellTracker(_make_config()) - ds0, stats0 = _one_cell_scan(T0, 2) - tracker.track(ds0, stats0, scan_id="scan0") - tracker.track(*_one_cell_scan(T1, 4, proj_labels=ds0["cell_labels"].values), scan_id="scan1") + tracker.track(*_cells_scan(T0, {1: (0, 3), 2: (5, 8)}), scan_id="scan0") + proj = np.zeros((8, 8), dtype=np.int32) + proj[2:4, 1:4] = 1 + proj[2:4, 4:7] = 2 + tracker.track(*_cells_scan(T1, {1: (2, 5)}, proj_labels=proj), scan_id="scan1") d = tracker.decisions() - (pair,) = d.candidates - assert (pair.outcome, pair.last_stage) == ("REJECTED_OVERLAP", "OVERLAP") - assert pair.intersection_px == 0 and pair.opc == 0.0 - assert pair.cost is None - prev, curr = d.unmatched - assert (prev.side, prev.reason, prev.best_candidate_label) == ( - "prev", - "ALL_REJECTED_OVERLAP", - 1, - ) - assert (curr.side, curr.reason) == ("curr", "ALL_REJECTED_OVERLAP") - assert d.frame.n_termination == 1 and d.frame.n_initiation == 1 + outcomes = sorted((p.prev_cell_label, p.outcome) for p in d.pairs) + assert [o for _, o in outcomes].count("CONTINUE") == 1 + assert d.scan.n_components == 1 + winner = next(p for p in d.pairs if p.outcome == "CONTINUE") + assert winner.margin > 0.0 -def test_kinematic_rejection_records_speed_and_code(): - tracker = CellTracker(_make_config(max_speed_ms=5.0)) - tracker.track(*_one_cell_scan(T0, 2), scan_id="scan0") - tracker.track(*_one_cell_scan(T1, 6), scan_id="scan1") # 13.3 m/s > 5 - (pair,) = tracker.decisions().candidates - assert pair.overlap_passed is True - assert (pair.outcome, pair.kinematic_code) == ("REJECTED_KINEMATIC", "VELOCITY_EXCEEDED") - assert pair.speed_ms == pytest.approx(4000.0 / 300.0) - assert tracker.decisions().unmatched[0].reason == "ALL_REJECTED_KINEMATIC" +def test_orphan_lineage_hypotheses_are_recorded_pass_or_fail(): + tracker = CellTracker(_make_config()) + tracker.track(*_cells_scan(T0, {1: (1, 7)}), scan_id="scan0") + proj = np.zeros((8, 8), dtype=np.int32) + proj[2:4, 1:7] = 1 + tracker.track(*_cells_scan(T1, {1: (1, 4), 2: (5, 7)}, proj_labels=proj), scan_id="scan1") + d = tracker.decisions() + orphan = next(c for c in d.cells if c.fate != "CONTINUE") + rows = [r for r in d.lineage if r.cell_label == orphan.cell_label and r.side == "curr"] + kinds = {r.hypothesis: r for r in rows} + assert kinds["SPLIT"].chosen and kinds["SPLIT"].passed + assert kinds["SPLIT"].fraction == pytest.approx(1.0) + assert not kinds["INITIATION"].chosen -def test_ambiguous_component_records_hungarian_ranks_and_the_loser(): - """A (cols 1-3) and B (cols 4-6) vs X (cols 2-4) and Y (cols 5-7): X overlaps - both, so nothing is forced and one 2x2 component goes to Hungarian.""" +def test_unlinked_track_records_merge_and_latent_hypotheses(): tracker = CellTracker(_make_config()) - ds0, stats0 = _cells_scan(T0, {1: (1, 4), 2: (4, 7)}) - tracker.track(ds0, stats0, scan_id="scan0") - tracker.track( - *_cells_scan(T1, {1: (2, 5), 2: (5, 8)}, proj_labels=ds0["cell_labels"].values), - scan_id="scan1", - ) + tracker.track(*_cells_scan(T0, {1: (1, 4), 2: (5, 7)}), scan_id="scan0") + proj = np.zeros((8, 8), dtype=np.int32) + proj[2:4, 1:4] = 1 + proj[2:4, 5:7] = 2 + tracker.track(*_cells_scan(T1, {1: (1, 7)}, proj_labels=proj), scan_id="scan1") d = tracker.decisions() - by_pair = {(c.prev_cell_label, c.curr_cell_label): c for c in d.candidates} - assert {(1, 1), (2, 1), (2, 2)} <= set(by_pair) - assert by_pair[(1, 1)].outcome == "CONTINUE" and by_pair[(2, 2)].outcome == "CONTINUE" - assert by_pair[(2, 1)].outcome == "LOST_ASSIGNMENT" - # A's buffered hull may graze Y: that pair exists only as an overlap rejection. - others = set(by_pair) - {(1, 1), (2, 1), (2, 2)} - assert all(by_pair[p].outcome == "REJECTED_OVERLAP" for p in others) - assert by_pair[(2, 1)].cost_rank == 2 and by_pair[(2, 2)].cost_rank == 1 - assert {c.match_method for c in d.candidates if c.outcome == "CONTINUE"} == {"HUNGARIAN"} - assert by_pair[(1, 1)].component_n_prev == 2 and by_pair[(1, 1)].component_n_curr == 2 - assert d.frame.n_hungarian == 2 - - -def test_split_test_is_recorded_pass_or_fail(): - """A 10-px parent splits into a 4-px and a 6-px piece. The parent continues - into the larger piece (cheaper); the born 4-px piece covers 0.4 of the - parent's hull, exactly split_overlap, and is recorded as a passing test.""" + prev_rows = {r.hypothesis: r for r in d.lineage if r.side == "prev"} + assert prev_rows["MERGE"].chosen and prev_rows["MERGE"].fraction == pytest.approx(1.0) + assert not prev_rows["LATENT"].chosen + (latent,) = d.latent + assert (latent.status, latent.origin, latent.age) == ("CREATED", "MERGED", 1) + assert d.scan.n_merge == 1 + + +def test_latent_resumption_is_recorded(): tracker = CellTracker(_make_config()) - ds0, stats0 = _cells_scan(T0, {1: (1, 6)}) - tracker.track(ds0, stats0, scan_id="scan0") + tracker.track(*_one_cell_scan(T0, 2), scan_id="scan0") + empty_ds, empty_stats = _cells_scan( + T1, {}, proj_labels=_one_cell_scan(T0, 2)[0]["cell_labels"].values + ) + tracker.track(empty_ds, empty_stats, scan_id="scan1") tracker.track( - *_cells_scan(T1, {1: (1, 3), 2: (3, 6)}, proj_labels=ds0["cell_labels"].values), - scan_id="scan1", + *_cells_scan(T2, {1: (2, 4)}, proj_labels=np.zeros((8, 8), np.int32)), scan_id="scan2" ) d = tracker.decisions() - tests = {(t.kind, t.tested_cell_label): t for t in d.split_merge_tests} - born = [u for u in d.unmatched if u.side == "curr"] - assert d.frame.n_split == 1 and born[0].reason == "SPLIT_CHILD" - assert tests[("SPLIT", born[0].cell_label)].passed is True - assert tests[("SPLIT", born[0].cell_label)].threshold == 0.4 + (pair,) = d.pairs + assert (pair.source, pair.steps, pair.outcome) == ("LATENT", 2, "RESUMED") + assert [c.fate for c in d.cells] == ["RESUMED"] + assert [lt.status for lt in d.latent] == ["RESUMED"] + assert any(r.hypothesis == "RESUME" and r.chosen for r in d.lineage) -def test_gap_reset_marks_every_cell_reset(): - tracker = CellTracker(_make_config(max_tracking_gap_minutes=10.0)) +def test_gap_reset_is_recorded(): + tracker = CellTracker(_make_config(max_tracking_gap_minutes=3.0)) tracker.track(*_one_cell_scan(T0, 2), scan_id="scan0") - tracker.track(*_one_cell_scan(np.datetime64("2024-01-01T12:20:00"), 2), scan_id="scan1") + tracker.track(*_one_cell_scan(T1, 2), scan_id="scan1") d = tracker.decisions() - assert d.frame.reset_code == "TRACK_GAP_EXCEEDED" - assert d.candidates == () - assert {u.reason for u in d.unmatched} == {"RESET"} - assert d.frame.n_termination == 1 and d.frame.n_initiation == 1 + assert d.scan.reset_code == "TRACK_GAP_EXCEEDED" + assert d.scan.n_termination == 1 + assert [c.fate for c in d.cells] == ["INITIATION"] diff --git a/tests/modules/tracking/test_gates.py b/tests/modules/tracking/test_gates.py new file mode 100644 index 0000000..35c6a29 --- /dev/null +++ b/tests/modules/tracking/test_gates.py @@ -0,0 +1,149 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Hard physical gates, the heading term, and the crossing test of tracker v2.""" + +import math + +import pytest + +from adapt.modules.tracking.matching.gates import ( + GATE_SPEED, + GATE_SPEED_CHANGE, + drop_crossing_links, + heading_term, + segments_cross, + speed_change_limit, + speed_gate, +) + +pytestmark = pytest.mark.unit + +A_MAX = 0.04 +SIGMA = 1000.0 + + +# ── speed and additive speed change ───────────────────────────────────────── + + +def test_speed_change_limit_is_acceleration_plus_centroid_noise(): + # a·Δt + 2√2·σ/Δt at Δt = 277 s: 11.08 + 10.21 ≈ 21.3 m/s (spec value) + limit = speed_change_limit(0.0, 277.0, A_MAX, SIGMA) + assert limit == pytest.approx(0.04 * 277 + 2 * math.sqrt(2) * 1000 / 277) + assert limit == pytest.approx(21.3, abs=0.05) + + +def test_speed_change_limit_is_additive_to_the_previous_speed(): + base = speed_change_limit(0.0, 277.0, A_MAX, SIGMA) + assert speed_change_limit(15.0, 277.0, A_MAX, SIGMA) == pytest.approx(15.0 + base) + + +def test_speed_change_noise_term_dominates_short_intervals(): + short = speed_change_limit(0.0, 60.0, A_MAX, SIGMA) + assert short > speed_change_limit(0.0, 277.0, A_MAX, SIGMA) + + +def test_speed_above_the_cap_is_rejected(): + gate = speed_gate(45.0, v_prev=None, dt_s=277.0, v_max=40.0, a_max=A_MAX, sigma_x_m=SIGMA) + assert gate.code == GATE_SPEED + + +def test_first_step_track_is_limited_only_by_the_cap(): + gate = speed_gate(39.0, v_prev=None, dt_s=277.0, v_max=40.0, a_max=A_MAX, sigma_x_m=SIGMA) + assert gate.code is None + assert gate.limit_ms is None + + +def test_speed_jump_beyond_the_additive_limit_is_rejected(): + # 5 + 21.3 = 26.3 m/s allowed; 30 m/s is not. + gate = speed_gate(30.0, v_prev=5.0, dt_s=277.0, v_max=40.0, a_max=A_MAX, sigma_x_m=SIGMA) + assert gate.code == GATE_SPEED_CHANGE + assert gate.limit_ms == pytest.approx(5.0 + 21.3, abs=0.05) + + +def test_a_slow_cell_may_start_moving(): + # The former ratio rule (3 × 0.5 m/s) rejected this; the additive one does not. + gate = speed_gate(12.0, v_prev=0.5, dt_s=277.0, v_max=40.0, a_max=A_MAX, sigma_x_m=SIGMA) + assert gate.code is None + + +# ── heading term ──────────────────────────────────────────────────────────── + + +def test_heading_term_is_zero_for_straight_continuation(): + h, turn = heading_term((5000.0, 0.0), (6000.0, 0.0), weight=1.0, min_step_m=2000.0) + assert h == pytest.approx(0.0) + assert turn == pytest.approx(0.0) + + +def test_heading_term_is_the_weight_for_a_full_reversal(): + h, turn = heading_term((5000.0, 0.0), (-5000.0, 0.0), weight=1.0, min_step_m=2000.0) + assert h == pytest.approx(1.0) + assert turn == pytest.approx(180.0) + + +def test_heading_term_is_half_the_weight_for_a_right_angle(): + h, _ = heading_term((5000.0, 0.0), (0.0, 5000.0), weight=0.8, min_step_m=2000.0) + assert h == pytest.approx(0.4) + + +def test_heading_is_not_applied_below_two_grid_lengths(): + # A 1.5 km step is dominated by position noise: no term, no angle. + assert heading_term((5000.0, 0.0), (-1500.0, 0.0), 1.0, 2000.0) == (0.0, None) + assert heading_term((1500.0, 0.0), (-5000.0, 0.0), 1.0, 2000.0) == (0.0, None) + + +def test_heading_needs_a_previous_step(): + assert heading_term(None, (5000.0, 0.0), 1.0, 2000.0) == (0.0, None) + + +# ── crossing ──────────────────────────────────────────────────────────────── + + +def test_exchanging_places_is_a_crossing(): + assert segments_cross((0, 0), (10, 10), (0, 10), (10, 0)) + + +def test_parallel_motion_is_not_a_crossing(): + assert not segments_cross((0, 0), (10, 0), (0, 5), (10, 5)) + + +def test_collinear_segments_are_not_a_crossing(): + assert not segments_cross((0, 0), (10, 0), (5, 0), (15, 0)) + + +def test_shared_endpoints_are_not_a_crossing(): + # A split (shared start) and a merge (shared end). + assert not segments_cross((0, 0), (10, 10), (0, 0), (10, -10)) + assert not segments_cross((0, 0), (10, 0), (5, 5), (10, 0)) + + +def test_segments_that_stop_short_do_not_cross(): + assert not segments_cross((0, 0), (4, 4), (0, 10), (10, 0)) + + +def test_the_costlier_of_two_crossing_links_is_dropped(): + links = { + "a": ((0.0, 0.0), (10.0, 10.0), 0.3), + "b": ((0.0, 10.0), (10.0, 0.0), 0.5), + "c": ((20.0, 0.0), (30.0, 0.0), 0.9), + } + kept, dropped = drop_crossing_links(links) + assert kept == ["a", "c"] + assert dropped == {"b": "a"} + + +def test_crossing_resolution_is_independent_of_input_order(): + links = { + "b": ((0.0, 10.0), (10.0, 0.0), 0.5), + "a": ((0.0, 0.0), (10.0, 10.0), 0.3), + } + kept, dropped = drop_crossing_links(links) + assert kept == ["a"] and dropped == {"b": "a"} + + +def test_crossing_accepts_numpy_coordinates(): + import numpy as np + + a0, a1, b0, b1 = (np.float64(v) for v in (0.0, 10.0, 10.0, 0.0)) + assert segments_cross((a0, a0), (a1, a1), (a0, b0), (a1, b1)) diff --git a/tests/modules/tracking/test_geometry.py b/tests/modules/tracking/test_geometry.py index f88ab5b..596e22a 100644 --- a/tests/modules/tracking/test_geometry.py +++ b/tests/modules/tracking/test_geometry.py @@ -1,7 +1,10 @@ # Copyright © 2026, UChicago Argonne, LLC # See LICENSE for terms and disclaimer. -"""Unit tests for geometry-first matching primitives (analytic values, no fixtures).""" +"""Geometry of tracker v2: growth, grown overlaps, core centre, shape residual. + +Synthetic masks with analytically known answers; no fixtures. +""" import math @@ -9,120 +12,189 @@ import pytest from adapt.modules.tracking.matching.geometry import ( - bidirectional_overlap, - buffer_pixels_from_km, - dilate_hull, - geometric_mismatch, - length_scale, - mask_centroid, - mass_weighted_centroid, - pair_cost, + cell_intensity, + core_centre, + footprint_pairs, + grow, + growth_radius_km, + mahalanobis, + second_moment, ) pytestmark = pytest.mark.unit -def test_bidirectional_overlap_asymmetric(): - # hull = 4 px column, cell = 8 px, intersection = 2 px - hull = np.zeros((4, 4), dtype=bool) - hull[0:4, 0] = True # 4 px - cell = np.zeros((4, 4), dtype=bool) - cell[0:2, 0:4] = True # 8 px - # intersection = column 0 rows 0..1 = 2 px - opc, ocp = bidirectional_overlap(hull, cell) - assert opc == pytest.approx(2.0 / 8.0) # intersection / candidate - assert ocp == pytest.approx(2.0 / 4.0) # intersection / hull +def _disk(shape, centre, radius): + rr, cc = np.mgrid[: shape[0], : shape[1]] + return (rr - centre[0]) ** 2 + (cc - centre[1]) ** 2 <= radius**2 + + +# ── growth radius r = r_max (1 − √A / ℓ0)₊ ────────────────────────────────── + + +def test_growth_radius_is_r_max_for_a_vanishing_cell(): + assert growth_radius_km(0.0, r_max_km=2.0, l0_km=12.0) == pytest.approx(2.0) + +def test_growth_radius_falls_linearly_in_size(): + # √A = 6 km is half of ℓ0 → half of r_max + assert growth_radius_km(36.0, r_max_km=2.0, l0_km=12.0) == pytest.approx(1.0) -def test_bidirectional_overlap_disjoint_is_zero(): - hull = np.zeros((4, 4), dtype=bool) - hull[0, 0] = True - cell = np.zeros((4, 4), dtype=bool) - cell[3, 3] = True - assert bidirectional_overlap(hull, cell) == (0.0, 0.0) +def test_growth_radius_is_zero_at_and_above_l0(): + assert growth_radius_km(144.0, r_max_km=2.0, l0_km=12.0) == 0.0 + assert growth_radius_km(400.0, r_max_km=2.0, l0_km=12.0) == 0.0 -def test_dilate_hull_single_pixel_by_one(): - m = np.zeros((5, 5), dtype=bool) - m[2, 2] = True - out = dilate_hull(m, 1) - assert out[1:4, 1:4].all() # 3x3 block around centre - assert int(out.sum()) == 9 +# ── growth from the boundary (distance transform) ─────────────────────────── -def test_dilate_hull_zero_buffer_is_identity(): - m = np.zeros((3, 3), dtype=bool) - m[1, 1] = True - assert np.array_equal(dilate_hull(m, 0), m) +def test_grow_by_zero_is_identity(): + mask = _disk((21, 21), (10, 10), 3) + assert np.array_equal(grow(mask, 0.0), mask) -def test_buffer_pixels_from_km_rounds_up(): - # 1 km buffer at 500 m/pixel = 2 px; at 800 m/pixel = ceil(1.25) = 2 - assert buffer_pixels_from_km(1.0, 500.0) == 2 - assert buffer_pixels_from_km(1.0, 800.0) == 2 - assert buffer_pixels_from_km(2.0, 1000.0) == 2 +def test_grow_is_measured_from_the_boundary_not_the_centre(): + # A 1-px-wide line grown by 2 px is 5 px wide along its whole length, + # exactly like the rim of a round cell: growth depends on distance to the shape. + line = np.zeros((11, 31), dtype=bool) + line[5, 5:26] = True + grown = grow(line, 2.0) + assert grown[:, 15].sum() == 5 + assert grown[3:8, 5:26].all() -def test_length_scale_equiv_diameter(): - # hull 100 px, pixel_area 1e6 m² (1 km²) → area 1e8 m² → diameter 2*sqrt(1e8/pi) - L = length_scale("hull_equiv_diameter", 100.0, 50.0, 1e6, fixed_km=5.0) - assert pytest.approx(2.0 * math.sqrt(1e8 / math.pi)) == L +def test_grow_is_continuous_in_the_radius(): + mask = np.zeros((21, 21), dtype=bool) + mask[10, 10] = True + assert grow(mask, 1.0).sum() == 5 # von Neumann neighbours at distance 1 + assert grow(mask, 1.5).sum() == 9 # diagonals at √2 join -def test_length_scale_sum_radii(): - L = length_scale("sum_radii", 100.0, 25.0, 1e6, fixed_km=5.0) - expected = math.sqrt(1e8 / math.pi) + math.sqrt(25e6 / math.pi) - assert pytest.approx(expected) == L +# ── pair search on grown shapes ───────────────────────────────────────────── -def test_length_scale_fixed_km_ignores_area(): - L = length_scale("fixed_km", 100.0, 25.0, 1e6, fixed_km=5.0) - assert pytest.approx(5000.0) == L +def test_identical_shapes_have_full_overlap_both_ways(): + labels = np.zeros((30, 30), dtype=np.int32) + labels[_disk((30, 30), (15, 15), 4)] = 7 + pairs = footprint_pairs(labels == 7, labels, radius_px=1.0) + (pair,) = pairs + assert pair.label == 7 + assert pair.o_c == pytest.approx(1.0) + assert pair.o_h == pytest.approx(1.0) + assert pair.u == pytest.approx(0.0) -def test_length_scale_unknown_raises(): - with pytest.raises(ValueError, match="Unknown length_scale"): - length_scale("nope", 1.0, 1.0, 1.0, fixed_km=1.0) +def test_overlap_fractions_are_normalised_by_each_grown_shape(): + # Footprint: a 4x4 block. Cell: a 4x8 block containing it. Radius 0. + foot = np.zeros((20, 20), dtype=bool) + foot[5:9, 5:9] = True + labels = np.zeros((20, 20), dtype=np.int32) + labels[5:9, 5:13] = 3 + (pair,) = footprint_pairs(foot, labels, radius_px=0.0) + assert pair.intersection_px == 16 + assert pair.o_c == pytest.approx(16 / 32) # share of the cell + assert pair.o_h == pytest.approx(16 / 16) # share of the footprint + assert pair.u == pytest.approx(1 - math.sqrt(0.5)) -def test_geometric_mismatch_perfect_and_none(): - assert geometric_mismatch(1.0, 1.0) == pytest.approx(0.0) - assert geometric_mismatch(0.0, 0.0) == pytest.approx(1.0) - # sqrt spreads: Opc=Ocp=0.25 → g=0.25 → m=0.75 - assert geometric_mismatch(0.25, 0.25) == pytest.approx(0.75) +def test_growth_turns_a_near_miss_into_a_candidate(): + # Two single pixels 3 px apart: disjoint when ungrown, overlapping when + # both are grown by 2 px (the grown disks meet in the middle). + foot = np.zeros((10, 20), dtype=bool) + foot[5, 5] = True + labels = np.zeros((10, 20), dtype=np.int32) + labels[5, 8] = 1 + assert footprint_pairs(foot, labels, radius_px=0.0) == [] + (pair,) = footprint_pairs(foot, labels, radius_px=2.0) + assert pair.label == 1 and pair.intersection_px > 0 -def test_pair_cost_combines_overlap_and_normalised_displacement(): - # m = 1 - sqrt(0.25)*sqrt(0.25) = 0.75; d/L = 1000/2000 = 0.5 → cost 1.25 - assert pair_cost(0.25, 0.25, 1000.0, 2000.0) == pytest.approx(1.25) +def test_cell_is_grown_whole_even_when_it_extends_beyond_the_footprint(): + # A long cell touching a small footprint: |M⁺| must be the full grown cell. + foot = np.zeros((20, 60), dtype=bool) + foot[10, 5] = True + labels = np.zeros((20, 60), dtype=np.int32) + labels[10, 6:56] = 2 + (pair,) = footprint_pairs(foot, labels, radius_px=1.0) + assert pair.cell_grown_px == grow(labels == 2, 1.0).sum() + assert pair.footprint_grown_px == 5 -def test_pair_cost_degenerate_length_falls_back_to_mismatch(): - assert pair_cost(0.25, 0.25, 1000.0, 0.0) == pytest.approx(0.75) +def test_empty_footprint_has_no_pairs(): + labels = np.ones((5, 5), dtype=np.int32) + assert footprint_pairs(np.zeros((5, 5), dtype=bool), labels, radius_px=2.0) == [] -def test_mask_centroid_geometric(): - m = np.zeros((3, 3), dtype=bool) - m[0, 0] = True - m[0, 2] = True - row, col = mask_centroid(m) - assert (row, col) == pytest.approx((0.0, 1.0)) +# ── intensity excess, mass, core centre ───────────────────────────────────── -def test_mass_weighted_centroid_shifts_toward_high_field(): - field = np.zeros((1, 3), dtype=float) - field[0, 0] = 1.0 - field[0, 2] = 9.0 - mask = np.ones((1, 3), dtype=bool) - # weights = field - min = [1,0,9]-0 → but min within mask is 0 (col1) → weights [1,0,9] - row, col = mass_weighted_centroid(field, mask) - # weighted col = (1*0 + 0*1 + 9*2)/10 = 1.8 → pulled toward col 2 - assert col == pytest.approx(1.8) + +def test_excess_is_positive_above_threshold_for_reflectivity(): + field = np.array([[30.0, 40.0, 50.0]]) + mask = np.ones_like(field, dtype=bool) + mass, mean_excess = cell_intensity(field, mask, threshold=30.0, polarity=1) + assert mass == pytest.approx(0 + 10 + 20) + assert mean_excess == pytest.approx(10.0) + + +def test_excess_is_positive_below_threshold_for_brightness_temperature(): + field = np.array([[235.0, 225.0, 215.0]]) + mask = np.ones_like(field, dtype=bool) + mass, _ = cell_intensity(field, mask, threshold=235.0, polarity=-1) + assert mass == pytest.approx(0 + 10 + 20) + + +def test_core_centre_is_pulled_to_the_stronger_core_by_the_square(): + # Two pixels, excess 1 and 3: e-weighting gives 0.75, e²-weighting 0.9. + field = np.array([[31.0, 33.0]]) + mask = np.ones_like(field, dtype=bool) + row, col = core_centre(field, mask, threshold=30.0, polarity=1) assert row == pytest.approx(0.0) + assert col == pytest.approx(9 / 10) + + +def test_core_centre_polarity_mirrors_for_brightness_temperature(): + field = np.array([[229.0, 227.0]]) # excess 1 and 3 below 230 K + mask = np.ones_like(field, dtype=bool) + _, col = core_centre(field, mask, threshold=230.0, polarity=-1) + assert col == pytest.approx(9 / 10) + + +def test_core_centre_of_a_cell_without_excess_is_its_geometric_centre(): + field = np.full((1, 4), 20.0) + mask = np.ones_like(field, dtype=bool) + assert core_centre(field, mask, threshold=30.0, polarity=1) == pytest.approx((0.0, 1.5)) + + +def test_missing_field_values_carry_no_excess(): + field = np.array([[np.nan, 40.0]]) + mask = np.ones_like(field, dtype=bool) + mass, _ = cell_intensity(field, mask, threshold=30.0, polarity=1) + assert mass == pytest.approx(10.0) + + +# ── second moment and shape-aware residual ────────────────────────────────── + + +def test_round_cell_residual_is_four_times_displacement_over_diameter(): + # For a disk of radius R, Σ = R²/4 I, so d = |δ| / (R/2) = 4 |δ| / D. + mask = _disk((201, 201), (100, 100), 40) + sigma = second_moment(mask) + d = mahalanobis(np.array([0.0, 10.0]), sigma) + assert d == pytest.approx(4 * 10 / 80, rel=0.02) + + +def test_residual_along_a_line_costs_less_than_across_it(): + line = np.zeros((41, 81), dtype=bool) + line[18:23, 5:76] = True # 5 px thick, 71 px long, along columns + sigma = second_moment(line) + along = mahalanobis(np.array([0.0, 5.0]), sigma) + across = mahalanobis(np.array([5.0, 0.0]), sigma) + assert along < across / 5 -def test_mass_weighted_centroid_uniform_field_falls_back_to_geometric(): - field = np.full((1, 3), 5.0, dtype=float) - mask = np.ones((1, 3), dtype=bool) - row, col = mass_weighted_centroid(field, mask) - assert (row, col) == pytest.approx((0.0, 1.0)) +def test_single_pixel_moment_is_finite(): + mask = np.zeros((3, 3), dtype=bool) + mask[1, 1] = True + sigma = second_moment(mask) + assert np.all(np.isfinite(np.linalg.inv(sigma))) diff --git a/tests/modules/tracking/test_hungarian_component.py b/tests/modules/tracking/test_hungarian_component.py index 4f73670..59647e2 100644 --- a/tests/modules/tracking/test_hungarian_component.py +++ b/tests/modules/tracking/test_hungarian_component.py @@ -1,7 +1,7 @@ # Copyright © 2026, UChicago Argonne, LLC # See LICENSE for terms and disclaimer. -"""Unit tests for per-component HungarianMatcher.""" +"""Per-component Hungarian assignment with a no-link option at the cost ceiling.""" import pytest @@ -9,28 +9,36 @@ pytestmark = pytest.mark.unit +C_MAX = 1.8 + def test_two_by_two_picks_min_cost_assignment(): - # Component prev {0,1} ↔ curr {0,1}. Cheapest pairing is the anti-diagonal. costs = {(0, 0): 0.9, (0, 1): 0.1, (1, 0): 0.2, (1, 1): 0.8} - matched = HungarianMatcher.match([0, 1], [0, 1], costs) - assert sorted(matched) == [(0, 1), (1, 0)] + assert sorted(HungarianMatcher.match([0, 1], [0, 1], costs, C_MAX)) == [(0, 1), (1, 0)] def test_unequal_component_leaves_surplus_unmatched(): - # Two previous, one current; only the cheaper prev keeps the cell. costs = {(0, 0): 0.7, (1, 0): 0.2} - matched = HungarianMatcher.match([0, 1], [0], costs) - assert matched == [(1, 0)] + assert HungarianMatcher.match([0, 1], [0], costs, C_MAX) == [(1, 0)] -def test_non_edge_assignment_is_dropped(): - # prev1 has no validated edge to curr1; padding must not create a fake match. +def test_non_edge_is_never_assigned(): costs = {(0, 0): 0.3} - matched = HungarianMatcher.match([0, 1], [0, 1], costs) - assert matched == [(0, 0)] + assert HungarianMatcher.match([0, 1], [0, 1], costs, C_MAX) == [(0, 0)] + + +def test_a_link_at_the_ceiling_is_accepted(): + assert HungarianMatcher.match([0], [0], {(0, 0): C_MAX}, C_MAX) == [(0, 0)] + + +def test_no_link_is_chosen_when_every_rearrangement_costs_more(): + # Without the no-link column prev 0 would be forced onto curr 1 to give + # prev 1 its only candidate (1.75 + 0.2 = 1.95); leaving prev 1 unlinked + # costs 0.1 + 1.8 = 1.9. + costs = {(0, 0): 0.1, (1, 0): 0.2, (0, 1): 1.75} + assert HungarianMatcher.match([0, 1], [0, 1], costs, C_MAX) == [(0, 0)] def test_empty_component_returns_nothing(): - assert HungarianMatcher.match([], [0], {}) == [] - assert HungarianMatcher.match([0], [], {}) == [] + assert HungarianMatcher.match([], [0], {}, C_MAX) == [] + assert HungarianMatcher.match([0], [], {}, C_MAX) == [] diff --git a/tests/modules/tracking/test_lineage.py b/tests/modules/tracking/test_lineage.py new file mode 100644 index 0000000..3797b03 --- /dev/null +++ b/tests/modules/tracking/test_lineage.py @@ -0,0 +1,106 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""Lineage partner choice, identity score and identity rule; latent carry.""" + +import math + +import numpy as np +import pytest + +from adapt.modules.tracking.latent import carry_footprint +from adapt.modules.tracking.lineage import ( + RULE_NEAREST, + RULE_SCORE, + IdentityCandidate, + best_partner, + choose_identity, + identity_score, +) + +pytestmark = pytest.mark.unit + + +# ── split parent / merge target ───────────────────────────────────────────── + + +def test_partner_is_the_largest_fraction_above_threshold(): + assert best_partner({3: 0.7, 5: 0.9, 8: 0.66}, threshold=0.65) == 5 + + +def test_no_partner_below_threshold(): + assert best_partner({3: 0.64}, threshold=0.65) is None + + +def test_partner_at_threshold_qualifies(): + assert best_partner({3: 0.65}, threshold=0.65) == 3 + + +def test_partner_tie_goes_to_the_smaller_key(): + assert best_partner({7: 0.8, 2: 0.8}, threshold=0.5) == 2 + + +# ── identity score S = log A + γ log ē ────────────────────────────────────── + + +def test_identity_score_with_gamma_one_ranks_by_intensity_mass(): + # A·ē is the mass: 10 km² at 5 dBZ excess outranks 20 km² at 2 dBZ. + assert identity_score(10.0, 5.0, gamma=1.0) > identity_score(20.0, 2.0, gamma=1.0) + + +def test_identity_score_with_gamma_zero_is_the_area_rule(): + assert identity_score(20.0, 2.0, gamma=0.0) == pytest.approx(math.log(20.0)) + + +def test_cell_without_excess_has_the_lowest_score(): + assert identity_score(50.0, 0.0, gamma=1.0) == -math.inf + + +# ── identity rule ─────────────────────────────────────────────────────────── + + +def test_identity_follows_the_clearly_larger_score(): + winner, rule = choose_identity( + [IdentityCandidate("near", 1.0, 0.1), IdentityCandidate("strong", 2.0, 3.0)], + margin=0.33, + ) + assert (winner, rule) == ("strong", RULE_SCORE) + + +def test_identity_within_noise_goes_to_the_nearer_candidate(): + winner, rule = choose_identity( + [IdentityCandidate("near", 1.9, 0.1), IdentityCandidate("strong", 2.0, 3.0)], + margin=0.33, + ) + assert (winner, rule) == ("near", RULE_NEAREST) + + +def test_identity_between_two_cells_without_excess_goes_to_the_nearer(): + winner, _ = choose_identity( + [IdentityCandidate("a", -math.inf, 2.0), IdentityCandidate("b", -math.inf, 1.0)], + margin=0.33, + ) + assert winner == "b" + + +# ── latent carry by the flow ──────────────────────────────────────────────── + + +def test_latent_footprint_moves_with_the_mean_flow_inside_it(): + mask = np.zeros((10, 10), dtype=bool) + mask[2:4, 2:4] = True + flow_x = np.zeros((10, 10)) + flow_y = np.zeros((10, 10)) + flow_x[2:4, 2:4] = 3.0 # 3 px right + flow_y[2:4, 2:4] = 1.0 # 1 px down + carried, shift = carry_footprint(mask, flow_x, flow_y) + assert shift == (1, 3) + assert carried[3:5, 5:7].all() and carried.sum() == 4 + + +def test_latent_footprint_leaving_the_domain_is_clipped(): + mask = np.zeros((5, 5), dtype=bool) + mask[2, 3:5] = True + flow_x = np.full((5, 5), 1.0) + carried, _ = carry_footprint(mask, flow_x, np.zeros((5, 5))) + assert carried.sum() == 1 and carried[2, 4] diff --git a/tests/modules/tracking/test_motion.py b/tests/modules/tracking/test_motion.py deleted file mode 100644 index b1d7ba3..0000000 --- a/tests/modules/tracking/test_motion.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright © 2026, UChicago Argonne, LLC -# See LICENSE for terms and disclaimer. - -"""Kinematic gate: absolute speed cap and a floored acceleration cap.""" - -import pytest - -from adapt.modules.tracking.models import TrackingError -from adapt.modules.tracking.motion import MotionValidator - -pytestmark = pytest.mark.unit - -DT_S = 270.0 - - -def _check(validator: MotionValidator, speed_ms: float, previous_speed: float | None): - return validator.check(0.0, 0.0, speed_ms * DT_S, 0.0, DT_S, previous_speed) - - -def test_floor_admits_a_step_above_the_relative_cap(): - decision = _check(MotionValidator(40.0, 3.0, 20.0), 13.3, previous_speed=3.8) - assert decision.ok # 13.3 > 3 x 3.8 = 11.4 but <= floor 20 - - -def test_floor_does_not_admit_a_step_above_itself(): - decision = _check(MotionValidator(40.0, 3.0, 20.0), 25.0, previous_speed=3.8) - assert decision.code is TrackingError.ACCELERATION_EXCEEDED - - -def test_relative_cap_still_applies_above_the_floor(): - validator = MotionValidator(40.0, 3.0, 20.0) # previous 10 m/s -> cap 30 - assert _check(validator, 25.0, previous_speed=10.0).ok - assert _check(validator, 35.0, previous_speed=10.0).code is TrackingError.ACCELERATION_EXCEEDED - - -def test_absolute_cap_is_independent_of_the_floor(): - decision = _check(MotionValidator(40.0, 3.0, 100.0), 45.0, previous_speed=None) - assert decision.code is TrackingError.VELOCITY_EXCEEDED diff --git a/tests/modules/tracking/test_split_merge_normalisation.py b/tests/modules/tracking/test_split_merge_normalisation.py deleted file mode 100644 index ad464b0..0000000 --- a/tests/modules/tracking/test_split_merge_normalisation.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright © 2026, UChicago Argonne, LLC -# See LICENSE for terms and disclaimer. - -"""The split and merge tests normalise by different areas. - -Before these were separated, both divided the intersection by the *projected -hull* area. For a merge that is the dissipating cell's own hull, so the fraction -can reach 1.0. For a split the hull belongs to the continuing parent while the -tested mask is a born fragment, so the fraction is bounded above by -child area / parent hull area — a split could not pass a high threshold no -matter how cleanly it split. These tests pin both denominators. -""" - -import numpy as np -import pytest - -from adapt.modules.tracking.module import CellTracker - -pytestmark = pytest.mark.unit - - -@pytest.fixture -def parent_and_child(): - """A 40 px parent hull and a 10 px born fragment fully inside it.""" - proj = np.zeros((10, 10), dtype=np.int32) - proj[0:4, 0:10] = 1 # parent hull, 40 px - child = np.zeros((10, 10), dtype=bool) - child[0:1, 0:10] = True # 10 px, entirely within the hull - return proj, child - - -def test_hull_and_cell_fractions_use_different_denominators(parent_and_child): - proj, child = parent_and_child - inter, hull_px, tested_px = CellTracker._overlap_areas(proj, 1, child) - assert (inter, hull_px, tested_px) == (10, 40, 10) - # merge normalisation: how much of the hull was covered - assert CellTracker._hull_overlap_fraction(proj, 1, child) == pytest.approx(0.25) - # split normalisation: how much of the born cell the parent explains - assert CellTracker._cell_overlap_fraction(proj, 1, child) == pytest.approx(1.0) - - -def test_a_perfectly_explained_child_fails_the_hull_normalisation(parent_and_child): - """The exact failure that made splits undetectable: a child wholly inside - its parent's hull still scores only 0.25 when normalised by the hull.""" - proj, child = parent_and_child - assert CellTracker._hull_overlap_fraction(proj, 1, child) < 0.7 - assert CellTracker._cell_overlap_fraction(proj, 1, child) >= 0.7 - - -def test_a_wholly_absorbed_cell_reaches_one_on_the_merge_normalisation(): - """The merge case is unaffected: a small cell absorbed by a big one - covers all of its own hull.""" - proj = np.zeros((10, 10), dtype=np.int32) - proj[0:2, 0:2] = 1 # dissipating cell's hull, 4 px - survivor = np.zeros((10, 10), dtype=bool) - survivor[0:6, 0:6] = True # the large continuing cell - assert CellTracker._hull_overlap_fraction(proj, 1, survivor) == pytest.approx(1.0) - - -def test_empty_hull_and_empty_mask_are_zero_not_an_error(): - empty = np.zeros((4, 4), dtype=np.int32) - mask = np.ones((4, 4), dtype=bool) - assert CellTracker._hull_overlap_fraction(empty, 1, mask) == 0.0 - assert CellTracker._cell_overlap_fraction(empty, 1, np.zeros((4, 4), dtype=bool)) == 0.0 diff --git a/tests/modules/tracking/test_tracker_quality.py b/tests/modules/tracking/test_tracker_quality.py index b4b5804..b3cc895 100644 --- a/tests/modules/tracking/test_tracker_quality.py +++ b/tests/modules/tracking/test_tracker_quality.py @@ -1,8 +1,8 @@ # Copyright © 2026, UChicago Argonne, LLC # See LICENSE for terms and disclaimer. -"""Phase-B tracking-quality behaviours: hard gap limits, physical motion -constraints, deterministic overlap-first matching, and persisted diagnostics. +"""Tracking-quality behaviours: hard gap limits, physical motion gates, joint +assignment, the heading term, and persisted diagnostics. Synthetic inputs with analytically known outcomes; no stored fixtures. """ @@ -189,53 +189,12 @@ def test_velocity_exceeded_rejects_match(): _, events1 = tracker.track(ds1, stats1, scan_id="site008scan") assert (events1["event_type"] == "CONTINUE").sum() == 0 - assert uid0 in set(events1[events1["event_type"] == "TERMINATION"]["source_cell_uid"]) + # Not continued: the track is kept latent for possible resumption. + assert uid0 in set(events1[events1["event_type"] == "LATENT"]["source_cell_uid"]) assert (events1["event_type"] == "INITIATION").sum() == 1 -def test_acceleration_exceeded_rejects_match(): - """A candidate far faster than the track's own prior speed is rejected.""" - # scan0→1: x2→x3 (3.33 m/s sets prior). scan1→2: x3→x6 (10 m/s) > 2×3.33. - cfg = _make_config( - max_speed_ms=40.0, - max_speed_multiplier=2.0, - acceleration_floor_ms=5.0, - max_tracking_gap_minutes=60.0, - ) - tracker = CellTracker(cfg) - - t0 = np.datetime64("2024-01-01T12:00:00") - t1 = np.datetime64("2024-01-01T12:05:00") - t2 = np.datetime64("2024-01-01T12:10:00") - - ds0, stats0 = _one_cell_scan(t0, 2) - tracker.track(ds0, stats0, scan_id="site009scan") - ds1, stats1 = _one_cell_scan(t1, 3) - _, events1 = tracker.track(ds1, stats1, scan_id="site010scan") - assert (events1["event_type"] == "CONTINUE").sum() == 1, "slow step must continue" - - ds2, stats2 = _one_cell_scan(t2, 6) - _, events2 = tracker.track(ds2, stats2, scan_id="site011scan") - assert (events2["event_type"] == "CONTINUE").sum() == 0, "accelerating step must be rejected" - - -def test_acceleration_floor_admits_storm_motion_after_a_slow_step(): - """A slow prior step (centroid jitter) must not cap the next step below - plausible storm motion: the cap never falls below the floor.""" - cfg = _make_config(max_speed_multiplier=2.0, acceleration_floor_ms=12.0) - tracker = CellTracker(cfg) - t0, t1, t2 = (np.datetime64(f"2024-01-01T12:{m:02d}:00") for m in (0, 5, 10)) - - ds0, stats0 = _one_cell_scan(t0, 2) - tracker.track(ds0, stats0, scan_id="site032scan") - ds1, stats1 = _one_cell_scan(t1, 3) # 3.33 m/s → 2x cap 6.67, floor 12 wins - tracker.track(ds1, stats1, scan_id="site033scan") - ds2, stats2 = _one_cell_scan(t2, 6) # 10 m/s - _, events2 = tracker.track(ds2, stats2, scan_id="site034scan") - assert (events2["event_type"] == "CONTINUE").sum() == 1 - - -def _one_cell_scan_wide(time, x_pix): +def _one_cell_scan_wide(time, x_pix, proj_labels=None): """Like ``_one_cell_scan`` on a 16-column grid, for multi-step motion.""" labels = np.zeros((8, 16), dtype=np.int32) labels[2:4, x_pix : x_pix + 2] = 1 @@ -252,36 +211,39 @@ def _one_cell_scan_wide(time, x_pix): } ], ) - return _synthetic_ds(time, labels), stats + return _synthetic_ds(time, labels, proj_labels=proj_labels), stats -def test_acceleration_reference_is_the_tracks_mean_speed_not_its_last_step(): - """Steps of 13.3, 13.3 then 3.3 m/s give a mean of 10 m/s; a 13.3 m/s step - is within 2x the mean although it is 4x the last step.""" - cfg = _make_config(max_speed_multiplier=2.0, acceleration_floor_ms=5.0) - tracker = CellTracker(cfg) - t = [ - np.datetime64("2024-01-01T12:00:00") + np.timedelta64(s, "s") - for s in (0, 150, 300, 600, 750) - ] +def test_speed_jump_beyond_the_additive_limit_rejects_match(): + """After a 3.3 m/s step the next may reach 3.3 + a·Δt + 2√2σ/Δt ≈ 24.8 m/s + (Δt = 300 s); an 8 km jump (26.7 m/s) is rejected although under v_max.""" + tracker = CellTracker(_make_config()) + t0, t1, t2 = (np.datetime64(f"2024-01-01T12:{m:02d}:00") for m in (0, 5, 10)) + tracker.track(*_one_cell_scan_wide(t0, 2), scan_id="site009scan") + _, events1 = tracker.track(*_one_cell_scan_wide(t1, 3), scan_id="site010scan") + assert (events1["event_type"] == "CONTINUE").sum() == 1, "slow step must continue" + _, events2 = tracker.track(*_one_cell_scan_wide(t2, 11), scan_id="site011scan") + assert (events2["event_type"] == "CONTINUE").sum() == 0 + assert [p.gate for p in tracker.decisions().pairs] == ["SPEED_CHANGE"] + - for i, x in enumerate((1, 3, 5, 6)): # +2 px/150 s, +2 px/150 s, +1 px/300 s - ds, stats = _one_cell_scan_wide(t[i], x) - _, events = tracker.track(ds, stats, scan_id=f"site04{i}scan") - assert (events["event_type"] == "TERMINATION").sum() == 0 - ds, stats = _one_cell_scan_wide(t[4], 8) # +2 px/150 s = 13.3 m/s again - _, events = tracker.track(ds, stats, scan_id="site044scan") - assert (events["event_type"] == "CONTINUE").sum() == 1 +def test_a_slow_cell_may_start_moving(): + """The limit is additive: from 3.3 m/s a 20 m/s step (6×) is admitted.""" + tracker = CellTracker(_make_config()) + t0, t1, t2 = (np.datetime64(f"2024-01-01T12:{m:02d}:00") for m in (0, 5, 10)) + tracker.track(*_one_cell_scan_wide(t0, 2), scan_id="site032scan") + tracker.track(*_one_cell_scan_wide(t1, 3), scan_id="site033scan") + _, events2 = tracker.track(*_one_cell_scan_wide(t2, 9), scan_id="site034scan") + assert (events2["event_type"] == "CONTINUE").sum() == 1 # --------------------------------------------------------------------------- -# B4 — deterministic constraint propagation (no optimisation) +# Joint assignment # --------------------------------------------------------------------------- -def test_unique_candidate_resolved_by_constraint_propagation(): - """A mutually-unique candidate is matched deterministically (PROPAGATED), - never entering Hungarian assignment.""" +def test_unique_candidate_is_assigned_and_keeps_its_uid(): + """A cell facing a single footprint is linked when the cost is under the ceiling.""" cfg = _make_config() tracker = CellTracker(cfg) @@ -297,14 +259,14 @@ def test_unique_candidate_resolved_by_constraint_propagation(): cont = events1[events1["event_type"] == "CONTINUE"] assert len(cont) == 1 - assert cont.iloc[0]["match_method"] == "PROPAGATED", "unique match must not use Hungarian" + assert cont.iloc[0]["match_method"] == "ASSIGNED" assert str(tracked1.iloc[0]["cell_uid"]) == uid0, "continued cell keeps its uid" -def test_ambiguous_group_uses_hungarian(): - """Two projected hulls that each overlap both current cells form an ambiguous - 2×2 component resolved by Hungarian (match_method == HUNGARIAN). The hulls are - kept disjoint by placing them on different rows (one label per pixel).""" +def test_ambiguous_group_is_resolved_jointly(): + """Two projected hulls that each overlap both current cells form one 2×2 + group, resolved jointly: both tracks continue, to different cells. The hulls + are kept disjoint by placing them on different rows (one label per pixel).""" cfg = _make_config(max_tracking_gap_minutes=60.0) tracker = CellTracker(cfg) @@ -333,8 +295,9 @@ def test_ambiguous_group_uses_hungarian(): ) cont = events1[events1["event_type"] == "CONTINUE"] - assert len(cont) >= 1 - assert (cont["match_method"] == "HUNGARIAN").any(), "the ambiguous 2×2 must use Hungarian" + assert len(cont) == 2 + assert cont["target_cell_label"].nunique() == 2 + assert {p.component_id for p in tracker.decisions().pairs} == {0} # --------------------------------------------------------------------------- @@ -359,13 +322,12 @@ def test_continue_event_carries_diagnostics(): cont = events1[events1["event_type"] == "CONTINUE"] assert len(cont) == 1 row = cont.iloc[0] - assert row["match_method"] in {"PROPAGATED", "HUNGARIAN"} + assert row["match_method"] == "ASSIGNED" assert 0.0 <= float(row["candidate_opc"]) <= 1.0 assert 0.0 <= float(row["candidate_ocp"]) <= 1.0 - # cell moved 1000 m in 300 s ≈ 3.33 m/s (residual from the exact projection) - assert float(row["candidate_speed_ms"]) == pytest.approx( - float(row["candidate_centroid_distance_m"]) / 300.0, abs=1e-6 - ) + # The projection predicts the cell exactly (zero residual); it moved 1000 m in 300 s. + assert float(row["candidate_centroid_distance_m"]) == pytest.approx(0.0, abs=1e-6) + assert float(row["candidate_speed_ms"]) == pytest.approx(1000.0 / 300.0) assert pd.notna(row["candidate_final_cost"]) @@ -385,48 +347,34 @@ def test_initiation_event_has_null_diagnostics(): # --------------------------------------------------------------------------- -def test_heading_penalty_breaks_ambiguous_match_toward_consistent_track(): - """With an established +x velocity, a heading penalty steers an ambiguous - match to the heading-consistent candidate instead of the reversed one.""" - cfg = _make_config( - heading_change_penalty_weight=0.5, - max_tracking_gap_minutes=60.0, - ) - tracker = CellTracker(cfg) - - t0 = np.datetime64("2024-01-01T12:00:00") - t1 = np.datetime64("2024-01-01T12:05:00") - t2 = np.datetime64("2024-01-01T12:10:00") - - # scan0→1 establish a +x velocity (heading 0) for track label 1. - ds0, stats0 = _one_cell_scan(t0, 2) - tracker.track(ds0, stats0, scan_id="site019scan") - ds1, stats1 = _one_cell_scan(t1, 4) # perfect projection → CONTINUE, vx>0 - tracker.track(ds1, stats1, scan_id="site020scan") - - # scan2: the registration hull (label 1) fills the whole row band, so it - # overlaps P and Q identically (equal IoU). P and Q are equidistant from the - # prev centroid (x=4) — 2000 m each. Q is made *cheaper* on base cost (its - # reflectivity matches the track, P's differs) so that WITHOUT the heading - # penalty Q wins. Only the +x/−x heading asymmetry can flip it back to P. - labels2 = np.zeros((8, 8), dtype=np.int32) - labels2[2:4, 6:8] = 1 # P ahead (centroid x = 6500, +2000, +x consistent) - labels2[2:4, 2:4] = 2 # Q behind (centroid x = 2500, −2000, −x reversed) - proj2 = np.zeros((8, 8), dtype=np.int32) - proj2[2:4, 0:8] = 1 # symmetric hull spanning the full row band +def test_heading_term_breaks_an_ambiguous_match_toward_the_consistent_track(): + """With an established +x step, two candidates equally far either side of + the prediction are told apart only by the heading term (w_h = 1).""" + tracker = CellTracker(_make_config()) + t0, t1, t2 = (np.datetime64(f"2024-01-01T12:{m:02d}:00") for m in (0, 5, 10)) + tracker.track(*_one_cell_scan_wide(t0, 2), scan_id="site019scan") + tracker.track(*_one_cell_scan_wide(t1, 5), scan_id="site020scan") # +3 km: heading +x + + labels2 = np.zeros((8, 16), dtype=np.int32) + labels2[2:4, 8:10] = 1 # P: +3 km, straight on + labels2[2:4, 2:4] = 2 # Q: −3 km, reversed + proj2 = np.zeros((8, 16), dtype=np.int32) + proj2[2:4, 0:12] = 1 # footprint centred on the previous cell, covering both stats2 = _cell_stats( t2, [ - {"id": 1, "area": 4.0, "cx": 6500.0, "cy": 2500.0, "mean_refl": 42.0, "max_refl": 45.0}, + {"id": 1, "area": 4.0, "cx": 8500.0, "cy": 2500.0, "mean_refl": 40.0, "max_refl": 45.0}, {"id": 2, "area": 4.0, "cx": 2500.0, "cy": 2500.0, "mean_refl": 40.0, "max_refl": 45.0}, ], ) - ds2 = _synthetic_ds(t2, labels2, proj_labels=proj2) - _, events2 = tracker.track(ds2, stats2, scan_id="site021scan") - + _, events2 = tracker.track( + _synthetic_ds(t2, labels2, proj_labels=proj2), stats2, scan_id="site021scan" + ) cont = events2[events2["event_type"] == "CONTINUE"] - assert len(cont) == 1, "the track should continue to exactly one cell" - assert int(cont.iloc[0]["target_cell_label"]) == 1, "heading penalty must steer to the +x cell" + assert len(cont) == 1 + assert int(cont.iloc[0]["target_cell_label"]) == 1 + h = {p.curr_cell_label: p.h for p in tracker.decisions().pairs} + assert h == pytest.approx({1: 0.0, 2: 1.0}) # --------------------------------------------------------------------------- diff --git a/tests/modules/tracking/test_tracker_scan_local_outputs.py b/tests/modules/tracking/test_tracker_scan_local_outputs.py index dec4ae6..827e4d4 100644 --- a/tests/modules/tracking/test_tracker_scan_local_outputs.py +++ b/tests/modules/tracking/test_tracker_scan_local_outputs.py @@ -195,7 +195,7 @@ def test_split_event(tracker): assert tracked2["cell_uid"].nunique() == 2 -def test_merge_event_emits_death(tracker): +def test_merge_keeps_the_absorbed_track_latent(tracker): t1 = np.datetime64("2024-01-01T12:00:00") t2 = np.datetime64("2024-01-01T12:05:00") @@ -249,9 +249,13 @@ def test_merge_event_emits_death(tracker): tracked2, events2 = tracker.track(ds2, stats2, scan_id="site006scan") assert len(tracked2) == 1 assert len(events2[events2["event_type"] == "MERGE"]) == 1 - deaths = events2[events2["event_type"] == "TERMINATION"] - assert len(deaths) >= 1 - assert deaths["source_cell_uid"].notna().any() + # The absorbed track is not ended yet: it stays latent for possible resumption. + latent = events2[events2["event_type"] == "LATENT"] + assert len(latent) == 1 + assert ( + latent["source_cell_uid"].iloc[0] + == events2[events2["event_type"] == "MERGE"]["source_cell_uid"].iloc[0] + ) # --------------------------------------------------------------------------- diff --git a/tests/modules/tracking/test_tracker_v2.py b/tests/modules/tracking/test_tracker_v2.py new file mode 100644 index 0000000..6928891 --- /dev/null +++ b/tests/modules/tracking/test_tracker_v2.py @@ -0,0 +1,304 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + +"""CellTracker v2 end to end on synthetic scans with known answers. + +Grid 1 km; cells at 40 dBZ over a 30 dBZ cell threshold unless stated. +``proj`` is the previous scan's labels advected to the current scan +(``cell_projections[0]``); the flow fields carry latent footprints. +""" + +import numpy as np +import pandas as pd +import pytest +import xarray as xr + +from adapt.modules.tracking.module import CellTracker + +pytestmark = pytest.mark.unit + +T0 = np.datetime64("2025-06-17T14:00:00") +DT = np.timedelta64(277, "s") +SHAPE = (40, 60) + + +def _scan(k, labels, proj=None, refl=None, flow=(0.0, 0.0)): + if refl is None: + refl = np.where(labels > 0, 40.0, 0.0) + proj = labels * 0 if proj is None else proj + ds = xr.Dataset( + { + "cell_labels": (["y", "x"], labels.astype(np.int32)), + "reflectivity": (["y", "x"], refl.astype(np.float32)), + "cell_projections": (["frame_offset", "y", "x"], proj[None].astype(np.int32)), + "heading_x": (["y", "x"], np.full(SHAPE, flow[0], dtype=np.float32)), + "heading_y": (["y", "x"], np.full(SHAPE, flow[1], dtype=np.float32)), + }, + coords={ + "y": np.arange(SHAPE[0]) * 1000.0, + "x": np.arange(SHAPE[1]) * 1000.0, + "frame_offset": [0], + }, + ).assign_coords(time=T0 + k * DT) + stats = pd.DataFrame( + [ + { + "cell_label": int(lbl), + "cell_area_sqkm": float((labels == lbl).sum()), + "radar_reflectivity_mean": float(refl[labels == lbl].mean()), + "radar_reflectivity_max": float(refl[labels == lbl].max()), + "time_volume_start": T0 + k * DT, + "cell_centroid_mass_lat": 35.0, + "cell_centroid_mass_lon": -97.0, + } + for lbl in np.unique(labels[labels > 0]) + ] + ) + return ds, stats + + +def _box(r0, r1, c0, c1, label=1, grid=None): + grid = np.zeros(SHAPE, dtype=np.int32) if grid is None else grid + grid[r0:r1, c0:c1] = label + return grid + + +def _run(tracker, scans): + out = [] + for k, (labels, proj, *rest) in enumerate(scans): + refl = rest[0] if rest else None + ds, stats = _scan(k, labels, proj, refl) + out.append(tracker.track(ds, stats, scan_id=f"scan{k:03d}")) + return out + + +def _types(events): + return sorted(events["event_type"]) + + +def _uid(tracked, label): + return tracked.set_index("cell_label").loc[label, "cell_uid"] + + +@pytest.fixture +def tracker(make_tracking_config): + return CellTracker(make_tracking_config()) + + +# ── continuation ──────────────────────────────────────────────────────────── + + +def test_first_scan_initiates_every_cell(tracker): + labels = _box(5, 9, 5, 9, 1, _box(20, 24, 30, 34, 2)) + ((tracked, events),) = _run(tracker, [(labels, None)]) + assert _types(events) == ["INITIATION", "INITIATION"] + assert tracked["cell_uid"].nunique() == 2 + + +def test_a_cell_where_its_footprint_predicts_continues_its_identity(tracker): + a = _box(10, 14, 10, 14) + b = _box(10, 14, 12, 16) # moved 2 km east, as the flow predicted + (t0, _), (t1, e1) = _run(tracker, [(a, None), (b, b)]) + assert _types(e1) == ["CONTINUE"] + assert _uid(t1, 1) == _uid(t0, 1) + + +def test_continue_event_carries_the_link_evidence(tracker): + a = _box(10, 14, 10, 14) + _, (_, e1) = _run(tracker, [(a, None), (a, a)]) + row = e1.iloc[0] + assert row["candidate_opc"] == pytest.approx(1.0) + assert row["candidate_ocp"] == pytest.approx(1.0) + assert row["candidate_final_cost"] == pytest.approx(0.0) + assert row["match_method"] == "ASSIGNED" + assert row["source_scan_id"] == "scan000" + + +def test_a_cell_far_from_every_footprint_is_an_initiation(tracker): + a = _box(10, 14, 10, 14) + far = _box(30, 34, 45, 49) + _, (_, e1) = _run(tracker, [(a, None), (far, a)]) + assert "CONTINUE" not in set(e1["event_type"]) + assert "INITIATION" in set(e1["event_type"]) + + +def test_a_link_faster_than_the_speed_cap_is_rejected(make_tracking_config): + tracker = CellTracker(make_tracking_config(tracker={"max_speed_ms": 5.0})) + a = _box(10, 14, 10, 14) + b = _box(10, 14, 12, 16) # 2 km in 277 s = 7.2 m/s + _, (_, e1) = _run(tracker, [(a, None), (b, b)]) + assert "CONTINUE" not in set(e1["event_type"]) + pairs = tracker.decisions().pairs + assert [p.gate for p in pairs] == ["SPEED"] + + +def test_no_two_continuing_links_cross(tracker): + # Two cells whose footprints predict they exchanged places diagonally + # (8.5 km in 277 s, within the speed cap). + prev = _box(5, 9, 5, 9, 1, _box(5, 9, 13, 17, 2)) + curr = _box(8, 12, 5, 9, 1, _box(8, 12, 13, 17, 2)) + proj = _box(8, 12, 13, 17, 1, _box(8, 12, 5, 9, 2)) # 1 → right, 2 → left + _, (_, e1) = _run(tracker, [(prev, None), (curr, proj)]) + assert (e1["event_type"] == "CONTINUE").sum() == 1 + outcomes = sorted(p.outcome for p in tracker.decisions().pairs) + assert outcomes == ["CONTINUE", "CROSSING"] + + +# ── split, merge, identity ───────────────────────────────────────────────── + + +def test_a_fragment_inside_a_continuing_footprint_is_a_split_child(tracker): + parent = _box(10, 16, 10, 22) + children = _box(10, 16, 10, 15, 1, _box(10, 16, 17, 22, 2)) + (t0, _), (t1, e1) = _run(tracker, [(parent, None), (children, parent)]) + assert _types(e1) == ["CONTINUE", "SPLIT"] + split = e1[e1["event_type"] == "SPLIT"].iloc[0] + assert split["source_cell_uid"] == _uid(t0, 1) + assert split["target_cell_uid"] != _uid(t0, 1) + + +def test_a_split_child_far_from_the_parent_centre_is_still_a_split(tracker): + # A 30 km parent sheds a fragment at its far end: 12.5 km from the parent's + # centre in 277 s (45 m/s) is the parent's size, not motion. Lineage links + # are tested on the fractions and the crossing test, not on the speed gates. + parent = _box(10, 16, 5, 35) + children = _box(10, 16, 5, 28, 1, _box(10, 16, 30, 35, 2)) + _, (_, e1) = _run(tracker, [(parent, None), (children, parent)]) + assert _types(e1) == ["CONTINUE", "SPLIT"] + + +def test_a_footprint_inside_a_continuing_cell_is_a_merge_source(tracker): + prev = _box(10, 16, 10, 15, 1, _box(10, 16, 17, 22, 2)) + merged = _box(10, 16, 10, 22) + _, (_, e1) = _run(tracker, [(prev, None), (merged, prev)]) + assert _types(e1) == ["CONTINUE", "LATENT", "MERGE"] + + +def test_identity_at_a_merge_follows_the_stronger_source(tracker): + # Weak large cell 1 (35 dBZ, 20 px) and strong small cell 2 (55 dBZ, 12 px) + # merge. Cell 1 explains more of the survivor and wins the assignment, but + # cell 2 carries far more intensity mass, so it keeps the identity. + prev = _box(10, 15, 10, 14, 1, _box(10, 13, 15, 19, 2)) + refl0 = np.where(prev == 1, 35.0, np.where(prev == 2, 55.0, 0.0)) + merged = _box(10, 15, 10, 19) + refl1 = np.where(merged > 0, 45.0, 0.0) + (t0, _), (t1, e1) = _run(tracker, [(prev, None, refl0), (merged, prev, refl1)]) + cont = e1[e1["event_type"] == "CONTINUE"].iloc[0] + assert cont["source_cell_uid"] == _uid(t0, 2) + assert cont["match_method"] == "IDENTITY_MERGE" + assert _uid(t1, 1) == _uid(t0, 2) + merge = e1[e1["event_type"] == "MERGE"].iloc[0] + assert merge["source_cell_uid"] == _uid(t0, 1) + assert any(r.transferred and r.winner for r in tracker.decisions().identity) + + +def test_identity_never_moves_to_a_physically_impossible_continuation(make_tracking_config): + # The same merge as above, but with a speed cap the stronger source's jump to + # the survivor's centre violates: the identity stays with the assigned cell. + tracker = CellTracker(make_tracking_config(tracker={"max_speed_ms": 9.3})) + prev = _box(10, 15, 10, 14, 1, _box(10, 13, 15, 19, 2)) + refl0 = np.where(prev == 1, 35.0, np.where(prev == 2, 55.0, 0.0)) + merged = _box(10, 15, 10, 19) + refl1 = np.where(merged > 0, 45.0, 0.0) + (t0, _), (t1, e1) = _run(tracker, [(prev, None, refl0), (merged, prev, refl1)]) + # Centre to survivor centre: 2.5 km (9.0 m/s) for cell 1, 2.69 km (9.7 m/s) for cell 2. + assert _uid(t1, 1) == _uid(t0, 1) + assert e1[e1["event_type"] == "CONTINUE"].iloc[0]["match_method"] == "ASSIGNED" + + +def test_every_split_and_merge_event_has_one_chosen_lineage_record(tracker): + prev = _box(10, 15, 10, 14, 1, _box(10, 13, 15, 19, 2)) + refl0 = np.where(prev == 1, 35.0, np.where(prev == 2, 55.0, 0.0)) + merged = _box(10, 15, 10, 19) + refl1 = np.where(merged > 0, 45.0, 0.0) + (_, _), (_, e1) = _run(tracker, [(prev, None, refl0), (merged, prev, refl1)]) + lineage = tracker.decisions().lineage + for kind in ("MERGE", "SPLIT"): + sources = set(e1.loc[e1["event_type"] == kind, "source_cell_uid"]) + chosen = [r for r in lineage if r.hypothesis == kind and r.chosen] + side_uid = {r.cell_uid if kind == "MERGE" else r.partner_uid for r in chosen} + assert len(chosen) == len(e1[e1["event_type"] == kind]) + assert side_uid == sources + latent_chosen = {r.cell_uid for r in lineage if r.hypothesis == "LATENT" and r.chosen} + assert latent_chosen == set() # the only unlinked track merged + + +# ── latent tracks ────────────────────────────────────────────────────────── + + +def test_a_cell_missing_for_one_scan_resumes_its_identity(tracker): + a = _box(10, 14, 10, 14) + empty = np.zeros(SHAPE, dtype=np.int32) + (t0, _), (_, e1), (t2, e2) = _run(tracker, [(a, None), (empty, a), (a, empty)]) + assert _types(e1) == ["LATENT"] + assert _types(e2) == ["RESUMED"] + assert _uid(t2, 1) == _uid(t0, 1) + resumed = e2.iloc[0] + assert resumed["source_scan_id"] == "scan000" + + +def test_an_unresumed_latent_track_terminates_after_n_scans(tracker): + a = _box(10, 14, 10, 14) + empty = np.zeros(SHAPE, dtype=np.int32) + (t0, _), (_, e1), (_, e2) = _run(tracker, [(a, None), (empty, a), (empty, empty)]) + assert _types(e2) == ["TERMINATION"] + term = e2.iloc[0] + assert term["source_cell_uid"] == _uid(t0, 1) + assert term["source_scan_id"] == "scan000" + + +def test_without_latency_a_track_terminates_at_once(make_tracking_config): + tracker = CellTracker(make_tracking_config(tracker={"latent_scans": 0})) + a = _box(10, 14, 10, 14) + empty = np.zeros(SHAPE, dtype=np.int32) + _, (_, e1) = _run(tracker, [(a, None), (empty, a)]) + assert _types(e1) == ["TERMINATION"] + + +def test_a_latent_footprint_moves_with_the_flow(tracker): + a = _box(10, 14, 10, 14) + empty = np.zeros(SHAPE, dtype=np.int32) + moved = _box(10, 14, 16, 20) # 6 km east of the last footprint after one more scan + scans = [] + for k, (labels, proj) in enumerate([(a, None), (empty, a), (moved, empty)]): + scans.append(_scan(k, labels, proj, flow=(6.0, 0.0) if k == 2 else (0.0, 0.0))) + t0, _ = tracker.track(*scans[0], scan_id="s0") + tracker.track(*scans[1], scan_id="s1") + t2, e2 = tracker.track(*scans[2], scan_id="s2") + assert _types(e2) == ["RESUMED"] + assert _uid(t2, 1) == _uid(t0, 1) + + +# ── decisions and determinism ─────────────────────────────────────────────── + + +def test_every_candidate_pair_and_cell_is_logged(tracker): + a = _box(10, 14, 10, 14, 1, _box(10, 14, 20, 24, 2)) + _run(tracker, [(a, None), (a, a)]) + d = tracker.decisions() + assert d.scan.n_pairs == len(d.pairs) == 2 + assert {c.fate for c in d.cells} == {"CONTINUE"} + assert d.scan.n_continue == 2 + + +def test_identical_input_gives_identical_output(make_tracking_config): + prev = _box(10, 16, 10, 15, 1, _box(10, 16, 17, 22, 2)) + merged = _box(10, 16, 10, 22) + runs = [] + for _ in range(2): + tracker = CellTracker(make_tracking_config()) + outs = _run(tracker, [(prev, None), (merged, prev), (prev, merged)]) + runs.append(outs) + for (ta, ea), (tb, eb) in zip(*runs, strict=True): + pd.testing.assert_frame_equal(ta, tb) + pd.testing.assert_frame_equal(ea, eb) + + +def test_gap_beyond_the_limit_resets_every_track(tracker): + a = _box(10, 14, 10, 14) + ds0, s0 = _scan(0, a) + ds1, s1 = _scan(5, a, a) # 23 min later, limit 10 min + tracker.track(ds0, s0, scan_id="s0") + _, e1 = tracker.track(ds1, s1, scan_id="s1") + assert _types(e1) == ["INITIATION", "TERMINATION"] + assert tracker.decisions().scan.reset_code == "TRACK_GAP_EXCEEDED" diff --git a/tests/modules/tracking/test_tracking_graph.py b/tests/modules/tracking/test_tracking_graph.py deleted file mode 100644 index 74707c4..0000000 --- a/tests/modules/tracking/test_tracking_graph.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright © 2026, UChicago Argonne, LLC -# See LICENSE for terms and disclaimer. - -import numpy as np -import pytest - -from adapt.modules.tracking.module import TrackingGraph - -pytestmark = pytest.mark.unit - - -def _add_node( - graph: TrackingGraph, - time, - track_index: int = 1, - cell_id: int = 1, - area: float = 4.0, -) -> int: - return graph.add_observation( - time=time, - cell_id=cell_id, - track_index=track_index, - area=area, - centroid_x=10.0, - centroid_y=10.0, - mean_reflectivity=40.0, - max_reflectivity=50.0, - core_area=2.0, - cell_uid="TESTUIDA0", - track_signature="v1|sig", - ) - - -T1 = np.datetime64("2024-01-01T12:00:00") -T2 = np.datetime64("2024-01-01T12:05:00") -T3 = np.datetime64("2024-01-01T12:10:00") - - -def test_add_observation_returns_sequential_ids(): - g = TrackingGraph() - n0 = _add_node(g, T1) - n1 = _add_node(g, T2) - assert n0 == 0 - assert n1 == 1 - - -def test_get_new_track_index_starts_at_one(): - g = TrackingGraph() - assert g.get_new_track_index() == 1 - assert g.get_new_track_index() == 2 - - -def test_get_nodes_at_time_returns_nodes_for_that_time(): - g = TrackingGraph() - n0 = _add_node(g, T1, cell_id=1) - n1 = _add_node(g, T1, cell_id=2) - n2 = _add_node(g, T2, cell_id=1) - - at_t1 = g.get_nodes_at_time(T1) - at_t2 = g.get_nodes_at_time(T2) - - assert sorted(at_t1) == sorted([n0, n1]) - assert at_t2 == [n2] - - -def test_get_nodes_at_time_unknown_time_returns_empty(): - g = TrackingGraph() - _add_node(g, T1) - assert g.get_nodes_at_time(T3) == [] - - -def test_get_node_attr_returns_stored_value(): - g = TrackingGraph() - n = g.add_observation( - time=T1, - cell_id=7, - track_index=3, - area=12.5, - centroid_x=5.0, - centroid_y=6.0, - mean_reflectivity=38.0, - max_reflectivity=52.0, - core_area=3.1, - cell_uid="ABCDE12345", - track_signature="v1|test", - ) - assert g.get_node_attr(n, "area") == pytest.approx(12.5) - assert g.get_node_attr(n, "cell_id") == 7 - assert g.get_node_attr(n, "cell_uid") == "ABCDE12345" diff --git a/tests/modules/tracking/test_validation.py b/tests/modules/tracking/test_validation.py deleted file mode 100644 index 570c22f..0000000 --- a/tests/modules/tracking/test_validation.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright © 2026, UChicago Argonne, LLC -# See LICENSE for terms and disclaimer. - -"""Unit tests for the bidirectional-overlap GeometricValidator.""" - -import numpy as np -import pytest - -from adapt.modules.tracking.matching.validation import GeometricValidator - -pytestmark = pytest.mark.unit - - -def _mask(shape, r0, r1, c0, c1): - m = np.zeros(shape, dtype=bool) - m[r0:r1, c0:c1] = True - return m - - -def test_well_matched_pair_passes(): - hull = _mask((8, 8), 2, 6, 2, 6) # 16 px - cell = _mask((8, 8), 2, 6, 2, 6) # identical → Opc=Ocp=1 - v = GeometricValidator(0.2, 0.2).validate(hull, cell) - assert v.passed and v.opc == pytest.approx(1.0) and v.ocp == pytest.approx(1.0) - - -def test_tiny_cell_inside_large_hull_rejected(): - hull = _mask((10, 10), 0, 10, 0, 10) # 100 px - cell = _mask((10, 10), 4, 6, 4, 6) # 4 px fully inside - v = GeometricValidator(0.2, 0.2).validate(hull, cell) - # Opc = 4/4 = 1.0 (candidate fully covered) but Ocp = 4/100 = 0.04 → gate fails - assert v.opc == pytest.approx(1.0) - assert v.ocp == pytest.approx(0.04) - assert not v.passed - - -def test_large_cell_engulfing_prediction_rejected(): - hull = _mask((10, 10), 4, 6, 4, 6) # 4 px prediction - cell = _mask((10, 10), 0, 10, 0, 10) # 100 px engulfs it - v = GeometricValidator(0.2, 0.2).validate(hull, cell) - # Ocp = 4/4 = 1.0 but Opc = 4/100 = 0.04 → gate fails - assert v.ocp == pytest.approx(1.0) - assert v.opc == pytest.approx(0.04) - assert not v.passed - - -def test_grazing_contact_rejected(): - hull = _mask((10, 10), 0, 5, 0, 5) # 25 px - cell = _mask((10, 10), 4, 9, 4, 9) # 25 px, overlaps only rows/cols 4 → 1 px - v = GeometricValidator(0.2, 0.2).validate(hull, cell) - assert v.opc == pytest.approx(1.0 / 25.0) - assert v.ocp == pytest.approx(1.0 / 25.0) - assert not v.passed diff --git a/tests/persistence/test_collection_provisioning.py b/tests/persistence/test_collection_provisioning.py index b09b24b..fd55867 100644 --- a/tests/persistence/test_collection_provisioning.py +++ b/tests/persistence/test_collection_provisioning.py @@ -75,10 +75,12 @@ def test_decisions_has_the_decision_tables_only(self, store): store.close() assert _tables(store.root / "collections" / "KILX" / "decisions.db") == { - "tracking_frames", - "tracking_candidates", - "tracking_unmatched", - "tracking_split_merge_tests", + "tracking_scans", + "tracking_cells", + "tracking_pairs", + "tracking_lineage", + "tracking_identity", + "tracking_latent", "segmentation_frames", "segmentation_seeds", "segmentation_components", diff --git a/tests/persistence/test_decision_store.py b/tests/persistence/test_decision_store.py index 505b2cc..68e155e 100644 --- a/tests/persistence/test_decision_store.py +++ b/tests/persistence/test_decision_store.py @@ -5,6 +5,7 @@ and the router dispatches ``DecisionTableWrite`` there without touching products.""" import sqlite3 +from dataclasses import fields from datetime import UTC, datetime import pytest @@ -16,10 +17,13 @@ SegmentationDecisions, SegmentationFrame, SegmentationSeed, - TrackingCandidate, + TrackingCell, TrackingDecisions, - TrackingFrame, - TrackingUnmatched, + TrackingIdentity, + TrackingLatent, + TrackingLineage, + TrackingPair, + TrackingScan, ) from adapt.persistence.decision_store import DecisionStore from adapt.persistence.output_router import StoreOutputRouter @@ -27,6 +31,8 @@ pytestmark = pytest.mark.unit +SCAN_COUNTS = [f.name for f in fields(TrackingScan) if f.name.startswith("n_")] + SCAN_TIME = datetime(2026, 8, 13, 17, 31, 58, tzinfo=UTC) META = PersistenceMeta( scan_time=SCAN_TIME, scan_id="abc123", run_id="run-1", source_file="f", collection_id="KILX" @@ -34,38 +40,51 @@ def _tracking() -> TrackingDecisions: - frame = TrackingFrame(270.0, None, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0) - candidate = TrackingCandidate( + scan = TrackingScan(270.0, None, **dict.fromkeys(SCAN_COUNTS, 1)) + cell = TrackingCell(1, "AAAA", 4.0, 40.0, 10.0, 3500.0, 2500.0, 3.69, 4.0, "CONTINUE") + pair = TrackingPair( prev_cell_uid="AAAA", prev_cell_label=1, curr_cell_label=1, - track_steps_before=0, - hull_area_px=4, - hull_centroid_x=2500.0, - hull_centroid_y=2500.0, - curr_area_px=4, - curr_centroid_x=3500.0, - curr_centroid_y=2500.0, - intersection_px=2, - opc=0.5, - ocp=0.5, - overlap_passed=True, + source="LIVE", + steps=1, + growth_radius_km=1.67, + footprint_area_px=4, + footprint_grown_px=16, + cell_grown_px=16, + intersection_px=12, + o_c=0.75, + o_h=0.75, + u=0.25, + predicted_x=2500.0, + predicted_y=2500.0, + curr_centre_x=3500.0, + curr_centre_y=2500.0, + d=1.2, speed_ms=3.7, - previous_speed_ms=None, - accel_cap_ms=None, - kinematic_code=None, - displacement_m=1000.0, - length_scale_m=2256.0, + prev_speed_ms=None, + speed_limit_ms=None, heading_change_deg=None, - cost=0.94, - component_n_prev=1, - component_n_curr=1, + h=0.0, + cost=0.55, + gate="PASS", + component_id=0, cost_rank=1, - match_method="PROPAGATED", - last_stage="ASSIGNMENT", + margin=1.25, + crossing_with_uid=None, outcome="CONTINUE", ) - return TrackingDecisions(frame, (candidate,), (), ()) + lineage = ( + TrackingLineage("curr", "BBBB", 2, "SPLIT", "AAAA", 1, 0.8, None, 0.65, True, True, None), + TrackingLineage( + "curr", "BBBB", 2, "INITIATION", None, None, None, None, None, True, False, None + ), + ) + identity = (TrackingIdentity("SPLIT", "AAAA", "curr", 1, 3.69, 0.4, True, "SCORE", False),) + latent = ( + TrackingLatent("CCCC", 3, "prev-scan", 1, "TERMINATED", None, "CREATED", 5, 0.0, 0.0), + ) + return TrackingDecisions(scan, (cell,), (pair,), lineage, identity, latent) def _segmentation() -> SegmentationDecisions: @@ -117,16 +136,30 @@ def _rows(path, sql): def test_tracking_bundle_lands_in_its_tables_with_stamps(collection): with DecisionStore(collection.decisions_path) as store: store.write(_tracking(), META) - (frame,) = _rows(collection.decisions_path, "select * from tracking_frames") - (cand,) = _rows(collection.decisions_path, "select * from tracking_candidates") - assert (frame["run_id"], frame["scan_id"], frame["n_prev"]) == ("run-1", "abc123", 1) - assert frame["scan_time"].startswith("2026-08-13T17:31:58") - assert (cand["prev_cell_uid"], cand["outcome"], cand["overlap_passed"]) == ( - "AAAA", - "CONTINUE", - 1, - ) - assert cand["previous_speed_ms"] is None + (scan,) = _rows(collection.decisions_path, "select * from tracking_scans") + (pair,) = _rows(collection.decisions_path, "select * from tracking_pairs") + (cell,) = _rows(collection.decisions_path, "select * from tracking_cells") + assert (scan["run_id"], scan["scan_id"], scan["n_prev"]) == ("run-1", "abc123", 1) + assert scan["scan_time"].startswith("2026-08-13T17:31:58") + assert (pair["prev_cell_uid"], pair["outcome"], pair["gate"]) == ("AAAA", "CONTINUE", "PASS") + assert pair["prev_speed_ms"] is None + assert (cell["cell_uid"], cell["fate"]) == ("AAAA", "CONTINUE") + + +def test_every_lineage_hypothesis_is_kept_in_order(collection): + with DecisionStore(collection.decisions_path) as store: + store.write(_tracking(), META) + rows = _rows(collection.decisions_path, "select * from tracking_lineage order by seq") + assert [(r["hypothesis"], r["chosen"]) for r in rows] == [("SPLIT", 1), ("INITIATION", 0)] + + +def test_identity_and_latent_records_land_in_their_tables(collection): + with DecisionStore(collection.decisions_path) as store: + store.write(_tracking(), META) + (identity,) = _rows(collection.decisions_path, "select * from tracking_identity") + (latent,) = _rows(collection.decisions_path, "select * from tracking_latent") + assert (identity["rule"], identity["winner"]) == ("SCORE", 1) + assert (latent["cell_uid"], latent["status"]) == ("CCCC", "CREATED") def test_segmentation_bundle_numbers_seeds_in_order(collection): @@ -142,7 +175,7 @@ def test_rewriting_a_scan_replaces_its_rows(collection): with DecisionStore(collection.decisions_path) as store: store.write(_tracking(), META) store.write(_tracking(), META) - assert len(_rows(collection.decisions_path, "select * from tracking_candidates")) == 1 + assert len(_rows(collection.decisions_path, "select * from tracking_pairs")) == 1 def test_router_dispatches_decision_writes_without_a_scan_product(collection): @@ -151,7 +184,7 @@ class _Module: persistence = (DecisionTableWrite(key="tracking_decisions"),) StoreOutputRouter(collection).persist([_Module()], {"tracking_decisions": _tracking()}, META) - assert len(_rows(collection.decisions_path, "select * from tracking_frames")) == 1 + assert len(_rows(collection.decisions_path, "select * from tracking_scans")) == 1 assert _rows(collection.products_path, "select * from table_schemas") == [] @@ -161,17 +194,16 @@ class _Module: persistence = (DecisionTableWrite(key="tracking_decisions"),) StoreOutputRouter(collection).persist([_Module()], {}, META) - assert _rows(collection.decisions_path, "select * from tracking_frames") == [] + assert _rows(collection.decisions_path, "select * from tracking_scans") == [] -def test_unmatched_rows_keep_their_reason(collection): - unmatched = (TrackingUnmatched("prev", "AAAA", 1, 0, None, None, "NO_CANDIDATE"),) - bundle = TrackingDecisions(_tracking().frame, _tracking().candidates, unmatched, ()) +def test_a_decisions_db_from_the_previous_tracker_still_takes_new_rows(collection): + # Runs into an existing store: the v1 tracking tables stay as they are and + # the v2 records go to their own tables. + conn = sqlite3.connect(collection.decisions_path) + conn.execute("CREATE TABLE IF NOT EXISTS tracking_candidates (run_id TEXT, opc REAL)") + conn.commit() + conn.close() with DecisionStore(collection.decisions_path) as store: - store.write(bundle, META) - (row,) = _rows(collection.decisions_path, "select * from tracking_unmatched") - assert (row["side"], row["reason"], row["best_candidate_label"]) == ( - "prev", - "NO_CANDIDATE", - None, - ) + store.write(_tracking(), META) + assert len(_rows(collection.decisions_path, "select * from tracking_pairs")) == 1 diff --git a/tests/persistence/test_track_store_products.py b/tests/persistence/test_track_store_products.py index f3582a0..f73be50 100644 --- a/tests/persistence/test_track_store_products.py +++ b/tests/persistence/test_track_store_products.py @@ -187,6 +187,99 @@ def test_retroactive_split_flag_on_previous_scan(self, collection, ledger): assert events_df.loc[0, "target_scan_id"] == "s2" +def _event(event_type, source, target=None, source_scan=None, target_label=None): + return { + "event_type": event_type, + "source_cell_uid": source, + "target_cell_uid": target, + "source_cell_label": 1, + "target_cell_label": target_label, + "source_scan_id": source_scan, + "source_scan_time": None if source_scan is None else SCAN_TIME, + "cost": None, + "is_dominant": False, + "event_group_id": f"{event_type}:{source}", + } + + +class TestLatentTracks: + """A track that vanishes is kept latent; its end or resumption is decided scans + later and refers back to the scan it was last observed in.""" + + def _vanish(self, collection, ledger): + _write(collection, ledger, "s1") + _write( + collection, + ledger, + "s2", + offset_s=300, + stats=_stats([2]), + tracked=_tracked([(2, "u2")]), + events=pd.DataFrame([_event("LATENT", "u1", source_scan="s1")]), + ) + + def test_a_latent_track_is_not_terminated(self, collection, ledger): + self._vanish(collection, ledger) + with TrackStore(collection.products_path, ledger=ledger) as ts: + s1 = ts.get_cells_by_scan(RUN, "s1") + tracks = ts.get_cell_tracks(RUN).set_index("cell_uid") + assert s1.loc[0, "is_terminated_after_here"] == 0 + assert tracks.loc["u1", "termination_type"] == "ACTIVE_AT_END" + + def test_expiry_terminates_at_the_last_observed_scan(self, collection, ledger): + self._vanish(collection, ledger) + _write( + collection, + ledger, + "s3", + offset_s=600, + stats=_stats([2]), + tracked=_tracked([(2, "u2")]), + events=pd.DataFrame([_event("TERMINATION", "u1", source_scan="s1")]), + ) + with TrackStore(collection.products_path, ledger=ledger) as ts: + s1 = ts.get_cells_by_scan(RUN, "s1") + events = ts.get_cell_events(RUN, "u1").set_index("event_type") + tracks = ts.get_cell_tracks(RUN).set_index("cell_uid") + assert s1.loc[0, "is_terminated_after_here"] == 1 + assert events.loc["TERMINATION", "source_scan_id"] == "s1" + assert tracks.loc["u1", "termination_type"] == "TERMINATION" + + def test_an_expired_merge_source_ends_merged_into_its_survivor(self, collection, ledger): + self._vanish(collection, ledger) + _write( + collection, + ledger, + "s3", + offset_s=600, + stats=_stats([2]), + tracked=_tracked([(2, "u2")]), + events=pd.DataFrame([_event("TERMINATION", "u1", "u2", source_scan="s1")]), + ) + with TrackStore(collection.products_path, ledger=ledger) as ts: + tracks = ts.get_cell_tracks(RUN).set_index("cell_uid") + assert tracks.loc["u1", "termination_type"] == "MERGED" + assert tracks.loc["u1", "terminated_into_cell_uid"] == "u2" + + def test_a_resumed_track_continues_its_age(self, collection, ledger): + self._vanish(collection, ledger) + _write( + collection, + ledger, + "s3", + offset_s=600, + stats=_stats([1, 2]), + tracked=_tracked([(1, "u1"), (2, "u2")]), + events=pd.DataFrame([_event("RESUMED", "u1", "u1", source_scan="s1", target_label=1)]), + ) + with TrackStore(collection.products_path, ledger=ledger) as ts: + history = ts.get_track_history(RUN, "u1") + events = ts.get_cell_events(RUN, "u1").set_index("event_type") + assert history["age_seconds"].tolist() == [0.0, 600.0] + assert events.loc["RESUMED", "source_scan_id"] == "s1" + assert events.loc["RESUMED", "target_scan_id"] == "s3" + + class TestStrictTrackedColumns: def test_missing_max_reflectivity_column_raises(self, collection, ledger): # A tracked_cells frame without max_reflectivity must raise loudly, diff --git a/tests/validation/test_check_functions.py b/tests/validation/test_check_functions.py index 5b1bf92..01616a6 100644 --- a/tests/validation/test_check_functions.py +++ b/tests/validation/test_check_functions.py @@ -294,6 +294,12 @@ def test_fails_on_missing_column(self): with pytest.raises(ContractViolation): check_cell_events(df) + @pytest.mark.parametrize("event_type", ["LATENT", "RESUMED"]) + def test_latent_track_events_are_valid(self, event_type): + df = _valid_cell_events().copy() + df["event_type"] = event_type + check_cell_events(df) + def test_fails_on_invalid_event_type(self): df = _valid_cell_events().copy() df["event_type"] = "UNKNOWN" From a63ba36596bcb555b095713c9b0ae41812979511 Mon Sep 17 00:00:00 2001 From: Bhupendra Raut Date: Thu, 5 Nov 2026 12:06:33 -0600 Subject: [PATCH 11/12] DOCS: README tagline and large-scale analysis overview --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 34af2f7..5bba0eb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Adapt -**Real-time detection, projection, and tracking of convective storms in weather radar data, built to support adaptive radar scanning at the DOE ARM User Facility.** +**Real-time detection, projection, and tracking of convective storms in weather radar data — built for operational adaptive scanning at the DOE ARM User Facility, and for large-scale generation of value-added products and AI-ready storm datasets.** ![Status](https://img.shields.io/badge/STATUS-ACTIVE%20DEVELOPMENT-orange?style=for-the-badge&logo=github) ![API](https://img.shields.io/badge/API-BREAKING%20CHANGES-red?style=for-the-badge&logo=dependabot) @@ -60,6 +60,12 @@ applications such as storm lifecycle studies, radar–lightning correlation anal of analysis-ready, multi-sensor datasets combining radar, satellite, and Lightning Mapping Array observations. +## From operations to large-scale analysis + +The pipeline that selects targets in real time also reprocesses years of archived data, with identical, +traceable results. Its tracked cells, lifecycle events and decision logs make ready-made value-added +products and consistent datasets for AI-driven research. + ## Current status Adapt is at an alpha stage of development. From c3fb81519e6311670018c31417d38d6562b9c2b9 Mon Sep 17 00:00:00 2001 From: Bhupendra Raut Date: Thu, 5 Nov 2026 12:18:12 -0600 Subject: [PATCH 12/12] FIX: license header in utils/basemap.py --- src/adapt/utils/basemap.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/adapt/utils/basemap.py b/src/adapt/utils/basemap.py index db28750..689ab0b 100644 --- a/src/adapt/utils/basemap.py +++ b/src/adapt/utils/basemap.py @@ -1,3 +1,6 @@ +# Copyright © 2026, UChicago Argonne, LLC +# See LICENSE for terms and disclaimer. + """Tile-server identity and cache location shared by every basemap consumer. Tile providers require a stable User-Agent naming the application and a