diff --git a/src/vidxp/application_models.py b/src/vidxp/application_models.py index 395bde4..9b1121f 100644 --- a/src/vidxp/application_models.py +++ b/src/vidxp/application_models.py @@ -841,9 +841,36 @@ class EvidenceBoardCandidate(ApplicationModel): frame_index: int | None = Field(default=None, ge=0) frame_match: "EvidenceFrameMatch" score: float | None = None + score_kind: Literal["ordering_only"] | None = Field( + default=None, + description=( + "Score calibration indicator. ordering_only indicates relative " + "sorting rank, not confidence or probability." + ), + ) + score_direction: Literal["higher_is_better"] | None = Field( + default=None, + description="Ranking direction for the candidate score value.", + ) display_text: str | None = Field(default=None, max_length=512) provenance: dict[str, JsonValue] = Field(default_factory=dict) + @model_validator(mode="before") + @classmethod + def _populate_candidate_score_metadata(cls, value: Any) -> Any: + if not isinstance(value, Mapping): + return value + payload = dict(value) + if payload.get("score") is not None: + if payload.get("score_kind") is None: + payload["score_kind"] = "ordering_only" + if payload.get("score_direction") is None: + payload["score_direction"] = "higher_is_better" + else: + payload["score_kind"] = None + payload["score_direction"] = None + return payload + @model_validator(mode="after") def _valid_interval(self) -> "EvidenceBoardCandidate": if self.end < self.start: @@ -895,14 +922,49 @@ def _unique_modalities( class SearchHit(ApplicationModel): - rank: int = Field(gt=0) + rank: int = Field( + gt=0, + description="1-based rank within this search channel.", + ) + channel_rank: int | None = Field( + default=None, + gt=0, + description="Explicit 1-based rank within this search channel.", + ) media_id: MediaId video_id: VideoId generation_id: IndexGenerationId start: float = Field(ge=0) end: float = Field(gt=0) - score: float - raw_distance: float + score: float = Field( + description="Monotonic ordering score derived from raw distance; higher is better." + ) + score_kind: Literal["ordering_only"] = Field( + default="ordering_only", + description=( + "Score calibration indicator. ordering_only indicates relative " + "sorting rank, not confidence or probability." + ), + ) + score_direction: Literal["higher_is_better"] = Field( + default="higher_is_better", + description="Ranking direction for the score value.", + ) + score_conversion: Literal["negated_distance"] = Field( + default="negated_distance", + description="Formula or method used to convert raw distance to score.", + ) + raw_distance: float = Field( + description="Raw vector distance returned by the underlying index." + ) + distance_metric: Literal["cosine", "l2", "unspecified"] = Field( + default="cosine", + description="Vector distance metric used during retrieval.", + ) + distance_direction: Literal["lower_is_better"] = Field( + default="lower_is_better", + description="Ranking direction for the raw distance value.", + ) modality: str = Field(min_length=1) source_id: str = Field(min_length=1) metadata: dict[str, JsonValue] = Field(default_factory=dict) @@ -935,6 +997,17 @@ def inspect(item: JsonValue) -> None: raise ValueError("Search metadata contains internal location fields.") return value + @model_validator(mode="before") + @classmethod + def _populate_channel_rank(cls, value: Any) -> Any: + if not isinstance(value, Mapping): + return value + payload = dict(value) + if "channel_rank" not in payload or payload["channel_rank"] is None: + if "rank" in payload: + payload["channel_rank"] = payload["rank"] + return payload + @model_validator(mode="after") def _validate_interval(self) -> "SearchHit": if self.end <= self.start: @@ -965,18 +1038,84 @@ class FusionProvenance(ApplicationModel): overlap_rule: Literal["connected_intervals"] = "connected_intervals" requested_modalities: tuple[Identifier, ...] = () searched_modalities: tuple[Identifier, ...] = () + score_kind: Literal["ordering_only"] = Field( + default="ordering_only", + description=( + "Score calibration indicator. ordering_only indicates relative " + "sorting rank, not confidence or probability." + ), + ) + score_direction: Literal["higher_is_better"] = Field( + default="higher_is_better", + description="Ranking direction for the combined score value.", + ) + scoring_method: Literal["reciprocal_rank_fusion"] = Field( + default="reciprocal_rank_fusion", + description="Scoring method used to combine channels into a single moment score.", + ) class FusedMoment(ApplicationModel): moment_id: Sha256 | None = None - rank: int = Field(gt=0) - score: float = Field(gt=0) + rank: int = Field( + gt=0, + description="1-based combined rank of the fused moment across all search channels.", + ) + combined_rank: int | None = Field( + default=None, + gt=0, + description="Explicit 1-based combined rank across all search channels.", + ) + score: float = Field( + gt=0, + description="Reciprocal rank fusion (RRF) score; higher is better.", + ) + score_kind: Literal["ordering_only"] = Field( + default="ordering_only", + description=( + "Score calibration indicator. ordering_only indicates relative " + "sorting rank, not confidence or probability." + ), + ) + score_direction: Literal["higher_is_better"] = Field( + default="higher_is_better", + description="Ranking direction for the combined score value.", + ) + scoring_method: Literal["reciprocal_rank_fusion"] = Field( + default="reciprocal_rank_fusion", + description="Scoring method used to combine channels into a single moment score.", + ) media_id: MediaId start: float = Field(ge=0) end: float = Field(gt=0) modalities: tuple[Identifier, ...] + contributing_channels: tuple[Identifier, ...] = Field( + default=(), + description="Search channels that contributed hits to this returned moment.", + ) + channels_run: tuple[Identifier, ...] = Field( + default=(), + description="All search channels executed during the search/fusion run.", + ) hits: tuple[SearchHit, ...] = Field(min_length=1) + @model_validator(mode="before") + @classmethod + def _populate_fused_moment_defaults(cls, value: Any) -> Any: + if not isinstance(value, Mapping): + return value + payload = dict(value) + if "combined_rank" not in payload or payload["combined_rank"] is None: + if "rank" in payload: + payload["combined_rank"] = payload["rank"] + if ( + "contributing_channels" not in payload + or not payload["contributing_channels"] + ): + if "modalities" in payload: + payload["contributing_channels"] = payload["modalities"] + return payload + @model_validator(mode="after") def _validate_fused_moment(self) -> "FusedMoment": if self.end <= self.start: diff --git a/src/vidxp/capabilities/search.py b/src/vidxp/capabilities/search.py index 9c57ddc..bbcf0ed 100644 --- a/src/vidxp/capabilities/search.py +++ b/src/vidxp/capabilities/search.py @@ -72,9 +72,7 @@ def _to_hits( hits = [] for rank, row in enumerate(ordered, start=1): metadata = row["metadata"] - missing = sorted( - (required_metadata | {"generation_id"}) - metadata.keys() - ) + missing = sorted((required_metadata | {"generation_id"}) - metadata.keys()) if missing: raise IndexSchemaError( "The saved index predates the benchmark-ready schema and must " @@ -84,20 +82,25 @@ def _to_hits( end = float(metadata["end"]) if start < 0 or end <= start: raise IndexSchemaError( - f"Invalid {modality} interval in {row['source_id']}: " - f"[{start}, {end}]." + f"Invalid {modality} interval in {row['source_id']}: [{start}, {end}]." ) distance = float(row["raw_distance"]) hits.append( SearchHit( rank=rank, + channel_rank=rank, media_id=str(metadata["video_id"]), video_id=str(metadata["video_id"]), generation_id=str(metadata["generation_id"]), start=start, end=end, score=distance_to_score(distance), + score_kind="ordering_only", + score_direction="higher_is_better", + score_conversion="negated_distance", raw_distance=distance, + distance_metric="cosine", + distance_direction="lower_is_better", modality=modality, source_id=str(row["source_id"]), metadata={ @@ -129,9 +132,7 @@ def search_embeddings( if top_k <= 0: raise ValueError("top_k must be greater than zero.") if modality not in config.enabled_modalities: - raise ValueError( - f"The {modality} modality is not present in this index run." - ) + raise ValueError(f"The {modality} modality is not present in this index run.") rows = storage.query( modality, embedding, diff --git a/src/vidxp/cli_support.py b/src/vidxp/cli_support.py index 934c3b5..28ad85d 100644 --- a/src/vidxp/cli_support.py +++ b/src/vidxp/cli_support.py @@ -147,21 +147,32 @@ def emit_search( if not result.moments: typer.echo("No matching moments found.") return - table = Table(title="Fused search results") + table = Table( + title="Fused search results", + caption=( + "Scores are reciprocal rank fusion (RRF) ordering scores; higher is better. " + "They are uncalibrated sorting values, not confidence or probabilities." + ), + ) table.add_column("Rank", justify="right") table.add_column("Start", justify="right") table.add_column("End", justify="right") - table.add_column("Score", justify="right") + table.add_column("Score (RRF ↑)", justify="right") table.add_column("Video") - table.add_column("Modalities") + table.add_column("Contributing Channels") for moment in result.moments: + channels = moment.contributing_channels or moment.modalities table.add_row( - str(moment.rank), + str( + moment.combined_rank + if moment.combined_rank is not None + else moment.rank + ), f"{moment.start:.3f}s", f"{moment.end:.3f}s", f"{moment.score:.6f}", moment.media_id, - ", ".join(moment.modalities), + ", ".join(channels), ) Console().print(table) @@ -182,8 +193,7 @@ def emit_query( console.print(f" Evidence: {', '.join(claim.evidence_ids)}") elif result.evidence: console.print( - f"No generated answer; returning {len(result.evidence)} " - "evidence item(s)." + f"No generated answer; returning {len(result.evidence)} evidence item(s)." ) else: console.print("No supporting evidence found.") @@ -276,11 +286,7 @@ def parse_modalities( value: str, available: Iterable[str], ) -> tuple[str, ...]: - selected = tuple( - item.strip().lower() - for item in value.split(",") - if item.strip() - ) + selected = tuple(item.strip().lower() for item in value.split(",") if item.strip()) return selected_modalities(selected, available) diff --git a/src/vidxp/evidence_delivery.py b/src/vidxp/evidence_delivery.py index ec01cbe..64860a9 100644 --- a/src/vidxp/evidence_delivery.py +++ b/src/vidxp/evidence_delivery.py @@ -193,6 +193,8 @@ def _search_candidates( else EvidenceFrameMatch.representative ), score=moment.score, + score_kind="ordering_only", + score_direction="higher_is_better", display_text=EvidenceDeliveryService._display_text( selected.metadata ), @@ -302,15 +304,11 @@ def prepare_board_request( by_id = {candidate.evidence_id: candidate for candidate in candidates} if evidence_ids is None: selected = tuple( - candidate - for candidate in candidates - if candidate.rank >= start_rank + candidate for candidate in candidates if candidate.rank >= start_rank ) else: missing = tuple( - evidence_id - for evidence_id in evidence_ids - if evidence_id not in by_id + evidence_id for evidence_id in evidence_ids if evidence_id not in by_id ) if missing: raise ApplicationError( diff --git a/src/vidxp/mcp.py b/src/vidxp/mcp.py index c359c33..23c0ce7 100644 --- a/src/vidxp/mcp.py +++ b/src/vidxp/mcp.py @@ -1091,6 +1091,10 @@ def evidence_index( if board is not None and board.next_start_rank is not None: lines.append(f"More candidates start at rank {board.next_start_rank}.") + lines.append( + "Note: candidate ranks and scores are relative ordering values " + "(higher is better), not calibrated probabilities or confidence." + ) lines.append( "Use materialize_job_evidence with this job ID and up to ten evidence " "IDs for standalone frames or clips." @@ -1162,18 +1166,14 @@ def evidence_app_payload( else 0.0 ), "end": ( - resolved.source_end_seconds - if resolved is not None - else 0.0 + resolved.source_end_seconds if resolved is not None else 0.0 ), "display_text": None, "state": item.state.value, } ) requested_count = len(delivery.items) - rendered_count = sum( - item.state.value == "ready" for item in delivery.items - ) + rendered_count = sum(item.state.value == "ready" for item in delivery.items) failed_count = requested_count - rendered_count next_start_rank = None @@ -1636,10 +1636,13 @@ def submit(actor: Principal) -> Job: description=( "Submit a durable ranked moment search. Set command.media_id to " "search one registered video; omit it to search across every media " - "item in the active index snapshot. MCP returns an annotated board " - "of ranked results by default. Set command.evidence_delivery.mode " - "to keyframes or keyframes_and_clips only when standalone artifacts " - "are also needed, then use wait_job and get_job_evidence." + "item in the active index snapshot. Returned moments contain " + "uncalibrated reciprocal rank fusion (RRF) scores (ordering_only, higher " + "is better) distinguishing channel rank from combined moment rank. " + "MCP returns an annotated board of ranked results by default. " + "Set command.evidence_delivery.mode to keyframes or keyframes_and_clips " + "only when standalone artifacts are also needed, then use wait_job " + "and get_job_evidence." ), annotations=_SUBMIT, structured_output=True, diff --git a/src/vidxp/search_fusion.py b/src/vidxp/search_fusion.py index b7e4184..28091b5 100644 --- a/src/vidxp/search_fusion.py +++ b/src/vidxp/search_fusion.py @@ -170,6 +170,7 @@ def fuse_search_results( moments = tuple( FusedMoment( rank=rank, + combined_rank=rank, moment_id=_moment_id( snapshot_id=snapshot_id, media_id=candidate["media_id"], @@ -177,6 +178,11 @@ def fuse_search_results( end=candidate["end"], hits=candidate["hits"], ), + score_kind="ordering_only", + score_direction="higher_is_better", + scoring_method="reciprocal_rank_fusion", + contributing_channels=candidate["modalities"], + channels_run=searched_modalities, **candidate, ) for rank, candidate in enumerate(candidates[:top_k], start=1) @@ -194,5 +200,8 @@ def fuse_search_results( fusion=FusionProvenance( requested_modalities=requested_modalities, searched_modalities=searched_modalities, + score_kind="ordering_only", + score_direction="higher_is_better", + scoring_method="reciprocal_rank_fusion", ), ) diff --git a/tests/test_search.py b/tests/test_search.py index accb7d8..717a99e 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -99,8 +99,14 @@ def test_top_k_filter_order_distance_and_score_are_preserved(self): ], ) self.assertEqual([hit.rank for hit in result.hits], [1, 2, 3]) + self.assertEqual([hit.channel_rank for hit in result.hits], [1, 2, 3]) self.assertEqual(result.hits[0].raw_distance, 0.1) + self.assertEqual(result.hits[0].distance_metric, "cosine") + self.assertEqual(result.hits[0].distance_direction, "lower_is_better") self.assertEqual(result.hits[0].score, -0.1) + self.assertEqual(result.hits[0].score_kind, "ordering_only") + self.assertEqual(result.hits[0].score_direction, "higher_is_better") + self.assertEqual(result.hits[0].score_conversion, "negated_distance") self.assertEqual( result.hits[0].metadata, {"text": "fresh bread", "phrase_id": 3}, diff --git a/tests/test_search_fusion.py b/tests/test_search_fusion.py index 622e021..06feac2 100644 --- a/tests/test_search_fusion.py +++ b/tests/test_search_fusion.py @@ -96,9 +96,7 @@ def test_rewritten_atomic_query_identity_changes_fused_identity(self): modality="scene", hits=(hit("scene", 1, 1, 2, "scene:1"),), ) - rewritten = original.model_copy( - update={"query_id": "scene:rewritten"} - ) + rewritten = original.model_copy(update={"query_id": "scene:rewritten"}) arguments = { "query": "Where is the taxi?", "requested_modalities": ("scene",), @@ -109,6 +107,106 @@ def test_rewritten_atomic_query_identity_changes_fused_identity(self): self.assertNotEqual(first.query_id, second.query_id) + def test_search_hit_and_fused_moment_ranking_score_semantics(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + hits=(hit("scene", 1, 1.0, 3.0, "scene:1"),), + ) + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + + self.assertEqual(len(result.moments), 1) + moment = result.moments[0] + + # Combined rank vs channel rank + self.assertEqual(moment.rank, 1) + self.assertEqual(moment.combined_rank, 1) + self.assertEqual(moment.score_kind, "ordering_only") + self.assertEqual(moment.score_direction, "higher_is_better") + self.assertEqual(moment.scoring_method, "reciprocal_rank_fusion") + self.assertEqual(moment.contributing_channels, ("scene",)) + self.assertEqual(moment.channels_run, ("scene",)) + + # Channel hit score and rank metadata + first_hit = moment.hits[0] + self.assertEqual(first_hit.rank, 1) + self.assertEqual(first_hit.channel_rank, 1) + self.assertEqual(first_hit.score_kind, "ordering_only") + self.assertEqual(first_hit.score_direction, "higher_is_better") + self.assertEqual(first_hit.score_conversion, "negated_distance") + self.assertEqual(first_hit.distance_metric, "cosine") + self.assertEqual(first_hit.distance_direction, "lower_is_better") + + # Fusion provenance semantics + self.assertEqual(result.fusion.score_kind, "ordering_only") + self.assertEqual(result.fusion.score_direction, "higher_is_better") + self.assertEqual(result.fusion.scoring_method, "reciprocal_rank_fusion") + self.assertEqual(result.fusion.searched_modalities, ("scene",)) + + def test_multimodal_moment_distinguishes_channel_ranks_from_combined_rank(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + hits=(hit("scene", 3, 1.0, 3.0, "scene:3"),), + ) + speech = SearchResult( + query_id="speech:q", + query="taxi", + modality="speech", + hits=(hit("speech", 1, 2.0, 4.0, "speech:1"),), + ) + result = fuse_search_results( + query="taxi", + requested_modalities=("scene", "speech"), + results=(scene, speech), + ) + + self.assertEqual(len(result.moments), 1) + moment = result.moments[0] + self.assertEqual(moment.combined_rank, 1) + self.assertEqual(moment.contributing_channels, ("scene", "speech")) + self.assertEqual(moment.channels_run, ("scene", "speech")) + + hits_by_modality = {h.modality: h for h in moment.hits} + self.assertEqual(hits_by_modality["scene"].channel_rank, 3) + self.assertEqual(hits_by_modality["speech"].channel_rank, 1) + + def test_score_serialization_clear_for_callers(self): + scene = SearchResult( + query_id="scene:q", + query="taxi", + modality="scene", + hits=(hit("scene", 1, 1.0, 2.0, "scene:1"),), + ) + result = fuse_search_results( + query="taxi", + requested_modalities=("scene",), + results=(scene,), + ) + dumped = result.model_dump(mode="json") + moment_dump = dumped["moments"][0] + + self.assertEqual(moment_dump["combined_rank"], 1) + self.assertEqual(moment_dump["score_kind"], "ordering_only") + self.assertEqual(moment_dump["score_direction"], "higher_is_better") + self.assertEqual(moment_dump["scoring_method"], "reciprocal_rank_fusion") + self.assertEqual(moment_dump["contributing_channels"], ["scene"]) + self.assertEqual(moment_dump["channels_run"], ["scene"]) + + hit_dump = moment_dump["hits"][0] + self.assertEqual(hit_dump["channel_rank"], 1) + self.assertEqual(hit_dump["score_kind"], "ordering_only") + self.assertEqual(hit_dump["score_direction"], "higher_is_better") + self.assertEqual(hit_dump["score_conversion"], "negated_distance") + self.assertEqual(hit_dump["distance_metric"], "cosine") + self.assertEqual(hit_dump["distance_direction"], "lower_is_better") + if __name__ == "__main__": unittest.main()