From 0ff37d2321b15e9a74234c59aa4e37cf0f5a4d03 Mon Sep 17 00:00:00 2001 From: ik020 Date: Sat, 5 Sep 2026 15:05:12 +0500 Subject: [PATCH] Persist real face embeddings during actor indexing (#74 prep) Actor indexing computed a normalized face encoding per detection but never stored it: _actor_records() wrote a placeholder embedding=[0.0] into every StorageRecord, so the actor Chroma collection's vector index has never contained anything searchable. Matching only ever happened in-memory, within a single indexing run, via ActorIndexState.known_encodings, and was discarded once that run finished. This blocks issue #74 (find people using a reference image), which requires comparing a reference image's embedding against previously indexed faces across the whole repository. Fix: - Carry the computed encoding through into each detection dict. - _actor_records() now writes the real normalized encoding as the StorageRecord embedding instead of the placeholder. - Cluster-summary records (_actor_cluster_records) are left unchanged: a summary rolls up multiple detections and has no single face image to encode. Because this changes what's actually stored per detection, any existing generation with the actor modality enabled has placeholder vectors and is no longer valid. Bump INDEX_SCHEMA_VERSION 7 -> 8 so CompletedGenerationManifest's Literal[INDEX_SCHEMA_VERSION] check rejects old generations with a clear IndexSchemaError instead of silently treating placeholder-vector indexes as complete and searchable. Affected generations need to be re-indexed. Adds a new end-to-end test driving process_actor_samples through mocked detector/recognizer calls, asserting the stored embedding is the real normalized encoding rather than [0.0]. Updates the existing _actor_records test, which built detection dicts without the now-required encoding key. --- src/vidxp/capabilities/actor/indexing.py | 3 +- src/vidxp/core/contracts.py | 2 +- tests/test_indexing.py | 68 ++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/vidxp/capabilities/actor/indexing.py b/src/vidxp/capabilities/actor/indexing.py index 35da06c2..483694b9 100644 --- a/src/vidxp/capabilities/actor/indexing.py +++ b/src/vidxp/capabilities/actor/indexing.py @@ -74,7 +74,7 @@ def _actor_records( records.append( StorageRecord( source_id=source_id, - embedding=[0.0], + embedding=detection["encoding"], metadata={ **config.record_identity("actor", source_id), "detection_id": detection["detection_id"], @@ -209,6 +209,7 @@ def process_actor_samples( min(height, int(face[1] + face[3])), max(0, int(face[0])), ), + "encoding": encoding.tolist(), } ) state.processed_frames += 1 diff --git a/src/vidxp/core/contracts.py b/src/vidxp/core/contracts.py index c58d632f..34162bb8 100644 --- a/src/vidxp/core/contracts.py +++ b/src/vidxp/core/contracts.py @@ -11,7 +11,7 @@ from urllib.parse import quote -INDEX_SCHEMA_VERSION = 7 +INDEX_SCHEMA_VERSION = 8 MANIFEST_SCHEMA_VERSION = 2 diff --git a/tests/test_indexing.py b/tests/test_indexing.py index b60b2fb4..d72b53b8 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -444,6 +444,7 @@ def test_actor_records_preserve_stable_detection_metadata(self): "frame_index": 0, "timestamp": 0.0, "bbox": (1, 2, 3, 0), + "encoding": [0.6, 0.8], } ], config, @@ -455,6 +456,73 @@ def test_actor_records_preserve_stable_detection_metadata(self): "generation-1:actors:video-1:actor-cluster:1", ) self.assertEqual(records[0].metadata["bbox_top"], 1) + self.assertEqual(records[0].embedding, [0.6, 0.8]) + + def test_actor_indexing_persists_real_face_embeddings_not_placeholders(self): + import numpy as np + from unittest.mock import Mock + + from vidxp.capabilities.actor.indexing import ( + ActorIndexState, + process_actor_samples, + ) + from vidxp.core.video import FrameSample + + config = IndexConfig( + dataset="sample", + split="test", + run_id="actors", + video_id="video-1", + generation_id="generation-1", + enabled_modalities=("actor",), + ) + + raw_encoding = np.array([3.0, 4.0], dtype="float32") + expected_normalized = (raw_encoding / np.linalg.norm(raw_encoding)).tolist() + + detector = Mock() + detector.setInputSize = Mock() + detector.setScoreThreshold = Mock() + detector.detect.return_value = ( + True, + np.array([[10.0, 10.0, 20.0, 20.0]], dtype="float32"), + ) + + recognizer = Mock() + recognizer.alignCrop.return_value = np.zeros((112, 112, 3), dtype="uint8") + recognizer.feature.return_value = raw_encoding.reshape(1, -1) + + state = ActorIndexState( + models=Mock(detector=detector, recognizer=recognizer) + ) + storage = Mock() + + samples = [ + FrameSample( + frame_index=0, + timestamp=0.0, + frame=np.zeros((48, 48, 3), dtype="uint8"), + ), + ] + + process_actor_samples( + samples, + state=state, + config=config, + storage=storage, + cancellation=CancellationToken(), + ) + + self.assertEqual(storage.upsert.call_count, 1) + (_, records), _ = storage.upsert.call_args + self.assertEqual(len(records), 1) + record = records[0] + + self.assertIsNotNone(record.embedding) + self.assertNotEqual(list(record.embedding), [0.0]) + self.assertEqual(len(record.embedding), 2) + for actual, expected in zip(record.embedding, expected_normalized): + self.assertAlmostEqual(actual, expected, places=5) def test_actor_cluster_identity_is_unique_by_media_and_generation(self): def config(video_id, generation_id):