From 66429927a03ab6902f6a6656447c1852d6f5963a Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Sun, 8 Nov 2026 12:34:43 +0900 Subject: [PATCH 1/4] docs(compat): register copied packet keyframe checks Signed-off-by: Gunwoo Kim --- docs/design/github-shapes-200.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/design/github-shapes-200.md b/docs/design/github-shapes-200.md index 3e3586795..2b270c1c4 100644 --- a/docs/design/github-shapes-200.md +++ b/docs/design/github-shapes-200.md @@ -114,3 +114,9 @@ The implementation plan is: 3. Require exact complete decoded video and audio, all existing stream fields, and identical presented frame timelines, including both copied and encoded audio. Every video and audio frame in every output needs a known presentation timestamp and a positive duration. No alignment, sample tolerance or precision reduction applies. Every other output and copied stream must still pass its original gates. A later miss overrides an earlier compatible output as `different`, including a later unavailable oracle. The original predicate had already reached `different` on that earlier packet mismatch, so no old `different` result becomes `no_oracle`. 4. Report compatible shape counts and repository weights separately. Preserve the exact numerator and every exact share. Replay recomputes the category from bound output bytes, not stored labels. Historical bundles retain their original method binding and cannot be rejudged by substituting these tools. 5. Test only generated byte records, fake executables and independent generated controls before merge. Cover positive removal, unchanged exact classification, changed SPS or PPS, other NAL changes, retained candidate parameter sets, malformed configuration and packets, `avc3`, non-copy streams, timing, decoded samples, mixed outputs, separate accounting and replay. Do not execute judged commands, the judged fixture generator, held-out inputs or the new predicate on retained judged outputs before this registration merges. Any later measurement needs a separate result submission without relabeling frozen exact results. + +## Copied-packet keyframe amendment + +Registered on 2026-11-08 before implementation or judged execution. The maintainer requested keyframe comparison on both the exact and avc1-compatible paths. Equal payloads and timestamps alone do not establish equal random-access signalling. For every copied packet, both paths additionally compare whether the pinned ffprobe packet `flags` contains `K`. A keyframe mismatch is `different`, never exact or compatible credit. Missing keyframe evidence cannot earn credit. Other packet flags and side data remain outside this amendment. + +Commit this amendment first, then add regressions that change only a packet's keyframe flag on each path, including copied audio and later packets. Preserve the existing `github_shapes.method()` SHA-256 binding for the plan and comparator files. No population, fixture, threshold or unrelated judging registration changes. Before merge, test only synthetic records and independent controls, never judged commands, judged fixtures, retained judged outputs or held-out inputs. Any later judged run requires the merged method and a separate result submission without relabeling historical results. From 7b939fe0207e4262c3b9cd3ed39bdf7bfd6dbdfe Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Sun, 8 Nov 2026 12:35:33 +0900 Subject: [PATCH 2/4] test(compat): reject copied packet keyframe changes Signed-off-by: Gunwoo Kim --- .../compat-corpus/tests/test_github_shapes.py | 56 ++++++++++++++++++- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/tools/compat-corpus/tests/test_github_shapes.py b/tools/compat-corpus/tests/test_github_shapes.py index ecd7623f6..038b9b39d 100644 --- a/tools/compat-corpus/tests/test_github_shapes.py +++ b/tools/compat-corpus/tests/test_github_shapes.py @@ -296,14 +296,50 @@ def test_copy_needs_packets_even_when_decoded_output_agrees(self): def test_packet_timing_uses_output_stream_and_rational_clocks(self): doc = {'streams': [{'index': 1, 'time_base': '1/10'}], - 'packets': [{'stream_index': 1, 'data_hash': 'SHA256:test', 'pts': 3, 'dts': 2, 'duration': 1}]} + 'packets': [{'stream_index': 1, 'data_hash': 'SHA256:test', 'pts': 3, 'dts': 2, 'duration': 1, + 'flags': 'K__'}]} with mock.patch.object(decoded_output, 'invoke', return_value=(0, json.dumps(doc).encode(), b'')): - self.assertEqual(run.packet_rows('output', 'probe', {}, ()), [(1, 'SHA256:test', '3/10', '1/5', '1/10')]) + self.assertEqual(run.packet_rows('output', 'probe', {}, ()), [(1, 'SHA256:test', '3/10', '1/5', '1/10', True)]) with tempfile.TemporaryDirectory() as tmp: folder = Path(tmp) (folder / 'command.stderr').write_text('Stream #0:7 -> #1:0 (copy)\nStream #0:1 -> #0:2 (copy)\n') self.assertEqual(run.jobs(folder), {(1, 0): 'copy', (0, 2): 'copy'}) + def test_exact_copy_rejects_each_keyframe_only_difference(self): + command = inputs.instantiate(shape('ffmpeg')) + document = {'streams': [{'index': i, 'time_base': '1/10', 'codec_name': codec} + for i, codec in enumerate(('h264', 'aac'))], + 'packets': [{'stream_index': i, 'data_hash': 'SHA256:control', 'pts': pts, + 'dts': pts, 'duration': 1, 'flags': 'K__'} + for pts in (0, 1) for i in (0, 1)]} + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + for arm in ('reference', 'candidate'): + folder = base / arm + folder.mkdir() + (folder / 'output0.mp4').write_bytes(b'control') + (folder / 'command.stderr').write_text( + 'Stream #0:0 -> #0:0 (copy)\nStream #0:1 -> #0:1 (copy)\n') + candidate = copy.deepcopy(document) + def invoke(argv, env, **kwargs): + source = Path(argv[argv.index('-i') + 1]) + return 0, json.dumps(candidate if source.parent.name == 'candidate' else document).encode(), b'' + with mock.patch.object(decoded_output, 'probe', return_value=document), \ + mock.patch.object(decoded_output, 'compare', return_value=('verified', 'exact')), \ + mock.patch.object(decoded_output, 'invoke', side_effect=invoke): + def result(): + return run.compare_outputs(command, base, {'$FFMPEG': 'ffmpeg', '$FFPROBE': 'ffprobe'}, {}, base) + self.assertEqual(result(), 'counted') + for index in range(len(candidate['packets'])): + for ours, theirs in (('___', 'K__'), ('K__', '___')): + with self.subTest(packet=index, ours=ours, theirs=theirs): + candidate['packets'][index]['flags'] = ours + document['packets'][index]['flags'] = theirs + self.assertEqual(result(), 'different') + candidate['packets'][index]['flags'] = document['packets'][index]['flags'] = 'K__' + candidate['packets'][0]['flags'] = 'KD_' + self.assertEqual(result(), 'counted') + class Avc1Carriage(unittest.TestCase): SPS, PPS = b'\x67\x42\x00\x0a', b'\x68\x12' @@ -321,7 +357,7 @@ def sample(*units, length=4): def packet(self, data, index=0, pts=0): return {'stream_index': index, 'pts': pts, 'dts': pts, 'duration': 1, 'size': str(len(data)), - 'data': self.dump(data), 'data_hash': 'SHA256:' + hashlib.sha256(data).hexdigest()} + 'data': self.dump(data), 'data_hash': 'SHA256:' + hashlib.sha256(data).hexdigest(), 'flags': 'K__'} def configuration(self, data, document): stream = document['streams'][0] @@ -442,6 +478,20 @@ def test_timing_packet_order_counts_and_copied_audio_bytes_stay_exact(self): self.actual['streams'][0]['duration_ts'] = 2 self.assertFalse(self.compatible()) + def test_compatible_copy_rejects_each_keyframe_only_difference(self): + for document in (self.actual, self.reference): + document['packets'].extend([self.packet(self.sample(self.SLICE), pts=1), self.packet(b'audio', 1, pts=1)]) + self.assertTrue(self.compatible()) + for index in range(len(self.actual['packets'])): + for ours, theirs in (('___', 'K__'), ('K__', '___')): + with self.subTest(packet=index, ours=ours, theirs=theirs): + self.actual['packets'][index]['flags'] = ours + self.reference['packets'][index]['flags'] = theirs + self.assertFalse(self.compatible()) + self.actual['packets'][index]['flags'] = self.reference['packets'][index]['flags'] = 'K__' + self.actual['packets'][0]['flags'] = 'KD_' + self.assertTrue(self.compatible()) + def test_decoded_pixels_samples_and_all_audio_frame_timelines_stay_exact(self): for attribute in ('audio_shift', 'changed_pixels', 'changed_audio'): setattr(self, attribute, True) From 936cb9f6aa56b83119775b9568ac7ac060cb3418 Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Sun, 8 Nov 2026 12:38:03 +0900 Subject: [PATCH 3/4] fix(compat): compare copied packet keyframe flags Signed-off-by: Gunwoo Kim --- docs/agents/compat-corpus.md | 1 + tools/compat-corpus/decoded_output.py | 2 +- tools/compat-corpus/github_shapes.py | 2 +- tools/compat-corpus/tests/test_github_shapes.py | 14 ++++++++++++++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/agents/compat-corpus.md b/docs/agents/compat-corpus.md index 38d6a0fb2..fef21faab 100644 --- a/docs/agents/compat-corpus.md +++ b/docs/agents/compat-corpus.md @@ -4,6 +4,7 @@ Corpus exclusions, compatibility and codec-coverage workflows with their candida | Rule | Enforced by | |---|---| +| Both exact and avc1-compatible top-200 copy comparisons preserve every packet's keyframe flag | `packet_rows` and `avc1_carriage` include the `K` flag in their compared rows. The keyframe-only regressions in `test_github_shapes.py` reject both mismatch directions for first and later video and audio packets, while other flag bits stay outside this predicate. External probe records require runtime comparison, not a declared stream type | | HEVC correctness shards retain the complete original roster and cannot count a partial, reordered or substituted part as complete | `corpus_shard` owns the deterministic assignment. `test_fixed_corpus_shards_cover_each_original_identity_once` proves the 54-file union without duplicates; `test_shard_completion_requires_its_exact_members_and_every_eof_arm` rejects missing, duplicate, reordered and changed identities and missing EOF arms. The workflow test requires all four non-fail-fast matrix entries and distinct artifact names | | Metadata-only HEVC acquisition does not apply the complete-decoder acceptance predicate | The existing `packet_native` mode selects that predicate only for C2. `test_online_census_releases_each_owned_media_file_and_keeps_receipts` preserves metadata-only cleanup and false full-coverage flags on its two-file fixture; the complete-roster/EOF tests retain the 54-file C2 denominator. The command boundary owns runtime mode selection | | HEVC C2 keeps container follow-ups separate from completed pixel comparisons and never treats a decoder, clock or pixel failure as one | Closed `PACKET_TOOLING_REFUSALS`, zero native output before classification, and `test_container_follow_ups_preserve_all_identities_without_pixel_credit`; the regression preserves all 54 identities and rejects missing EOF arms, decoder errors, clock errors and pixel mismatches. Runtime tool results require boundary validation | diff --git a/tools/compat-corpus/decoded_output.py b/tools/compat-corpus/decoded_output.py index 7e879aaa8..463171672 100644 --- a/tools/compat-corpus/decoded_output.py +++ b/tools/compat-corpus/decoded_output.py @@ -421,7 +421,7 @@ def avc1_carriage(mine, reference, ffmpeg, ffprobe, env, scratch, copied_streams timing = tuple(time(clock, key) for key in ('pts', 'dts', 'duration')) if any(t is None for t in timing) or timing[2] <= 0: return False - packets.append((index, payload, *timing)) + packets.append((index, payload, *timing, 'K' in packet['flags'])) if seen != set(copied_streams): return False rows.append(packets) diff --git a/tools/compat-corpus/github_shapes.py b/tools/compat-corpus/github_shapes.py index 299dbbb8a..1d3d29f68 100644 --- a/tools/compat-corpus/github_shapes.py +++ b/tools/compat-corpus/github_shapes.py @@ -221,7 +221,7 @@ def packet_rows(path, ffprobe, env, raw): clocks = {s['index']: fractions.Fraction(s['time_base']) for s in doc['streams']} return [(p['stream_index'], p['data_hash'], *[ str(int(p[key]) * clocks[p['stream_index']]) if p.get(key) not in (None, 'N/A') else None - for key in ('pts', 'dts', 'duration')]) for p in doc['packets']] + for key in ('pts', 'dts', 'duration')], 'K' in p['flags']) for p in doc['packets']] def jobs(reference): diff --git a/tools/compat-corpus/tests/test_github_shapes.py b/tools/compat-corpus/tests/test_github_shapes.py index 038b9b39d..9ecb059e0 100644 --- a/tools/compat-corpus/tests/test_github_shapes.py +++ b/tools/compat-corpus/tests/test_github_shapes.py @@ -305,6 +305,14 @@ def test_packet_timing_uses_output_stream_and_rational_clocks(self): (folder / 'command.stderr').write_text('Stream #0:7 -> #1:0 (copy)\nStream #0:1 -> #0:2 (copy)\n') self.assertEqual(run.jobs(folder), {(1, 0): 'copy', (0, 2): 'copy'}) + def test_missing_packet_flags_cannot_supply_exact_keyframe_evidence(self): + document = {'streams': [{'index': 0, 'time_base': '1/10'}], + 'packets': [{'stream_index': 0, 'data_hash': 'SHA256:control', 'pts': 0, + 'dts': 0, 'duration': 1}]} + with mock.patch.object(decoded_output, 'invoke', return_value=(0, json.dumps(document).encode(), b'')): + with self.assertRaises(KeyError): + run.packet_rows('output', 'probe', {}, ()) + def test_exact_copy_rejects_each_keyframe_only_difference(self): command = inputs.instantiate(shape('ffmpeg')) document = {'streams': [{'index': i, 'time_base': '1/10', 'codec_name': codec} @@ -492,6 +500,12 @@ def test_compatible_copy_rejects_each_keyframe_only_difference(self): self.actual['packets'][0]['flags'] = 'KD_' self.assertTrue(self.compatible()) + def test_missing_packet_flags_cannot_supply_compatible_keyframe_evidence(self): + for document in (self.actual, self.reference): + for packet in document['packets']: + del packet['flags'] + self.assertFalse(self.compatible()) + def test_decoded_pixels_samples_and_all_audio_frame_timelines_stay_exact(self): for attribute in ('audio_shift', 'changed_pixels', 'changed_audio'): setattr(self, attribute, True) From 313b2f0acd10f78ba19f5f70003c587fee6cf6f9 Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Sun, 8 Nov 2026 13:01:13 +0900 Subject: [PATCH 4/4] test(compat): prove exact keyframe rejection independently Signed-off-by: Gunwoo Kim --- docs/agents/compat-corpus.md | 2 +- .../compat-corpus/tests/test_github_shapes.py | 21 ++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/agents/compat-corpus.md b/docs/agents/compat-corpus.md index fef21faab..32eb89e99 100644 --- a/docs/agents/compat-corpus.md +++ b/docs/agents/compat-corpus.md @@ -4,7 +4,7 @@ Corpus exclusions, compatibility and codec-coverage workflows with their candida | Rule | Enforced by | |---|---| -| Both exact and avc1-compatible top-200 copy comparisons preserve every packet's keyframe flag | `packet_rows` and `avc1_carriage` include the `K` flag in their compared rows. The keyframe-only regressions in `test_github_shapes.py` reject both mismatch directions for first and later video and audio packets, while other flag bits stay outside this predicate. External probe records require runtime comparison, not a declared stream type | +| Both exact and avc1-compatible top-200 copy comparisons preserve every packet's keyframe flag | `packet_rows` and `avc1_carriage` include the `K` flag in their compared rows. The keyframe-only regressions in `test_github_shapes.py` use complete packet records and assert the exact-path documents differ only in the selected flag. They reject both mismatch directions for first and later video and audio packets, while other flag bits stay outside this predicate. External probe records require runtime comparison, not a declared stream type | | HEVC correctness shards retain the complete original roster and cannot count a partial, reordered or substituted part as complete | `corpus_shard` owns the deterministic assignment. `test_fixed_corpus_shards_cover_each_original_identity_once` proves the 54-file union without duplicates; `test_shard_completion_requires_its_exact_members_and_every_eof_arm` rejects missing, duplicate, reordered and changed identities and missing EOF arms. The workflow test requires all four non-fail-fast matrix entries and distinct artifact names | | Metadata-only HEVC acquisition does not apply the complete-decoder acceptance predicate | The existing `packet_native` mode selects that predicate only for C2. `test_online_census_releases_each_owned_media_file_and_keeps_receipts` preserves metadata-only cleanup and false full-coverage flags on its two-file fixture; the complete-roster/EOF tests retain the 54-file C2 denominator. The command boundary owns runtime mode selection | | HEVC C2 keeps container follow-ups separate from completed pixel comparisons and never treats a decoder, clock or pixel failure as one | Closed `PACKET_TOOLING_REFUSALS`, zero native output before classification, and `test_container_follow_ups_preserve_all_identities_without_pixel_credit`; the regression preserves all 54 identities and rejects missing EOF arms, decoder errors, clock errors and pixel mismatches. Runtime tool results require boundary validation | diff --git a/tools/compat-corpus/tests/test_github_shapes.py b/tools/compat-corpus/tests/test_github_shapes.py index 9ecb059e0..34b6d7d6e 100644 --- a/tools/compat-corpus/tests/test_github_shapes.py +++ b/tools/compat-corpus/tests/test_github_shapes.py @@ -315,11 +315,19 @@ def test_missing_packet_flags_cannot_supply_exact_keyframe_evidence(self): def test_exact_copy_rejects_each_keyframe_only_difference(self): command = inputs.instantiate(shape('ffmpeg')) - document = {'streams': [{'index': i, 'time_base': '1/10', 'codec_name': codec} - for i, codec in enumerate(('h264', 'aac'))], - 'packets': [{'stream_index': i, 'data_hash': 'SHA256:control', 'pts': pts, - 'dts': pts, 'duration': 1, 'flags': 'K__'} - for pts in (0, 1) for i in (0, 1)]} + sps, pps = Avc1Carriage.SPS, Avc1Carriage.PPS + config = b'\x01\x42\x00\x0a\xff\xe1' + len(sps).to_bytes(2, 'big') + sps + ( + b'\x01' + len(pps).to_bytes(2, 'big') + pps) + video = {'index': 0, 'time_base': '1/10', 'codec_type': 'video', 'codec_name': 'h264', + 'codec_tag_string': 'avc1', 'extradata': Avc1Carriage.dump(config), 'extradata_size': len(config), + 'extradata_hash': 'SHA256:' + hashlib.sha256(config).hexdigest()} + document = {'streams': [video, {'index': 1, 'time_base': '1/10', 'codec_type': 'audio', 'codec_name': 'aac'}], + 'packets': []} + for pts in (0, 1): + for index, payload in enumerate((Avc1Carriage.sample(Avc1Carriage.SLICE), b'audio')): + document['packets'].append({'stream_index': index, 'pts': pts, 'dts': pts, 'duration': 1, + 'flags': 'K__', 'size': str(len(payload)), 'data': Avc1Carriage.dump(payload), + 'data_hash': 'SHA256:' + hashlib.sha256(payload).hexdigest()}) with tempfile.TemporaryDirectory() as tmp: base = Path(tmp) for arm in ('reference', 'candidate'): @@ -343,6 +351,9 @@ def result(): with self.subTest(packet=index, ours=ours, theirs=theirs): candidate['packets'][index]['flags'] = ours document['packets'][index]['flags'] = theirs + normalized = copy.deepcopy(candidate) + normalized['packets'][index]['flags'] = theirs + self.assertEqual(normalized, document) self.assertEqual(result(), 'different') candidate['packets'][index]['flags'] = document['packets'][index]['flags'] = 'K__' candidate['packets'][0]['flags'] = 'KD_'