diff --git a/.github/workflows/hevc-conformance.yml b/.github/workflows/hevc-conformance.yml index daccd2003..164185d47 100644 --- a/.github/workflows/hevc-conformance.yml +++ b/.github/workflows/hevc-conformance.yml @@ -70,7 +70,20 @@ jobs: toolchain: stable - run: cargo test --locked -p coreav-hevc -p coreav-kernels-hevc -p coreav-backend-hevc --no-default-features - run: cargo test --locked -p coreav-hevc -p coreav-kernels-hevc -p coreav-backend-hevc -p coreav-thumbnail --features native-hevc --all-targets + - name: Exercise the real native reference driver on generated pixels + run: | + set -euo pipefail + cargo test --locked -p coreav-backend-hevc --features native-hevc,test-support --example reference | tee "$RUNNER_TEMP/native-reference-tests.log" + grep -F 'test tests::native_reference_checks_generated_depths_at_every_worker_boundary ... ok' "$RUNNER_TEMP/native-reference-tests.log" - run: cargo test --locked -p coreav-hevc -p coreav-kernels-hevc --features native-hevc,scalar --all-targets + - name: Check diagnostic-only extraction and row reads + if: runner.os == 'Linux' + run: | + set -euo pipefail + RUSTFLAGS='--cfg hevc_thread_diagnostics' cargo test --locked -p coreav-hevc -p coreav-kernels-hevc --features native-hevc --lib | tee "$RUNNER_TEMP/hevc-diagnostic-tests.log" + grep -Fx 'test decoder::scheduler::tests::header_only_dependency_replay_distinguishes_chains_and_siblings ... ok' "$RUNNER_TEMP/hevc-diagnostic-tests.log" + grep -Fx 'test decoder::scheduler::tests::row_dependency_trace_observes_generated_predictions ... ok' "$RUNNER_TEMP/hevc-diagnostic-tests.log" + grep -Fx 'test inter::tests::diagnostic_read_bounds_match_interpolation_support ... ok' "$RUNNER_TEMP/hevc-diagnostic-tests.log" - run: cargo clippy --locked -p coreav-hevc -p coreav-kernels-hevc -p coreav-backend-hevc -p coreav-thumbnail --features native-hevc --all-targets -- -D warnings - name: Check the opt-in CLI registration with and without default features run: | @@ -86,6 +99,10 @@ jobs: if grep -E 'coreav-(kernels-)?hevc' "$RUNNER_TEMP/backend-off.txt"; then exit 1; fi cargo tree --locked -p coreav-cli -e normal,build > "$RUNNER_TEMP/default-consumer.txt" if grep -E 'coreav-(kernels-|backend-)?hevc' "$RUNNER_TEMP/default-consumer.txt"; then exit 1; fi + for feature in release-default release-full; do + cargo tree --locked -p coreav-cli --no-default-features --features "$feature" -e normal,build,features > "$RUNNER_TEMP/$feature.txt" + if grep -E 'native-hevc-private|coreav-(kernels-|backend-)?hevc' "$RUNNER_TEMP/$feature.txt"; then exit 1; fi + done - name: Check Linux acquisition and subprocess isolation if: runner.os == 'Linux' run: python3 -m unittest discover -s scripts/tests -p 'test_hevc*.py' diff --git a/.github/workflows/hevc-native-rows.yml b/.github/workflows/hevc-native-rows.yml new file mode 100644 index 000000000..3f0750ac2 --- /dev/null +++ b/.github/workflows/hevc-native-rows.yml @@ -0,0 +1,136 @@ +name: hevc-native-rows +run-name: Native HEVC rows ${{ inputs.candidate }} (${{ inputs.mode }}) +# workflow-check: not required + +on: + workflow_call: + inputs: + candidate: + required: true + type: string + mode: + type: string + default: all + only: + type: string + default: '' + frames: + type: number + default: 32 + workflow_dispatch: + inputs: + candidate: + description: Exact decoder commit + required: true + type: string + mode: + description: Native reference calls, complete checks, or both + type: choice + options: [all, reference, checks] + default: all + only: + description: Optional comma-separated non-wavefront input IDs for reference calls + type: string + default: '' + frames: + description: Maximum pictures per reference input + type: number + default: 32 + +permissions: + contents: read + +env: + CANDIDATE: ${{ inputs.candidate }} + ONLY: ${{ inputs.only }} + FRAMES: ${{ inputs.frames }} + +jobs: + reference: + if: inputs.mode == 'all' || inputs.mode == 'reference' + name: hevc-native-rows-reference (${{ matrix.shard }}) + runs-on: ubuntu-24.04 + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON(inputs.only != '' && '[0]' || '[0,1,2]') }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + ref: ${{ inputs.candidate }} + fetch-depth: 0 + - name: Admit exact candidate and bounded reference selection + run: | + set -euo pipefail + [[ "$CANDIDATE" =~ ^[0-9a-f]{40}$ ]] + test "$(git rev-parse HEAD)" = "$CANDIDATE" + test -n "$(git branch -r --contains "$CANDIDATE")" + python3 -I - "$ONLY" "$FRAMES" <<'PY' + import sys + sys.path.insert(0, 'scripts') + from hevc_threading import ROW_INPUT_IDS + selected = set(sys.argv[1].split(',')) if sys.argv[1] else set(ROW_INPUT_IDS) + if not selected or not selected <= set(ROW_INPUT_IDS) or not 1 <= int(sys.argv[2]) <= 32: + raise SystemExit('invalid_native_rows_selection') + PY + - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 + with: + toolchain: stable + - name: Check the actual native driver before release building + run: | + set -euo pipefail + cargo test --locked -p coreav-hevc --features native-hevc-rows-private,test-support --lib decoder::scheduler::tests + cargo test --locked -p coreav-backend-hevc --features native-hevc,test-support --example reference + cargo build --locked --release -p coreav-backend-hevc --features native-hevc,test-support --example reference + - name: Install the pinned binary oracle + env: + R2_ENDPOINT: ${{ secrets.COREAV_CORPUS_R2_ENDPOINT }} + R2_ACCESS_KEY_ID: ${{ secrets.COREAV_CORPUS_R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.COREAV_CORPUS_R2_SECRET_ACCESS_KEY }} + R2_BUCKET: ${{ vars.COREAV_CORPUS_R2_BUCKET }} + run: scripts/install-oracle-ffmpeg.sh "$RUNNER_TEMP/oracle" + - name: Verify every worker count before paired native reference calls + env: + R2_ENDPOINT: ${{ secrets.COREAV_CORPUS_R2_ENDPOINT }} + R2_ACCESS_KEY_ID: ${{ secrets.COREAV_CORPUS_R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.COREAV_CORPUS_R2_SECRET_ACCESS_KEY }} + R2_BUCKET: ${{ vars.COREAV_CORPUS_R2_BUCKET }} + run: | + set -euo pipefail + umask 077 + mkdir -p "$RUNNER_TEMP/private" + python3 -I scripts/hevc_threading.py native-reference \ + --native "$PWD/target/release/examples/reference" --oracle-bin "$RUNNER_TEMP/oracle" \ + --out "$RUNNER_TEMP/private/native" --candidate "$CANDIDATE" \ + --shard-index '${{ matrix.shard }}' --shard-count '${{ inputs.only != '' && 1 || 3 }}' \ + --only "$ONLY" --frames "$FRAMES" > "$RUNNER_TEMP/private/native.log" 2>&1 + - name: Retain private reference results + if: always() + run: | + set -euo pipefail + cd "$RUNNER_TEMP" + [ -d private ] || exit 0 + tar -czf records.tgz private + mkdir -p sealed + openssl rand -hex -out key.txt 32 + openssl enc -aes-256-cbc -pbkdf2 -salt -pass file:key.txt -in records.tgz -out sealed/records.tgz.enc + openssl pkeyutl -encrypt -pubin -inkey "$GITHUB_WORKSPACE/bench/hevc-conformance/evidence-public.pem" -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 -in key.txt -out sealed/key.txt.enc + shred -u key.txt records.tgz + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: hevc-native-rows-reference-${{ matrix.shard }}-${{ inputs.candidate }} + path: ${{ runner.temp }}/sealed + if-no-files-found: error + retention-days: 7 + + checks: + if: inputs.mode == 'all' || inputs.mode == 'checks' + uses: ./.github/workflows/hevc-threading.yml + with: + candidate: ${{ inputs.candidate }} + baseline: '' + frames: 32 + secrets: inherit diff --git a/.github/workflows/hevc-reference.yml b/.github/workflows/hevc-reference.yml index 57fe1f049..4cb47885b 100644 --- a/.github/workflows/hevc-reference.yml +++ b/.github/workflows/hevc-reference.yml @@ -15,7 +15,7 @@ on: required: true type: string baseline: - description: Optional full baseline SHA for a comparison + description: Optional full baseline SHA, or native for same-head row evidence required: false default: '' type: string @@ -24,6 +24,21 @@ on: required: false default: false type: boolean + thread_diagnostics_only: + description: Run only untimed 4T picture-job diagnostics on all 30 inputs + required: false + default: false + type: boolean + thread_dependencies_only: + description: Replay only header reference dependencies on all 30 inputs + required: false + default: false + type: boolean + thread_rows_only: + description: Decode only CTU-row reference reads on the 24 non-WPP inputs + required: false + default: false + type: boolean only: description: Optional comma-separated input IDs required: false @@ -54,7 +69,7 @@ jobs: secrets: inherit reference: - if: ${{ !inputs.inverse && inputs.baseline == '' && !inputs.sanitizer_only }} + if: ${{ !inputs.inverse && inputs.baseline == '' && !inputs.sanitizer_only && !inputs.thread_diagnostics_only && !inputs.thread_dependencies_only && !inputs.thread_rows_only }} runs-on: ubuntu-24.04 timeout-minutes: 150 env: @@ -128,12 +143,25 @@ jobs: retention-days: 30 threading: - if: ${{ !inputs.inverse && (inputs.baseline != '' || inputs.sanitizer_only) }} + if: ${{ !inputs.inverse && inputs.baseline != 'native' && (inputs.baseline != '' || inputs.sanitizer_only || inputs.thread_diagnostics_only || inputs.thread_dependencies_only || inputs.thread_rows_only) }} uses: ./.github/workflows/hevc-threading.yml with: candidate: ${{ inputs.candidate }} baseline: ${{ inputs.baseline }} sanitizer_only: ${{ inputs.sanitizer_only }} + thread_diagnostics_only: ${{ inputs.thread_diagnostics_only }} + thread_dependencies_only: ${{ inputs.thread_dependencies_only }} + thread_rows_only: ${{ inputs.thread_rows_only }} + only: ${{ inputs.only }} + frames: ${{ fromJSON(inputs.frames) }} + secrets: inherit + + native-rows: + if: ${{ !inputs.inverse && inputs.baseline == 'native' }} + uses: ./.github/workflows/hevc-native-rows.yml + with: + candidate: ${{ inputs.candidate }} + mode: ${{ inputs.only != '' && 'reference' || inputs.diagnose_only && 'checks' || 'all' }} only: ${{ inputs.only }} frames: ${{ fromJSON(inputs.frames) }} secrets: inherit diff --git a/.github/workflows/hevc-threading.yml b/.github/workflows/hevc-threading.yml index db7b0b910..4aa750b76 100644 --- a/.github/workflows/hevc-threading.yml +++ b/.github/workflows/hevc-threading.yml @@ -14,6 +14,18 @@ on: required: false default: false type: boolean + thread_diagnostics_only: + required: false + default: false + type: boolean + thread_dependencies_only: + required: false + default: false + type: boolean + thread_rows_only: + required: false + default: false + type: boolean only: required: false default: '' @@ -33,8 +45,106 @@ env: FRAMES: ${{ inputs.frames }} jobs: + diagnostics: + if: inputs.thread_diagnostics_only || inputs.thread_dependencies_only || inputs.thread_rows_only + name: hevc-threading-diagnostics + runs-on: ubuntu-24.04 + timeout-minutes: 180 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + ref: ${{ inputs.candidate }} + fetch-depth: 0 + path: candidate + - name: Admit the exact diagnostic candidate before building or using credentials + working-directory: candidate + env: + DEPENDENCIES_ONLY: ${{ inputs.thread_dependencies_only }} + ROWS_ONLY: ${{ inputs.thread_rows_only }} + run: | + set -euo pipefail + [[ "$CANDIDATE" =~ ^[0-9a-f]{40}$ ]] + test "$(git rev-parse HEAD)" = "$CANDIDATE" + test -n "$(git branch -r --contains "$CANDIDATE")" + test -z "$ONLY" + test "$FRAMES" = 32 + test "$DEPENDENCIES_ONLY" != true || test "$ROWS_ONLY" != true + - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 + with: + toolchain: stable + - name: Check diagnostic validation and workflow selection + run: | + python3 -m unittest discover -s candidate/scripts/tests -p test_hevc_threading.py + python3 -m unittest discover -s candidate/scripts/tests -p test_hevc_dependency.py + python3 -m unittest discover -s candidate/scripts/tests -p test_hevc_row_dependency.py + - name: Check cfg-only dependency extraction and scheduling tests + working-directory: candidate + run: | + set -euo pipefail + RUSTFLAGS='--cfg hevc_thread_diagnostics' cargo test --locked -p coreav-hevc --features native-hevc --lib | tee "$RUNNER_TEMP/diagnostic-tests.log" + grep -Eq '^test result: ok\. [1-9][0-9]* passed; 0 failed;' "$RUNNER_TEMP/diagnostic-tests.log" + grep -Fx 'test decoder::scheduler::tests::header_only_dependency_replay_distinguishes_chains_and_siblings ... ok' "$RUNNER_TEMP/diagnostic-tests.log" + grep -Fx 'test decoder::scheduler::tests::row_dependency_trace_observes_generated_predictions ... ok' "$RUNNER_TEMP/diagnostic-tests.log" + RUSTFLAGS='--cfg hevc_thread_diagnostics' cargo test --locked -p coreav-kernels-hevc --features native-hevc --lib | tee "$RUNNER_TEMP/kernel-diagnostic-tests.log" + grep -Eq '^test result: ok\. [1-9][0-9]* passed; 0 failed;' "$RUNNER_TEMP/kernel-diagnostic-tests.log" + grep -Fx 'test inter::tests::diagnostic_read_bounds_match_interpolation_support ... ok' "$RUNNER_TEMP/kernel-diagnostic-tests.log" + - name: Build only the candidate diagnostic executable + working-directory: candidate + run: | + set -euo pipefail + cp bench/hevc-reference/decode.rs crates/coreav-hevc/examples/reference.rs + RUSTFLAGS='--cfg hevc_thread_diagnostics --check-cfg=cfg(hevc_thread_diagnostics) --check-cfg=cfg(hevc_synchronous_baseline)' cargo build --locked --release -p coreav-hevc --features native-hevc --example reference + umask 077 + mkdir -p "$RUNNER_TEMP/private" + rustc -Vv > "$RUNNER_TEMP/private/compiler.txt" + - name: Install the pinned binary oracle + env: + R2_ENDPOINT: ${{ secrets.COREAV_CORPUS_R2_ENDPOINT }} + R2_ACCESS_KEY_ID: ${{ secrets.COREAV_CORPUS_R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.COREAV_CORPUS_R2_SECRET_ACCESS_KEY }} + R2_BUCKET: ${{ vars.COREAV_CORPUS_R2_BUCKET }} + run: candidate/scripts/install-oracle-ffmpeg.sh "$RUNNER_TEMP/oracle" + - name: Inspect each prepared input once + env: + DEPENDENCIES_ONLY: ${{ inputs.thread_dependencies_only }} + ROWS_ONLY: ${{ inputs.thread_rows_only }} + R2_ENDPOINT: ${{ secrets.COREAV_CORPUS_R2_ENDPOINT }} + R2_ACCESS_KEY_ID: ${{ secrets.COREAV_CORPUS_R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.COREAV_CORPUS_R2_SECRET_ACCESS_KEY }} + R2_BUCKET: ${{ vars.COREAV_CORPUS_R2_BUCKET }} + run: | + set -euo pipefail + umask 077 + EXTRA=() + if [ "$DEPENDENCIES_ONLY" = true ]; then EXTRA+=(--dependencies-only); fi + if [ "$ROWS_ONLY" = true ]; then EXTRA+=(--rows-only); fi + python3 -I candidate/scripts/hevc_threading.py diagnostics \ + --native "$PWD/candidate/target/release/examples/reference" \ + --oracle-bin "$RUNNER_TEMP/oracle" --out "$RUNNER_TEMP/private" \ + --candidate "$CANDIDATE" "${EXTRA[@]}" > "$RUNNER_TEMP/private/diagnostics.log" 2>&1 + - name: Seal diagnostic metadata + if: always() + run: | + set -euo pipefail + cd "$RUNNER_TEMP" + [ -d private ] || exit 0 + tar -czf records.tgz private + mkdir -p sealed + openssl rand -hex -out key.txt 32 + openssl enc -aes-256-cbc -pbkdf2 -salt -pass file:key.txt -in records.tgz -out sealed/records.tgz.enc + openssl pkeyutl -encrypt -pubin -inkey "$GITHUB_WORKSPACE/candidate/bench/hevc-conformance/evidence-public.pem" -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 -in key.txt -out sealed/key.txt.enc + shred -u key.txt records.tgz + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: hevc-threading-diagnostics-${{ inputs.candidate }} + path: ${{ runner.temp }}/sealed + if-no-files-found: error + retention-days: 7 + reference: - if: '!inputs.sanitizer_only' + if: ${{ inputs.baseline != '' && !inputs.sanitizer_only && !inputs.thread_diagnostics_only && !inputs.thread_dependencies_only && !inputs.thread_rows_only }} name: hevc-threading-reference (${{ matrix.shard }}) runs-on: ubuntu-24.04 timeout-minutes: 180 @@ -61,17 +171,18 @@ jobs: with: persist-credentials: false ref: ${{ inputs.baseline }} + fetch-depth: 0 path: baseline - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 with: toolchain: stable - - name: Build distinct candidate and serial-main executables + - name: Build distinct candidate and current-main executables run: | set -euo pipefail cp candidate/bench/hevc-reference/decode.rs candidate/crates/coreav-hevc/examples/reference.rs cp candidate/bench/hevc-reference/decode.rs baseline/crates/coreav-hevc/examples/reference.rs - (cd candidate && RUSTFLAGS='--check-cfg=cfg(hevc_serial_baseline)' cargo build --locked --release -p coreav-hevc --features native-hevc --example reference --example headers) - (cd baseline && test "$(git rev-parse HEAD)" = "$BASELINE" && RUSTFLAGS='--cfg hevc_serial_baseline --check-cfg=cfg(hevc_serial_baseline)' cargo build --locked --release -p coreav-hevc --features native-hevc --example reference) + (cd candidate && RUSTFLAGS='--check-cfg=cfg(hevc_synchronous_baseline) --check-cfg=cfg(hevc_thread_diagnostics)' cargo build --locked --release -p coreav-hevc --features native-hevc --example reference --example headers) + (cd baseline && test "$(git rev-parse HEAD)" = "$BASELINE" && RUSTFLAGS='--cfg hevc_synchronous_baseline --check-cfg=cfg(hevc_synchronous_baseline) --check-cfg=cfg(hevc_thread_diagnostics)' cargo build --locked --release -p coreav-hevc --features native-hevc --example reference) - name: Install the pinned binary oracle env: R2_ENDPOINT: ${{ secrets.COREAV_CORPUS_R2_ENDPOINT }} @@ -115,7 +226,7 @@ jobs: retention-days: 7 conformance: - if: ${{ !inputs.sanitizer_only && inputs.only == '' }} + if: ${{ !inputs.thread_diagnostics_only && !inputs.thread_dependencies_only && !inputs.thread_rows_only && !inputs.sanitizer_only && inputs.only == '' }} name: hevc-threading-conformance runs-on: ubuntu-24.04 timeout-minutes: 360 @@ -135,6 +246,7 @@ jobs: with: toolchain: stable - run: cargo build --locked --release -p coreav-hevc --features native-hevc --example headers --example decode + - run: cargo build --locked --release -p coreav-backend-hevc --features native-hevc,test-support --example reference - run: scripts/install-oracle-ffmpeg.sh "$RUNNER_TEMP/oracle" - name: Prepare complete conformance references run: | @@ -145,12 +257,25 @@ jobs: python3 -I scripts/hevc_conformance.py acquire --dest "$RUNNER_TEMP/data" > "$RUNNER_TEMP/private/acquire.log" 2>&1 python3 -I scripts/hevc_conformance.py manifest --data "$RUNNER_TEMP/data" --out "$RUNNER_TEMP/private/archives.json" > "$RUNNER_TEMP/private/manifest.log" 2>&1 python3 -I scripts/hevc_conformance.py run --registration "$RUNNER_TEMP/private/archives.json" --data "$RUNNER_TEMP/data" --out "$RUNNER_TEMP/private/census" --parser "$PWD/target/release/examples/headers" --oracle-bin "$RUNNER_TEMP/oracle" --synthetic-data "$RUNNER_TEMP/synthetic" > "$RUNNER_TEMP/private/census.log" 2>&1 - - name: Compare complete outputs at one two four and eight threads + - name: Prepare qualified complete outputs for the selected native boundary run: | set -euo pipefail - for threads in 1 2 4 8; do + counts='1 2 4 8' + if [ -z "$BASELINE" ]; then counts=1; fi + for threads in $counts; do python3 -I scripts/hevc_decode.py --registration "$RUNNER_TEMP/private/archives.json" --census "$RUNNER_TEMP/private/census/census.json" --data "$RUNNER_TEMP/data" --out "$RUNNER_TEMP/private/threads-$threads" --parser "$PWD/target/release/examples/headers" --native "$PWD/target/release/examples/decode" --oracle-bin "$RUNNER_TEMP/oracle" --threads "$threads" > "$RUNNER_TEMP/private/threads-$threads.log" 2>&1 done + if [ -n "$BASELINE" ]; then + python3 -I scripts/hevc_threading.py compare-conformance --root "$RUNNER_TEMP/private" --out "$RUNNER_TEMP/private/thread-comparison.json" + fi + - name: Compare real native row output at every worker boundary + run: | + set -euo pipefail + python3 -I scripts/hevc_threading.py native-conformance \ + --registration "$RUNNER_TEMP/private/archives.json" --census "$RUNNER_TEMP/private/census/census.json" \ + --data "$RUNNER_TEMP/data" --out "$RUNNER_TEMP/private/native-rows" \ + --parser "$PWD/target/release/examples/headers" --native "$PWD/target/release/examples/reference" \ + --scalar-result "$RUNNER_TEMP/private/threads-1/result.json" > "$RUNNER_TEMP/private/native-rows.log" 2>&1 - name: Seal conformance records if: always() run: | @@ -172,7 +297,7 @@ jobs: retention-days: 7 thread-sanitizer: - if: ${{ inputs.sanitizer_only || inputs.only == '' }} + if: ${{ !inputs.thread_diagnostics_only && !inputs.thread_dependencies_only && !inputs.thread_rows_only && (inputs.sanitizer_only || inputs.only == '') }} name: hevc-threading-sanitizer runs-on: ubuntu-24.04 timeout-minutes: 30 @@ -185,7 +310,7 @@ jobs: with: toolchain: nightly components: rust-src - - name: Stress wavefront completion and failure under ThreadSanitizer + - name: Stress wavefront and picture jobs under ThreadSanitizer env: RUSTFLAGS: -Zsanitizer=thread RUSTDOCFLAGS: -Zsanitizer=thread @@ -193,5 +318,12 @@ jobs: set -euo pipefail sudo sysctl vm.mmap_rnd_bits=28 for repeat in $(seq 1 20); do - cargo +nightly test --locked -Zbuild-std --target x86_64-unknown-linux-gnu -p coreav-hevc --features native-hevc --lib wavefront + for group in wavefront decoder::scheduler::tests; do + cargo +nightly test --locked -Zbuild-std --target x86_64-unknown-linux-gnu -p coreav-hevc --features native-hevc-rows-private --lib "$group" | tee "$RUNNER_TEMP/threading-tests.log" + grep -Eq '^test result: ok\. [1-9][0-9]* passed;' "$RUNNER_TEMP/threading-tests.log" + if [ "$group" = decoder::scheduler::tests ]; then + grep -F 'test decoder::scheduler::tests::dependent_raster_job_reads_a_building_predecessor_before_its_first_publication ... ok' "$RUNNER_TEMP/threading-tests.log" + grep -F 'test decoder::scheduler::tests::capacity_probe_mismatch_preserves_legacy_frames_and_summary ... ok' "$RUNNER_TEMP/threading-tests.log" + fi + done done diff --git a/Cargo.lock b/Cargo.lock index 113940231..5816569e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -269,6 +269,7 @@ dependencies = [ name = "coreav-backend-hevc" version = "0.0.1" dependencies = [ + "coreav-counting-alloc", "coreav-decode", "coreav-frame", "coreav-hevc", diff --git a/Cargo.toml b/Cargo.toml index 0351cd23c..87c8915b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ clap = { version = "4.3.3", features = ["derive"] } [workspace.lints.rust] unsafe_code = "forbid" missing_docs = "warn" +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(hevc_thread_diagnostics)'] } [workspace.lints.clippy] all = { level = "warn", priority = -1 } diff --git a/bench/hevc-reference/decode.rs b/bench/hevc-reference/decode.rs index a7a47cd9c..219de35c9 100644 --- a/bench/hevc-reference/decode.rs +++ b/bench/hevc-reference/decode.rs @@ -8,6 +8,193 @@ mod common; #[path = "common/decoded.rs"] mod decoded; +#[cfg(hevc_thread_diagnostics)] +fn dependencies(writer: &mut impl Write, bytes: &[u8]) -> Result<()> { + struct Picture { + index: u64, + poc: i32, + reconstruction_required: bool, + wpp: bool, + info: coreav_hevc::PictureInfo, + dpb_capacity: u32, + highest_sps_layer: u32, + slice_types: [bool; 3], + rps: [Option<(u64, i32, bool)>; 16], + active: [Option<(u64, i32, bool)>; 16], + } + let mut parser = coreav_hevc::HeaderParser::new(coreav_hevc::Limits::default()); + parser.diagnostic_trace_outputs()?; + let mut pictures: Vec = Vec::new(); + pictures + .try_reserve_exact(32) + .map_err(|_| Error::Limit("dependency_records"))?; + common::read_nals( + bytes, + coreav_hevc::Limits::default().max_nal_bytes, + |_, nal| { + if let coreav_hevc::NalInfo::Slice { + header, + picture, + order, + } = parser.read_nal(nal, |_| Ok(()))? + { + if header.first { + if pictures.len() == 32 { + return Err(Error::Limit("dependency_picture_count")); + } + let order = order.ok_or(Error::Internal("dependency_picture_order"))?; + let (dpb_capacity, highest_sps_layer) = parser.diagnostic_sps_envelope()?; + pictures.push(Picture { + index: pictures.len() as u64, + poc: order.poc, + reconstruction_required: order.reconstruction_required, + wpp: picture.wpp, + info: picture, + dpb_capacity, + highest_sps_layer, + slice_types: [false; 3], + rps: [None; 16], + active: [None; 16], + }); + } + let current = pictures + .last_mut() + .ok_or(Error::InvalidData("dependency_first_slice"))?; + if let Some(kind) = header.slice_type { + *current + .slice_types + .get_mut(kind as usize) + .ok_or(Error::InvalidData("dependency_slice_type"))? = true; + } + if current.reconstruction_required { + let snapshot = parser.diagnostic_dependencies()?; + if !header.first && current.rps != snapshot.rps { + return Err(Error::InvalidData("dependency_rps_changed")); + } + current.rps = snapshot.rps; + for (entry, value) in current.active.iter_mut().zip(snapshot.active) { + if value.is_some() { + *entry = value; + } + } + } + } + Ok(()) + }, + )?; + let summary = parser.finish(|_| Ok(()))?; + let output_events: Vec<_> = parser + .diagnostic_outputs()? + .into_iter() + .flatten() + .map(|output| { + json!({"picture_index":output.event.picture_index,"poc":output.event.poc, + "before_picture":output.before_picture}) + }) + .collect(); + let required = pictures + .iter() + .filter(|picture| picture.reconstruction_required) + .count(); + let rows: Vec<_> = pictures.into_iter().map(|picture| json!({ + "index":picture.index,"poc":picture.poc,"reconstruction_required":picture.reconstruction_required, + "wpp":picture.wpp,"envelope":{"coded_width":picture.info.width, + "coded_height":picture.info.height,"width":picture.info.visible_width, + "height":picture.info.visible_height,"bit_depth":picture.info.bit_depth, + "ctb_log2":picture.info.ctb_log2,"dpb_capacity":picture.dpb_capacity, + "highest_sps_layer":picture.highest_sps_layer}, + "slice_types":picture.slice_types.into_iter().enumerate() + .filter_map(|(kind, present)| present.then_some(kind)).collect::>(), + "rps":picture.rps.into_iter().flatten().collect::>(), + "active":picture.active.into_iter().flatten().collect::>() + })).collect(); + serde_json::to_writer( + &mut *writer, + &json!({"mode":"header_dependencies","header_only":true, + "complete":true,"coded_pictures":summary.pictures,"reconstruction_required":required, + "output_pictures":summary.output_pictures,"nals":summary.nals,"pictures":rows, + "output_events":output_events + }), + ) + .map_err(|_| Error::Limit("reference_output"))?; + writer + .write_all(b"\n") + .map_err(|_| Error::Limit("reference_output")) +} + +#[cfg(hevc_thread_diagnostics)] +fn row_dependencies(writer: &mut impl Write, bytes: &[u8]) -> Result<()> { + let mut header_bytes = Vec::new(); + dependencies(&mut header_bytes, bytes)?; + let header: serde_json::Value = + serde_json::from_slice(&header_bytes).map_err(|_| Error::Internal("row_trace_header"))?; + coreav_hevc::thread_diagnostics::begin_rows()?; + let limits = DecodeLimits { + max_memory_bytes: usize::try_from(coreav_decode::Limits::default().max_memory_bytes) + .map_err(|_| Error::Limit("reference_memory"))?, + threads: 1, + ..DecodeLimits::default() + }; + let mut decoder = PictureDecoder::new(limits); + let mut output_pictures = 0u64; + let mut emit = |_: coreav_hevc::DecodedPicture<'_>| { + output_pictures = output_pictures + .checked_add(1) + .ok_or(Error::Limit("row_trace_outputs"))?; + Ok(()) + }; + common::read_nals(bytes, limits.headers.max_nal_bytes, |_, nal| { + decoder.read_nal(nal, None, &mut emit) + })?; + let summary = decoder.finish(&mut emit)?; + drop(decoder); + if header + .get("output_pictures") + .and_then(serde_json::Value::as_u64) + != Some(output_pictures) + { + return Err(Error::Internal("row_trace_output_count")); + } + let pictures: Vec<_> = coreav_hevc::thread_diagnostics::take_rows(summary.max_workers)? + .into_iter() + .map(|picture| { + let rows: Vec<_> = picture + .rows + .into_iter() + .enumerate() + .map(|(row, reads)| { + let references: Vec<_> = reads + .references + .into_iter() + .flatten() + .map(|reference| { + json!([ + reference.index, + reference.poc, + reference.sample_row, + reference.motion_row + ]) + }) + .collect(); + json!({"row":row,"references":references,"sample_calls":reads.sample_calls, + "motion_calls":reads.motion_calls}) + }) + .collect(); + json!({"index":picture.index,"poc":picture.poc,"width":picture.info.width, + "height":picture.info.height,"ctb_log2":picture.info.ctb_log2, + "tiles":picture.info.tiles,"wpp":picture.info.wpp,"rows":rows}) + }) + .collect(); + serde_json::to_writer(&mut *writer, &json!({"mode":"row_dependencies","complete":true, + "header_only":false,"requested_threads":1,"thread_limit":1,"max_workers":summary.max_workers, + "decoded_pictures":summary.decoded_pictures,"output_pictures":output_pictures, + "header_dependencies":header,"pictures":pictures + })).map_err(|_| Error::Limit("reference_output"))?; + writer + .write_all(b"\n") + .map_err(|_| Error::Limit("reference_output")) +} + fn run(writer: &mut impl Write) -> Result<()> { let mut args = std::env::args_os().skip(1); let path = args @@ -16,7 +203,7 @@ fn run(writer: &mut impl Write) -> Result<()> { let mode = args .next() .ok_or(Error::InvalidData("reference_arguments"))?; - let threads = args + let requested_threads = args .next() .map(|value| { value @@ -27,16 +214,21 @@ fn run(writer: &mut impl Write) -> Result<()> { .unwrap_or(Some(1)) .filter(|value| *value <= 16) .ok_or(Error::InvalidData("reference_arguments"))?; - let threads = if threads == 0 { - std::thread::available_parallelism() - .map_or(1, usize::from) - .min(16) + let host_threads = std::thread::available_parallelism().map_or(1, usize::from); + let threads = if requested_threads == 0 { + host_threads.min(16) } else { - threads + requested_threads }; - if args.next().is_some() || (mode != "check" && mode != "decode") { + let header_only = cfg!(hevc_thread_diagnostics) && mode == "dependencies"; + let row_reads = cfg!(hevc_thread_diagnostics) && mode == "rows"; + if args.next().is_some() || (mode != "check" && mode != "decode" && !header_only && !row_reads) + { return Err(Error::InvalidData("reference_arguments")); } + if row_reads && requested_threads != 1 { + return Err(Error::InvalidData("row_trace_threads")); + } let source = std::fs::File::open(path).map_err(|_| Error::InvalidData("input_open"))?; let mut bytes = Vec::new(); source @@ -46,20 +238,20 @@ fn run(writer: &mut impl Write) -> Result<()> { if bytes.len() >= 64 << 20 { return Err(Error::Limit("reference_input")); } + #[cfg(hevc_thread_diagnostics)] + if header_only { + return dependencies(writer, &bytes); + } + #[cfg(hevc_thread_diagnostics)] + if row_reads { + return row_dependencies(writer, &bytes); + } let limits = DecodeLimits { max_memory_bytes: usize::try_from(coreav_decode::Limits::default().max_memory_bytes) .map_err(|_| Error::Limit("reference_memory"))?, ..DecodeLimits::default() }; - let mut decoder = PictureDecoder::new(limits); - #[cfg(not(hevc_serial_baseline))] - { - decoder = decoder.with_threads(threads); - } - #[cfg(hevc_serial_baseline)] - { - std::hint::black_box(threads); - } + let mut decoder = PictureDecoder::new(limits).with_threads(threads); let mut output = decoded::Output::new(writer); let mut pictures = 0u64; let mut callback = |picture: coreav_hevc::DecodedPicture<'_>| { @@ -72,19 +264,60 @@ fn run(writer: &mut impl Write) -> Result<()> { Ok(()) }; let receipt = common::read_nals(bytes.as_slice(), limits.headers.max_nal_bytes, |_, nal| { - decoder.read_nal(nal, None, &mut callback) - })?; + #[cfg(hevc_synchronous_baseline)] + { + decoder.read_nal(nal, None, &mut callback) + } + #[cfg(not(hevc_synchronous_baseline))] + { + decoder.read_nal_scheduled(nal, None, None, |picture, _| callback(picture)) + } + }); + let receipt = match receipt { + Ok(receipt) => receipt, + Err(error) => { + #[cfg(not(hevc_synchronous_baseline))] + decoder.synchronize_pending()?; + return Err(error); + } + }; let summary = decoder.finish(&mut callback)?; if mode == "check" { output.finish(summary, receipt, limits) } else { - let mut record = json!({"pictures":pictures,"thread_limit":threads,"max_memory_bytes":limits.max_memory_bytes, + let record = json!({"pictures":pictures,"requested_threads":requested_threads, + "host_threads":host_threads,"thread_limit":threads,"max_memory_bytes":limits.max_memory_bytes, "decoded_pictures":summary.decoded_pictures,"bins":summary.reconstruction.bins}); - #[cfg(not(hevc_serial_baseline))] - { - record["parallel_pictures"] = json!(summary.parallel_pictures); - record["max_workers"] = json!(summary.max_workers); - } + #[cfg(not(hevc_synchronous_baseline))] + let record = { + let mut record = record; + let fields = record + .as_object_mut() + .ok_or(Error::Internal("reference_record"))?; + fields.insert("thread_limit".to_owned(), json!(threads.min(host_threads))); + fields.insert( + "parallel_pictures".to_owned(), + json!(summary.parallel_pictures), + ); + fields.insert( + "frame_parallel_pictures".to_owned(), + json!(summary.frame_parallel_pictures), + ); + fields.insert("max_workers".to_owned(), json!(summary.max_workers)); + record + }; + #[cfg(hevc_thread_diagnostics)] + let record = { + let mut record = record; + record + .as_object_mut() + .ok_or(Error::Internal("reference_record"))? + .insert( + "thread_diagnostics".to_owned(), + json!(coreav_hevc::thread_diagnostics::take().as_slice()), + ); + record + }; serde_json::to_writer(&mut *writer, &record) .map_err(|_| Error::Limit("reference_output"))?; writer diff --git a/crates/coreav-backend-hevc/Cargo.toml b/crates/coreav-backend-hevc/Cargo.toml index 0575b25bd..06a680d36 100644 --- a/crates/coreav-backend-hevc/Cargo.toml +++ b/crates/coreav-backend-hevc/Cargo.toml @@ -11,7 +11,8 @@ publish = false [features] default = [] -native-hevc = ["dep:coreav-hevc", "coreav-hevc/native-hevc"] +native-hevc = ["dep:coreav-hevc", "coreav-hevc/native-hevc-rows-private", "coreav-decode/native-hevc-private"] +test-support = ["native-hevc", "coreav-hevc/test-support"] [dependencies] coreav-decode = { path = "../coreav-decode" } @@ -19,6 +20,7 @@ coreav-frame = { path = "../coreav-frame" } coreav-hevc = { path = "../coreav-hevc", default-features = false, optional = true } [dev-dependencies] +coreav-counting-alloc = { path = "../../tools/counting-alloc" } serde_json.workspace = true md-5 = "0.10" sha2 = "0.10" @@ -29,5 +31,9 @@ coreav-thumbnail = { path = "../coreav-thumbnail", default-features = false } name = "boundary" required-features = ["native-hevc"] +[[example]] +name = "reference" +required-features = ["native-hevc", "test-support"] + [lints] workspace = true diff --git a/crates/coreav-backend-hevc/examples/boundary.rs b/crates/coreav-backend-hevc/examples/boundary.rs index 77217a6e1..ac2384589 100644 --- a/crates/coreav-backend-hevc/examples/boundary.rs +++ b/crates/coreav-backend-hevc/examples/boundary.rs @@ -10,6 +10,8 @@ use md5::{Digest, Md5}; use serde_json::{Value, json}; use std::io::Write; +#[path = "common/access_units.rs"] +mod access_units; #[allow(dead_code)] #[path = "../../coreav-hevc/examples/common/mod.rs"] mod common; @@ -157,10 +159,11 @@ fn annex_b(path: std::ffi::OsString, maximum: u64, writer: &mut impl Write) -> R c5: None, full_records: Some(Vec::new()), }; - let (bytes, packets) = - random_access::units(&path, limits.max_packet_bytes, |data, pts, _, _| { - send(data, pts, &mut *opened.decoder, &mut output) - })?; + let (bytes, packets) = random_access::units( + std::fs::File::open(&path)?, + limits.max_packet_bytes, + |data, pts, _, _| send(data, pts, &mut *opened.decoder, &mut output), + )?; opened.decoder.send_eof()?; output.drain(&mut *opened.decoder)?; let full = output diff --git a/crates/coreav-backend-hevc/examples/boundary/random_access.rs b/crates/coreav-backend-hevc/examples/boundary/random_access.rs index fc32667d5..b774b5baf 100644 --- a/crates/coreav-backend-hevc/examples/boundary/random_access.rs +++ b/crates/coreav-backend-hevc/examples/boundary/random_access.rs @@ -1,84 +1,7 @@ use super::*; use std::collections::BTreeMap; -pub(super) fn units( - path: &std::ffi::OsStr, - maximum: u64, - mut emit: impl FnMut(&[u8], Option, Option, &[Vec]) -> Result<()>, -) -> Result<(u64, u64)> { - let (mut packet, mut parameters) = (Vec::new(), Vec::new()); - let mut parameter_bytes = 0usize; - let (mut has_vcl, mut pts, mut kind) = (false, None, None); - let (mut coded, mut packets) = (0i64, 0u64); - let mut failure = None; - let receipt = common::read_nals( - std::fs::File::open(path)?, - coreav_hevc::Limits::default().max_nal_bytes, - |_, nal| { - let header = NalHeader::parse(nal)?; - let first = header.unit_type < 32 && nal.get(2).is_some_and(|byte| byte & 128 != 0); - let prefix = matches!(header.unit_type, 32..=35 | 39 | 41..=44 | 48..=55); - if has_vcl && (prefix || first) { - emit(&packet, pts, kind, ¶meters).map_err(|error| { - failure = Some(error); - coreav_hevc::Error::InvalidData("boundary_access_unit") - })?; - packets += 1; - packet.clear(); - has_vcl = false; - pts = None; - kind = None; - } - if (32..=34).contains(&header.unit_type) { - let mut owned = Vec::new(); - parameter_bytes = parameter_bytes - .checked_add(nal.len()) - .filter(|n| *n <= 64 << 20) - .ok_or(coreav_hevc::Error::Limit("parameter_replay_bytes"))?; - owned - .try_reserve_exact(nal.len()) - .map_err(|_| coreav_hevc::Error::Limit("parameter_replay_allocation"))?; - owned.extend_from_slice(nal); - parameters - .try_reserve(1) - .map_err(|_| coreav_hevc::Error::Limit("parameter_replay_allocation"))?; - parameters.push(owned); - } - if first { - pts = Some(coded); - kind = Some(header.unit_type); - coded = coded - .checked_add(1) - .ok_or(coreav_hevc::Error::Limit("picture_count"))?; - } - has_vcl |= header.unit_type < 32; - let wanted = packet - .len() - .checked_add(4) - .and_then(|n| n.checked_add(nal.len())) - .filter(|n| *n as u64 <= maximum) - .ok_or(coreav_hevc::Error::Limit("access_unit_bytes"))?; - packet - .try_reserve_exact(wanted - packet.len()) - .map_err(|_| coreav_hevc::Error::Limit("access_unit_allocation"))?; - if packet.capacity() as u64 > maximum { - return Err(coreav_hevc::Error::Limit("access_unit_capacity")); - } - packet.extend_from_slice(&[0, 0, 0, 1]); - packet.extend_from_slice(nal); - Ok(()) - }, - ); - let receipt = match receipt { - Ok(receipt) => receipt, - Err(error) => return Err(failure.unwrap_or_else(|| error.into())), - }; - if !packet.is_empty() { - emit(&packet, pts, kind, ¶meters)?; - packets += 1; - } - Ok((receipt.bytes, packets)) -} +pub(super) use crate::access_units::units; struct Seek { decoder: Box, @@ -146,7 +69,7 @@ pub(super) fn check(path: &std::ffi::OsStr, limits: &Limits, full: &[Value]) -> let mut results = Vec::new(); let mut keyframes = 0usize; units( - path, + std::fs::File::open(path)?, limits.max_packet_bytes, |data, pts, kind, parameters| { if kind.is_some_and(|kind| (16..=21).contains(&kind)) { diff --git a/crates/coreav-backend-hevc/examples/common/access_units.rs b/crates/coreav-backend-hevc/examples/common/access_units.rs new file mode 100644 index 000000000..b553a8895 --- /dev/null +++ b/crates/coreav-backend-hevc/examples/common/access_units.rs @@ -0,0 +1,81 @@ +use crate::{Result, common}; +use coreav_hevc::NalHeader; + +pub(crate) fn units( + reader: impl std::io::Read, + maximum: u64, + mut emit: impl FnMut(&[u8], Option, Option, &[Vec]) -> Result<()>, +) -> Result<(u64, u64)> { + let (mut packet, mut parameters) = (Vec::new(), Vec::new()); + let mut parameter_bytes = 0usize; + let (mut has_vcl, mut pts, mut kind) = (false, None, None); + let (mut coded, mut packets) = (0i64, 0u64); + let mut failure = None; + let receipt = common::read_nals( + reader, + coreav_hevc::Limits::default().max_nal_bytes, + |_, nal| { + let header = NalHeader::parse(nal)?; + let first = header.unit_type < 32 && nal.get(2).is_some_and(|byte| byte & 128 != 0); + let prefix = matches!(header.unit_type, 32..=35 | 39 | 41..=44 | 48..=55); + if has_vcl && (prefix || first) { + emit(&packet, pts, kind, ¶meters).map_err(|error| { + failure = Some(error); + coreav_hevc::Error::InvalidData("boundary_access_unit") + })?; + packets += 1; + packet.clear(); + has_vcl = false; + pts = None; + kind = None; + } + if (32..=34).contains(&header.unit_type) { + let mut owned = Vec::new(); + parameter_bytes = parameter_bytes + .checked_add(nal.len()) + .filter(|n| *n <= 64 << 20) + .ok_or(coreav_hevc::Error::Limit("parameter_replay_bytes"))?; + owned + .try_reserve_exact(nal.len()) + .map_err(|_| coreav_hevc::Error::Limit("parameter_replay_allocation"))?; + owned.extend_from_slice(nal); + parameters + .try_reserve(1) + .map_err(|_| coreav_hevc::Error::Limit("parameter_replay_allocation"))?; + parameters.push(owned); + } + if first { + pts = Some(coded); + kind = Some(header.unit_type); + coded = coded + .checked_add(1) + .ok_or(coreav_hevc::Error::Limit("picture_count"))?; + } + has_vcl |= header.unit_type < 32; + let wanted = packet + .len() + .checked_add(4) + .and_then(|n| n.checked_add(nal.len())) + .filter(|n| *n as u64 <= maximum) + .ok_or(coreav_hevc::Error::Limit("access_unit_bytes"))?; + packet + .try_reserve_exact(wanted - packet.len()) + .map_err(|_| coreav_hevc::Error::Limit("access_unit_allocation"))?; + if packet.capacity() as u64 > maximum { + return Err(coreav_hevc::Error::Limit("access_unit_capacity")); + } + packet.extend_from_slice(&[0, 0, 0, 1]); + packet.extend_from_slice(nal); + Ok(()) + }, + ); + let receipt = match receipt { + Ok(receipt) => receipt, + Err(error) => return Err(failure.unwrap_or_else(|| error.into())), + }; + if !packet.is_empty() { + emit(&packet, pts, kind, ¶meters)?; + packets += 1; + } + Ok((receipt.bytes, packets)) +} diff --git a/crates/coreav-backend-hevc/examples/reference.rs b/crates/coreav-backend-hevc/examples/reference.rs new file mode 100644 index 000000000..d5c31caf1 --- /dev/null +++ b/crates/coreav-backend-hevc/examples/reference.rs @@ -0,0 +1,348 @@ +#![doc = "Ordinary native HEVC packet reference and admission evidence."] + +use coreav_backend_hevc::{NativeHevc, NativeOptions, test_support}; +use coreav_decode::{BackendKind, Codec, Decoder, Limits, Packet, Policy, StreamConfig}; +use coreav_frame::Pixels; +use md5::{Digest, Md5}; +use serde_json::json; +use std::io::Write; + +#[path = "common/access_units.rs"] +mod access_units; +#[allow(dead_code)] +#[path = "../../coreav-hevc/examples/common/mod.rs"] +mod common; + +#[path = "../../coreav-hevc/examples/common/counters.rs"] +mod counters; + +type Result = std::result::Result>; + +fn drain( + decoder: &mut dyn Decoder, + writer: &mut impl Write, + check: bool, + pictures: &mut u64, + bytes: &mut u64, + hash: &mut Md5, +) -> Result<()> { + while let Some(frame) = decoder.receive_frame()? { + *pictures += 1; + if !check { + std::hint::black_box(frame); + continue; + } + let Pixels::Cpu(picture) = &frame.pixels else { + return Err("native reference returned non-CPU pixels".into()); + }; + let mut picture_hash = Md5::new(); + let mut picture_bytes = 0u64; + for plane in picture.planes() { + picture_hash.update(plane.data()); + hash.update(plane.data()); + picture_bytes += plane.data().len() as u64; + } + *bytes += picture_bytes; + let color = frame.color; + let chroma_location = color.chroma_location.map(|value| match value { + coreav_frame::ChromaLocation::Left => 0, + coreav_frame::ChromaLocation::Center => 1, + coreav_frame::ChromaLocation::TopLeft => 2, + coreav_frame::ChromaLocation::Top => 3, + coreav_frame::ChromaLocation::BottomLeft => 4, + coreav_frame::ChromaLocation::Bottom => 5, + }); + serde_json::to_writer( + &mut *writer, + &json!({ + "type":"picture", "timestamp":frame.pts, "width":picture.width(), + "height":picture.height(), "bit_depth":picture.format().depth(), + "packing":if picture.format().depth()==8 {"yuv420p"} else {"yuv420p10le"}, + "color":{"primaries":color.primaries,"transfer":color.transfer,"matrix":color.matrix, + "full_range":color.full_range,"chroma_location":chroma_location}, "output_bytes":picture_bytes, + "output_md5":format!("{:x}", picture_hash.finalize()) + }), + )?; + writer.write_all(b"\n")?; + } + Ok(()) +} + +fn run(writer: &mut impl Write) -> Result<()> { + let mut args = std::env::args_os().skip(1); + let path = args.next().ok_or("missing native reference source")?; + let mode = args.next().ok_or("missing native reference mode")?; + let requested: usize = args + .next() + .map(|s| -> Result { + Ok(s.into_string() + .map_err(|_| "invalid native threads")? + .parse()?) + }) + .transpose()? + .unwrap_or(1); + let memory = args + .next() + .map(|s| { + s.into_string() + .ok() + .and_then(|s| s.parse::().ok()) + .ok_or("invalid native memory") + }) + .transpose()?; + if args.next().is_some() || requested > 16 || mode != "check" && mode != "decode" { + return Err("invalid native reference arguments".into()); + } + reference( + std::fs::File::open(path)?, + mode == "check", + requested, + memory, + writer, + ) +} + +fn reference( + reader: impl std::io::Read, + check: bool, + requested: usize, + memory: Option, + writer: &mut impl Write, +) -> Result<()> { + let host = std::thread::available_parallelism().map_or(1, usize::from); + let threads = if requested == 0 { + host.min(16) + } else { + requested + }; + let limits = Limits { + max_memory_bytes: memory.unwrap_or(Limits::default().max_memory_bytes), + max_decoded_bytes: 64 << 30, + timeout: std::time::Duration::from_secs(600), + ..Limits::default() + }; + test_support::begin(); + let backend = NativeHevc::with_options(NativeOptions { + threads: Some(threads), + }); + let mut opened = coreav_decode::open( + &StreamConfig::new(Codec::Hevc), + &Policy::default(), + &limits, + &[&backend], + )?; + let (mut pictures, mut output_bytes, mut hash) = (0, 0, Md5::new()); + let input = access_units::units(reader, limits.max_packet_bytes, |data, pts, _, _| { + opened.decoder.send_packet(&Packet { + data, + pts, + dts: None, + keyframe: false, + discard: false, + })?; + drain( + &mut *opened.decoder, + writer, + check, + &mut pictures, + &mut output_bytes, + &mut hash, + ) + }); + let (input_bytes, packets) = match input { + Ok(receipt) => receipt, + Err(error) => { + let error = match error.downcast::() { + Ok(error) => *error, + Err(error) => coreav_decode::Error::new( + coreav_decode::ErrorKind::InvalidData, + error.to_string(), + ), + }; + return Err(opened.decoder.settle_native_hevc_refusal(error).into()); + } + }; + opened.decoder.send_eof()?; + drain( + &mut *opened.decoder, + writer, + check, + &mut pictures, + &mut output_bytes, + &mut hash, + )?; + if opened.decoder.info().backend != BackendKind::Native || opened.decoder.info().fell_back() { + return Err("native reference changed backend".into()); + } + let report = test_support::take().ok_or("native report is missing")?; + if report.overflow { + return Err("native admission report overflow".into()); + } + let summary = report.summary.ok_or("native complete summary is missing")?; + let mut semantic = summary; + semantic.frame_parallel_pictures = 0; + semantic.parallel_pictures = 0; + semantic.max_workers = 0; + semantic.memory_bytes = 0; + semantic.reconstruction.memory_bytes = 0; + let semantic = json!({ + "headers":counters::header_stats(semantic.headers), + "decoded_pictures":semantic.decoded_pictures,"skipped_pictures":semantic.skipped_pictures, + "skipped_picture_hashes":semantic.skipped_picture_hashes,"skipped_reserved_hashes":semantic.skipped_reserved_hashes, + "stats":common::reconstruction_stats(semantic.reconstruction),"inter":counters::inter_stats(semantic.inter), + }); + let mut admissions = Vec::new(); + for admission in report.admissions { + if admission + .lag + .checked_mul(admission.frame_bytes) + .and_then(|n| n.checked_add(admission.reserved_bytes)) + .is_none_or(|n| n > admission.limit_bytes) + { + return Err("native admission certificate exceeds its limit".into()); + } + admissions.push(json!({ + "sps_id":admission.sps_id, "sps_sha256":admission.sps_hash, + "ceiling":admission.ceiling, "workers":admission.workers, "lag":admission.lag, + "frame_bytes":admission.frame_bytes, "reserved_bytes":admission.reserved_bytes, + "limit_bytes":admission.limit_bytes + })); + } + serde_json::to_writer( + &mut *writer, + &json!({ + "type":"summary", "complete":true, "boundary":true, "backend":"native", + "input_bytes":input_bytes, "sent_packets":packets, "pictures":pictures,"output_pictures":pictures, + "inherited_color":{"primaries":null,"transfer":null,"matrix":null,"full_range":null,"chroma_location":null}, + "output_bytes":if check { Some(output_bytes) } else { None }, + "output_md5":if check { Some(format!("{:x}", hash.finalize())) } else { None }, + "semantic":semantic, "decoded_pictures":summary.decoded_pictures, + "bins":summary.reconstruction.bins, "requested_threads":requested, "host_threads":host, + "thread_limit":threads.min(host), "max_workers":summary.max_workers,"max_memory_bytes":limits.max_memory_bytes, + "memory_bytes":summary.memory_bytes, + "parallel_pictures":summary.parallel_pictures, "frame_parallel_pictures":summary.frame_parallel_pictures, + "row_parallel_pictures":report.row_parallel_pictures, + "admissions":admissions + }), + )?; + writer.write_all(b"\n")?; + Ok(()) +} + +#[cfg(test)] +#[path = "../../coreav-hevc/test-support/pcm.rs"] +mod pcm; +#[cfg(test)] +#[path = "../../coreav-hevc/test-support/writer.rs"] +mod writer; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn native_reference_checks_generated_depths_at_every_worker_boundary() -> Result<()> { + for depth in [8, 10] { + let fixture = pcm::pcm_fixture(depth, depth, false)?; + let mut input = Vec::new(); + for nal in fixture.sets.iter().chain([&fixture.nal, &fixture.nal]) { + input.extend_from_slice(&[0, 0, 0, 1]); + input.extend_from_slice(nal); + } + let mut pixels = Vec::new(); + for _ in 0..2 { + for plane in &fixture.planes { + for sample in plane { + pixels.extend_from_slice( + sample + .to_le_bytes() + .get(..if depth == 10 { 2 } else { 1 }) + .ok_or("generated sample depth")?, + ); + } + } + } + let expected = format!("{:x}", Md5::digest(&pixels)); + let mut semantic = None; + for threads in [1, 2, 4, 8, 0] { + for check in [false, true] { + let mut output = Vec::new(); + reference(input.as_slice(), check, threads, None, &mut output)?; + let records: Vec = std::str::from_utf8(&output)? + .lines() + .map(serde_json::from_str) + .collect::>()?; + let summary = records.last().ok_or("missing summary")?; + assert_eq!(summary["type"], "summary"); + assert_eq!(summary["complete"], true); + assert_eq!(summary["boundary"], true); + assert_eq!(summary["pictures"], 2); + assert_eq!(summary["decoded_pictures"], 2); + assert_eq!(summary["sent_packets"], 2); + let workers = summary["max_workers"] + .as_u64() + .ok_or("missing worker count")?; + let ceiling = summary["thread_limit"] + .as_u64() + .ok_or("missing worker ceiling")?; + assert!((1..=ceiling).contains(&workers)); + let accounted = summary["memory_bytes"] + .as_u64() + .ok_or("missing memory count")?; + assert!( + accounted > 0 + && accounted + <= summary["max_memory_bytes"] + .as_u64() + .ok_or("missing memory limit")? + ); + assert_eq!(summary["semantic"]["headers"]["pictures"], 2); + assert_eq!(summary["semantic"]["stats"]["ctus"], 2); + assert_eq!(summary["semantic"]["stats"]["bins"], summary["bins"]); + assert_eq!( + *semantic.get_or_insert_with(|| summary["semantic"].clone()), + summary["semantic"] + ); + assert_eq!(records.len(), if check { 3 } else { 1 }); + if check { + assert_eq!(summary["output_md5"], expected); + assert_eq!(summary["output_bytes"], pixels.len()); + assert_eq!( + records.first().ok_or("missing first picture")?["timestamp"], + 0 + ); + assert_eq!( + records.get(1).ok_or("missing second picture")?["timestamp"], + 1 + ); + } else { + assert!(summary["output_md5"].is_null()); + } + let admitted = summary["admissions"] + .as_array() + .ok_or("missing admissions")?; + let effective = if threads == 0 { 16 } else { threads } + .min(std::thread::available_parallelism().map_or(1, usize::from)); + assert_eq!(admitted.is_empty(), effective == 1); + } + } + } + Ok(()) + } +} + +fn main() -> std::process::ExitCode { + let mut writer = std::io::BufWriter::new(std::io::stdout().lock()); + match run(&mut writer) { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(error) => { + let mut stderr = std::io::stderr().lock(); + let _ = serde_json::to_writer( + &mut stderr, + &json!({"type":"refusal","reason":error.to_string()}), + ); + let _ = stderr.write_all(b"\n"); + std::process::ExitCode::FAILURE + } + } +} diff --git a/crates/coreav-backend-hevc/fuzz/fuzz_targets/boundary.rs b/crates/coreav-backend-hevc/fuzz/fuzz_targets/boundary.rs index 10f7187f2..1f59ba929 100644 --- a/crates/coreav-backend-hevc/fuzz/fuzz_targets/boundary.rs +++ b/crates/coreav-backend-hevc/fuzz/fuzz_targets/boundary.rs @@ -47,6 +47,7 @@ fuzz_target!(|data: &[u8]| { break; }; remaining = tail; + let mut received = false; let result = match operation & 3 { 0 => { let Some(prefix) = remaining.first_chunk::<2>() else { @@ -72,17 +73,26 @@ fuzz_target!(|data: &[u8]| { } 1 => decoder .receive_frame() - .map(|frame| delivered.extend(frame.map(|frame| frame.pts))), + .map(|frame| { + received = frame.is_some(); + delivered.extend(frame.map(|frame| frame.pts)); + }), 2 => decoder.send_eof(), _ => break, }; if failed { - assert!(result.is_err()); + assert!(result.is_err() || received); } failed |= result.is_err(); } if failed { - assert!(decoder.receive_frame().is_err()); + loop { + match decoder.receive_frame() { + Ok(Some(frame)) => delivered.push(frame.pts), + Ok(None) => panic!("failed decoder lost its terminal result"), + Err(_) => break, + } + } assert!(decoder.send_eof().is_err()); } else { let _ = decoder.send_eof(); diff --git a/crates/coreav-backend-hevc/src/lib.rs b/crates/coreav-backend-hevc/src/lib.rs index d9a01085c..33e20f9eb 100644 --- a/crates/coreav-backend-hevc/src/lib.rs +++ b/crates/coreav-backend-hevc/src/lib.rs @@ -5,6 +5,9 @@ mod configuration; mod output; +#[cfg(feature = "test-support")] +#[doc(hidden)] +pub mod test_support; use std::time::Instant; @@ -154,12 +157,12 @@ fn scope(stream: &StreamConfig, traits: &StreamTraits) -> Result<(), Error> { Ok(()) } -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(PartialEq, Eq)] enum State { Active, Draining, Ended, - Failed, + Failed(Error), } struct NativeDecoder { @@ -171,11 +174,110 @@ struct NativeDecoder { next_origin: u64, } +struct RowOutput<'a> { + output: &'a mut Output, + error: Option<(usize, Error)>, +} + +impl RowOutput<'_> { + fn capture(&mut self, result: Result, offset: usize) -> coreav_hevc::Result { + result.map_err(|error| { + if self + .error + .as_ref() + .is_none_or(|(earlier, _)| *earlier >= offset) + { + self.error = Some((offset, error)); + } + coreav_hevc::Error::Limit("boundary_output") + }) + } + + fn finish(self, error: coreav_hevc::Error) -> Error { + if error == coreav_hevc::Error::Limit("boundary_output") { + self.error + .map_or_else(|| codec_error(error), |(_, error)| error) + } else { + codec_error(error) + } + } +} + +impl coreav_hevc::native_rows::RowSink for RowOutput<'_> { + fn prepare( + &mut self, + header: coreav_hevc::native_rows::OutputHeader, + memory: &mut coreav_hevc::OutputMemory<'_>, + bounded: bool, + ) -> coreav_hevc::Result { + let offset = self.output.ready.len(); + let result = self.output.prepare(header, memory, bounded); + self.capture(result, offset) + } + + fn reserve_queue( + &mut self, + records: usize, + memory: &mut coreav_hevc::OutputMemory<'_>, + admit: &mut dyn FnMut(usize) -> coreav_hevc::Result, + ) -> coreav_hevc::Result { + self.output.reserve_queue(records, memory, admit) + } + + fn qualify( + &self, + info: coreav_hevc::PictureInfo, + probe: &mut coreav_hevc::native_rows::CapacityProbe, + ) -> bool { + self.output.qualify(info, probe) + } + + fn fill( + &mut self, + offset: usize, + picture: coreav_hevc::DecodedPicture<'_>, + ) -> coreav_hevc::Result<()> { + let result = self.output.fill(offset, picture); + self.capture(result, offset) + } + + fn truncate(&mut self, keep: usize) { + self.output.truncate(keep); + } + + fn cost( + &self, + debt: usize, + frame_bytes: usize, + ) -> coreav_hevc::Result { + self.output.cost(debt, frame_bytes) + } +} + impl NativeDecoder { - fn fail(&mut self) { - self.state = State::Failed; + fn terminal_result(&mut self, result: Result) -> Result { + let Err(error) = result else { + return result; + }; + if let State::Failed(error) = &self.state { + return Err(error.clone()); + } + let pending = match &mut self.decoder { + Some(decoder) => { + let mut sink = RowOutput { + output: &mut self.output, + error: None, + }; + decoder + .settle_rows(&mut sink) + .map_err(|error| sink.finish(error)) + } + None => Ok(()), + }; + let error = pending.err().unwrap_or(error); self.decoder = None; - self.output.clear(); + self.state = State::Failed(error.clone()); + Err(error) } fn send(&mut self, packet: &Packet<'_>) -> Result<(), Error> { @@ -212,15 +314,13 @@ impl NativeDecoder { ) .map_err(codec_error)? .0; - let mut output_error = None; - let result = - decoder.read_nal_with_origin(nal, packet.pts, origin, |picture, memory| { - self.output.push(picture, memory).map_err(|error| { - output_error = Some(error); - coreav_hevc::Error::Limit("boundary_output") - }) - }); - result.map_err(|error| output_error.unwrap_or_else(|| codec_error(error))) + let mut sink = RowOutput { + output: &mut self.output, + error: None, + }; + decoder + .read_nal_rows(nal, packet.pts, Some(origin), &mut sink) + .map_err(|error| sink.finish(error)) }; match self.framing { Framing::AnnexB => { @@ -237,12 +337,14 @@ impl NativeDecoder { if count == 0 { return Err(invalid("empty HEVC access unit")); } + #[cfg(feature = "test-support")] + test_support::admitted(decoder); self.output.check() } fn eof(&mut self) -> Result<(), Error> { match self.state { - State::Failed => return Err(invalid("the HEVC decode request failed")), + State::Failed(ref error) => return Err(error.clone()), State::Ended => return Ok(()), State::Draining => return self.output.check(), State::Active => {} @@ -252,17 +354,19 @@ impl NativeDecoder { .decoder .as_mut() .ok_or(invalid("the HEVC decoder is closed"))?; - let mut output_error = None; - decoder - .finish_with_output(|picture, memory| { - self.output.push(picture, memory).map_err(|error| { - output_error = Some(error); - coreav_hevc::Error::Limit("boundary_output") - }) - }) - .map_err(|error| output_error.unwrap_or_else(|| codec_error(error)))?; + let mut sink = RowOutput { + output: &mut self.output, + error: None, + }; + let _summary = decoder + .finish_rows(&mut sink) + .map_err(|error| sink.finish(error))?; + #[cfg(feature = "test-support")] + test_support::completed(_summary); self.output.check()?; - self.decoder = None; + if !decoder.has_native_rows() { + self.decoder = None; + } self.state = if self.output.is_empty() { State::Ended } else { @@ -273,18 +377,52 @@ impl NativeDecoder { fn receive(&mut self) -> Result, Error> { match self.state { - State::Failed => return Err(invalid("the HEVC decode request failed")), + State::Failed(ref error) => { + return self.output.pop().map(Some).ok_or_else(|| error.clone()); + } State::Ended => return Ok(None), State::Active | State::Draining => {} } self.output.check()?; - let frame = self.output.pop(); - if let Some(decoder) = &mut self.decoder { - decoder - .set_external_memory(self.output.retained_bytes()?) - .map_err(codec_error)?; - } + let decision = self + .decoder + .as_mut() + .map(PictureDecoder::receive_rows) + .transpose() + .map_err(codec_error)? + .flatten(); + let frame = match decision { + Some(decision) => { + if let Some(offset) = decision.consume { + self.output.consume(offset)?; + } + if decision.pop { + let frame = self + .output + .pop_rows() + .ok_or(invalid("missing validated HEVC output"))?; + self.decoder + .as_mut() + .ok_or(invalid("the HEVC decoder is closed"))? + .release_rows_memory(self.output.retained_bytes()?) + .map_err(codec_error)?; + Some(frame) + } else { + None + } + } + None => { + let frame = self.output.pop(); + if let Some(decoder) = &mut self.decoder { + decoder + .set_external_memory(self.output.retained_bytes()?) + .map_err(codec_error)?; + } + frame + } + }; if self.state == State::Draining && self.output.is_empty() { + self.decoder = None; self.state = State::Ended; } Ok(frame) @@ -294,31 +432,44 @@ impl NativeDecoder { impl Decoder for NativeDecoder { fn send_packet(&mut self, packet: &Packet<'_>) -> Result<(), Error> { let result = self.send(packet); - if result.is_err() { - self.fail(); - } - result + self.terminal_result(result) } fn send_eof(&mut self) -> Result<(), Error> { let result = self.eof(); - if result.is_err() { - self.fail(); - } - result + self.terminal_result(result) } fn receive_frame(&mut self) -> Result, Error> { let result = self.receive(); - if result.is_err() { - self.fail(); - } - result + self.terminal_result(result) } fn info(&self) -> &DecoderInfo { &self.info } + + fn settle_native_hevc_refusal(&mut self, refusal: Error) -> Error { + match self.terminal_result::(Err(refusal)) { + Err(error) => error, + Ok(never) => match never {}, + } + } + + fn take_native_hevc_terminal_output( + &mut self, + ) -> Option { + if !matches!(self.state, State::Failed(_)) { + return None; + } + let State::Failed(error) = std::mem::replace(&mut self.state, State::Ended) else { + return None; + }; + Some(coreav_decode::native_hevc_private::TerminalOutput { + frames: std::mem::take(&mut self.output.ready), + error, + }) + } } fn invalid(message: &'static str) -> Error { diff --git a/crates/coreav-backend-hevc/src/output.rs b/crates/coreav-backend-hevc/src/output.rs index a93a8bd01..0d639f6d9 100644 --- a/crates/coreav-backend-hevc/src/output.rs +++ b/crates/coreav-backend-hevc/src/output.rs @@ -1,22 +1,20 @@ use std::{collections::VecDeque, time::Instant}; +use coreav_decode::native_hevc_private::OutputFrame; use coreav_decode::{Error, Limits}; use coreav_frame::{Chroma, ChromaLocation, ColorInfo, Frame, Picture, PixelFormat, Pixels, Plane}; -use coreav_hevc::{DecodedPicture, OutputMemory}; +use coreav_hevc::{ + DecodedPicture, IntraPlane, OutputMemory, PictureInfo, + native_rows::{OutputHeader, SinkCost}, +}; use crate::{check_deadline, codec_error, invalid, limit}; -struct Entry { - frame: Frame, - bytes: u64, - memory: usize, -} - pub(super) struct Output { pub(super) limits: Limits, deadline: Instant, color: ColorInfo, - ready: VecDeque, + pub(super) ready: VecDeque, pending: u64, pending_memory: usize, decoded: u64, @@ -53,16 +51,10 @@ impl Output { self.ready.is_empty() } - pub(super) fn clear(&mut self) { - self.ready = VecDeque::new(); - self.pending = 0; - self.pending_memory = 0; - } - pub(super) fn retained_bytes(&self) -> Result { self.ready .capacity() - .checked_mul(core::mem::size_of::()) + .checked_mul(core::mem::size_of::()) .and_then(|n| n.checked_add(self.pending_memory)) .and_then(|n| n.checked_add(self.owner_bytes)) .ok_or(limit("HEVC output memory overflow")) @@ -76,17 +68,68 @@ impl Output { }) } + pub(super) fn consume(&mut self, offset: usize) -> Result<(), Error> { + let entry = self + .ready + .get(offset) + .ok_or(invalid("missing HEVC serial output"))?; + self.pending = self + .pending + .checked_sub(entry.bytes) + .ok_or(invalid("HEVC pending byte underflow"))?; + Ok(()) + } + + pub(super) fn pop_rows(&mut self) -> Option { + self.ready.pop_front().map(|entry| { + self.pending_memory -= entry.memory; + entry.frame + }) + } + + #[cfg(test)] pub(super) fn push( &mut self, picture: DecodedPicture<'_>, memory: &mut OutputMemory<'_>, ) -> Result<(), Error> { + self.queue( + OutputHeader { + index: picture.index, + poc: picture.poc, + timestamp: picture.timestamp, + origin: picture.origin, + info: picture.info, + }, + Some(picture.planes), + memory, + false, + )?; + self.check() + } + + pub(super) fn prepare( + &mut self, + header: OutputHeader, + memory: &mut OutputMemory<'_>, + bounded: bool, + ) -> Result { + self.queue(header, None, memory, bounded) + } + + fn queue( + &mut self, + picture: OutputHeader, + sources: Option<[IntraPlane<'_>; 3]>, + memory: &mut OutputMemory<'_>, + bounded: bool, + ) -> Result { self.check()?; let origin = picture .origin .ok_or(invalid("HEVC output has no packet identity"))?; if origin & 1 != 0 { - return Ok(()); + return Ok(false); } let info = picture.info; if info.width == 0 @@ -134,16 +177,19 @@ impl Output { .filter(|n| *n <= self.limits.max_decoded_bytes) .ok_or(limit("decoded HEVC frames exceed the output budget"))?; if self.ready.len() == self.ready.capacity() { + if bounded { + return Err(codec_error(coreav_hevc::Error::Internal("output_records"))); + } let old = self .ready .capacity() - .checked_mul(core::mem::size_of::()) + .checked_mul(core::mem::size_of::()) .ok_or(limit("HEVC queue size overflow"))?; let requested = self .ready .len() .checked_add(1) - .and_then(|n| n.checked_mul(core::mem::size_of::())) + .and_then(|n| n.checked_mul(core::mem::size_of::())) .ok_or(limit("HEVC queue size overflow"))?; memory.reserve(requested).map_err(codec_error)?; self.ready @@ -152,7 +198,7 @@ impl Output { let actual = self .ready .capacity() - .checked_mul(core::mem::size_of::()) + .checked_mul(core::mem::size_of::()) .ok_or(limit("HEVC queue size overflow"))?; memory .reserve( @@ -181,22 +227,10 @@ impl Output { .ok_or(invalid("HEVC plane capacity"))?, ) .map_err(codec_error)?; - for (component, source) in picture.planes.into_iter().enumerate() { + for component in 0..3 { let scale = if component == 0 { 2usize } else { 1 }; let width = info.visible_width as usize * scale / 2; let height = info.visible_height as usize * scale / 2; - let left = info - .conformance_window - .first() - .copied() - .ok_or(invalid("HEVC crop offsets"))? as usize - * scale; - let top = info - .conformance_window - .get(2) - .copied() - .ok_or(invalid("HEVC crop offsets"))? as usize - * scale; let stride = width .checked_mul(sample_bytes) .ok_or(limit("HEVC row size overflow"))?; @@ -218,37 +252,9 @@ impl Output { .checked_add(data.capacity()) .ok_or(limit("HEVC output capacity overflow"))?; data.resize(length, 0); - for (row, destination) in data.chunks_exact_mut(stride).enumerate() { - let start = top - .checked_add(row) - .and_then(|n| n.checked_mul(source.stride)) - .and_then(|n| n.checked_add(left)) - .ok_or(limit("HEVC crop offset overflow"))?; - let end = start - .checked_add(width) - .ok_or(limit("HEVC crop offset overflow"))?; - let samples = source - .samples - .get(start..end) - .ok_or(invalid("HEVC cropped plane is short"))?; - for (sample, destination) in samples - .iter() - .zip(destination.chunks_exact_mut(sample_bytes)) - { - if u32::from(*sample) >= 1u32 << info.bit_depth { - return Err(invalid("HEVC output sample exceeds its depth")); - } - let packed = sample.to_le_bytes(); - destination.copy_from_slice( - packed - .get(..sample_bytes) - .ok_or(invalid("HEVC sample width"))?, - ); - } - } planes.push(Plane::new(data, stride)); } - let pixels = Picture::new( + let mut pixels = Picture::new( info.visible_width, info.visible_height, PixelFormat::Yuv { @@ -257,6 +263,9 @@ impl Output { }, planes, )?; + if let Some(sources) = sources { + Self::fill_planes(info, sources, &mut pixels)?; + } let vui = info.vui; let code = |value: Option| value.and_then(|n| u8::try_from(n).ok()); let color = ColorInfo { @@ -274,7 +283,7 @@ impl Output { .pending_memory .checked_add(owned) .ok_or(limit("HEVC output memory overflow"))?; - self.ready.push_back(Entry { + self.ready.push_back(OutputFrame { frame: Frame { pts: picture.timestamp, color, @@ -285,6 +294,259 @@ impl Output { }); self.pending = pending; self.decoded = decoded; - self.check() + Ok(true) + } + + pub(super) fn reserve_queue( + &mut self, + records: usize, + memory: &mut OutputMemory<'_>, + admit: &mut dyn FnMut(usize) -> coreav_hevc::Result, + ) -> coreav_hevc::Result { + if self.ready.capacity() >= records { + let accepted = admit(self.ready.capacity())?; + if accepted || self.ready.capacity() == records { + return Ok(accepted); + } + } + let record_bytes = core::mem::size_of::(); + let requested = records + .checked_mul(record_bytes) + .ok_or(coreav_hevc::Error::Limit("reconstruction_memory"))?; + if memory.reserve(requested).is_err() { + return Ok(false); + } + let mut allocation = Vec::::new(); + if allocation.try_reserve_exact(records).is_err() { + memory.release(requested)?; + return Ok(false); + } + let capacity = allocation.capacity(); + let actual = capacity + .checked_mul(record_bytes) + .ok_or(coreav_hevc::Error::Limit("reconstruction_memory"))?; + if memory.reserve(actual - requested).is_err() { + memory.release(requested)?; + return Ok(false); + } + let mut replacement = VecDeque::from(allocation); + if replacement.capacity() != capacity { + memory.release(actual)?; + return Ok(false); + } + let accepted = admit(capacity); + if !matches!(accepted, Ok(true)) { + memory.release(actual)?; + return accepted; + } + let old = self + .ready + .capacity() + .checked_mul(record_bytes) + .ok_or(coreav_hevc::Error::Limit("reconstruction_memory"))?; + while let Some(entry) = self.ready.pop_front() { + replacement.push_back(entry); + } + self.ready = replacement; + memory.release(old)?; + Ok(true) + } + + pub(super) fn qualify( + &self, + info: PictureInfo, + probe: &mut coreav_hevc::native_rows::CapacityProbe, + ) -> bool { + let Some(area) = (info.visible_width as usize).checked_mul(info.visible_height as usize) + else { + return false; + }; + let sample_bytes = if info.bit_depth == 10 { 2 } else { 1 }; + let Some(bytes) = area.checked_mul(sample_bytes) else { + return false; + }; + probe.check::(3) + && [bytes, bytes / 4] + .into_iter() + .all(|count| probe.check::(count)) + } + + pub(super) fn fill(&mut self, offset: usize, picture: DecodedPicture<'_>) -> Result<(), Error> { + let entry = self + .ready + .get_mut(offset) + .ok_or(invalid("missing prepared HEVC frame"))?; + let Pixels::Cpu(destination) = &mut entry.frame.pixels else { + return Err(invalid("prepared HEVC frame is not on CPU")); + }; + if entry.frame.pts != picture.timestamp + || ( + destination.width(), + destination.height(), + destination.format().depth(), + ) != ( + picture.info.visible_width, + picture.info.visible_height, + picture.info.bit_depth as u8, + ) + { + return Err(invalid("prepared HEVC frame identity")); + } + Self::fill_planes(picture.info, picture.planes, destination) + } + + fn fill_planes( + info: PictureInfo, + sources: [IntraPlane<'_>; 3], + destination: &mut Picture, + ) -> Result<(), Error> { + let sample_bytes = if info.bit_depth == 8 { 1 } else { 2 }; + for (component, source) in sources.into_iter().enumerate() { + let scale = if component == 0 { 2usize } else { 1 }; + let width = info.visible_width as usize * scale / 2; + let height = info.visible_height as usize * scale / 2; + let left = info + .conformance_window + .first() + .copied() + .ok_or(invalid("HEVC crop offsets"))? as usize + * scale; + let top = info + .conformance_window + .get(2) + .copied() + .ok_or(invalid("HEVC crop offsets"))? as usize + * scale; + for row in 0..height { + let start = top + .checked_add(row) + .and_then(|n| n.checked_mul(source.stride)) + .and_then(|n| n.checked_add(left)) + .ok_or(limit("HEVC crop offset overflow"))?; + let end = start + .checked_add(width) + .ok_or(limit("HEVC crop offset overflow"))?; + let samples = source + .samples + .get(start..end) + .ok_or(invalid("HEVC cropped plane is short"))?; + let target = destination + .row_mut(component, row) + .ok_or(invalid("prepared HEVC row is short"))?; + for (sample, packed) in samples.iter().zip(target.chunks_exact_mut(sample_bytes)) { + if u32::from(*sample) >= 1u32 << info.bit_depth { + return Err(invalid("HEVC output sample exceeds its depth")); + } + packed.copy_from_slice( + sample + .to_le_bytes() + .get(..sample_bytes) + .ok_or(invalid("HEVC sample width"))?, + ); + } + } + } + Ok(()) + } + + pub(super) fn truncate(&mut self, keep: usize) { + while self.ready.len() > keep { + if let Some(entry) = self.ready.pop_back() { + self.pending_memory -= entry.memory; + } + } + self.pending = self.ready.iter().map(|entry| entry.bytes).sum(); + } + + pub(super) fn cost(&self, debt: usize, frame_bytes: usize) -> coreav_hevc::Result { + if debt > self.ready.len() { + return Err(coreav_hevc::Error::Internal("output_ordinal")); + } + let mut remaining = self + .ready + .iter() + .take(debt) + .try_fold(0usize, |bytes, entry| { + bytes.checked_add(usize::try_from(entry.bytes).ok()?) + }) + .ok_or(coreav_hevc::Error::Limit("reconstruction_memory"))?; + let mut debt_peak_bytes = remaining; + for (index, entry) in self.ready.iter().take(debt).enumerate() { + remaining = remaining + .checked_sub( + usize::try_from(entry.bytes) + .map_err(|_| coreav_hevc::Error::Limit("reconstruction_memory"))?, + ) + .ok_or(coreav_hevc::Error::Internal("output_ordinal"))?; + let future = index + .checked_add(1) + .and_then(|count| count.checked_mul(frame_bytes)) + .and_then(|bytes| bytes.checked_add(remaining)) + .ok_or(coreav_hevc::Error::Limit("reconstruction_memory"))?; + debt_peak_bytes = debt_peak_bytes.max(future); + } + Ok(SinkCost { + max_packet_bytes: usize::try_from(self.limits.max_packet_bytes).unwrap_or(usize::MAX), + max_nals: super::MAX_NALS, + max_pending_bytes: self.limits.max_pending_bytes, + queue_capacity: self.ready.capacity(), + queued_frames: self.ready.len(), + record_bytes: core::mem::size_of::(), + plane_records_bytes: 3 * core::mem::size_of::(), + owner_bytes: self.owner_bytes, + debt_peak_bytes, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn packed(width: u32, height: u32) -> Result { + let width = width as usize; + let height = height as usize; + let bytes = width * height * 3 / 2; + let picture = Picture::new( + width as u32, + height as u32, + PixelFormat::Yuv { + chroma: Chroma::Cs420, + depth: 8, + }, + vec![ + Plane::new(vec![0; width * height], width), + Plane::new(vec![0; width * height / 4], width / 2), + Plane::new(vec![0; width * height / 4], width / 2), + ], + )?; + Ok(OutputFrame { + frame: Frame { + pts: None, + color: Default::default(), + pixels: Pixels::Cpu(picture), + }, + bytes: bytes as u64, + memory: bytes + 3 * core::mem::size_of::(), + }) + } + + #[test] + fn debt_capacity_covers_every_old_prefix_replaced_by_new_geometry() -> Result<(), Error> { + for (sizes, expected) in [([(2, 2), (20, 20)], 960), ([(20, 20), (2, 2)], 720)] { + let mut output = Output::new( + Limits::default(), + Instant::now() + std::time::Duration::from_secs(30), + Default::default(), + 0, + )?; + for (width, height) in sizes { + output.ready.push_back(packed(width, height)?); + } + assert_eq!(output.cost(2, 360).unwrap().debt_peak_bytes, expected); + assert_eq!(output.cost(0, 360).unwrap().debt_peak_bytes, 0); + assert!(output.cost(3, 360).is_err()); + } + Ok(()) } } diff --git a/crates/coreav-backend-hevc/src/test_support.rs b/crates/coreav-backend-hevc/src/test_support.rs new file mode 100644 index 000000000..ff0750845 --- /dev/null +++ b/crates/coreav-backend-hevc/src/test_support.rs @@ -0,0 +1,56 @@ +use coreav_hevc::{DecodeSummary, PictureDecoder, native_rows::Admission}; +use std::cell::RefCell; + +#[derive(Default)] +pub struct Report { + pub admissions: Vec, + pub summary: Option, + pub row_parallel_pictures: u64, + pub overflow: bool, +} + +thread_local! { + static REPORT: RefCell> = const { RefCell::new(None) }; +} + +pub fn begin() { + coreav_hevc::native_rows::begin_evidence(); + REPORT.with(|report| *report.borrow_mut() = Some(Report::default())); +} + +pub fn take() -> Option { + REPORT.with(|report| report.borrow_mut().take()) +} + +pub(crate) fn admitted(decoder: &PictureDecoder) { + let Some(admission) = decoder.native_row_admission() else { + return; + }; + REPORT.with(|report| { + let mut report = report.borrow_mut(); + let Some(report) = report.as_mut() else { + return; + }; + if report.admissions.last() == Some(&admission) { + return; + } + if report.admissions.len() == 1024 || report.admissions.try_reserve(1).is_err() { + report.overflow = true; + return; + } + report.admissions.push(admission); + }); +} + +pub(crate) fn completed(summary: DecodeSummary) { + let rows = coreav_hevc::native_rows::take_row_overlap(); + REPORT.with(|report| { + if let Some(report) = report.borrow_mut().as_mut() { + report.summary = Some(summary); + match rows { + Some(count) => report.row_parallel_pictures = count, + None => report.overflow = true, + } + } + }); +} diff --git a/crates/coreav-backend-hevc/src/tests.rs b/crates/coreav-backend-hevc/src/tests.rs index 1f5563a04..c9b0ff5c2 100644 --- a/crates/coreav-backend-hevc/src/tests.rs +++ b/crates/coreav-backend-hevc/src/tests.rs @@ -14,6 +14,8 @@ fn reconstruction_panics_are_backend_failures_not_media_limits() { #[path = "../../coreav-hevc/test-support/pcm.rs"] mod pcm; +#[path = "../../coreav-hevc/test-support/raster.rs"] +mod raster; #[path = "../../coreav-hevc/test-support/writer.rs"] mod writer; @@ -162,23 +164,67 @@ fn stored_terminal_zero_padding_follows_the_annex_b_rule_in_configuration_and_pa } #[test] -fn malformed_or_multi_picture_access_units_close_and_discard_queued_output() -> Result<()> { +fn malformed_or_multi_picture_access_units_close_after_valid_output() -> Result<()> { let fixture = pcm::pcm_fixture(8, 8, false)?; - for bad in [ - access_unit([&fixture.nal, &fixture.nal], false), - access_unit([fixture.nal.as_slice(), &[0, 0, 0x80]], false), + for (bad, outputs) in [ + (access_unit([&fixture.nal, &fixture.nal], false), 1), + ( + access_unit([fixture.nal.as_slice(), &[0, 0, 0x80]], false), + 0, + ), ] { let mut decoder = NativeHevc::new().open(&StreamConfig::new(Codec::Hevc), &Limits::default())?; decoder.send_packet(&packet(&access_unit(&fixture.sets, false), None, false))?; assert!(decoder.send_packet(&packet(&bad, Some(7), false)).is_err()); + for _ in 0..outputs { + assert_eq!( + decoder.receive_frame()?.ok_or("lost prior frame")?.pts, + Some(7) + ); + } terminal(&mut *decoder); } Ok(()) } #[test] -fn a_packet_after_eof_fails_and_drops_unreceived_frames() -> Result<()> { +fn polling_keeps_deferred_entropy_errors_ahead_of_later_packet_framing_errors() -> Result<()> { + let mut fixture = pcm::pcm_fixture(8, 8, false)?; + fixture.nal.pop(); + for length in [false, true] { + let mut stream = StreamConfig::new(Codec::Hevc); + if length { + stream.config = Some(configuration(&fixture.sets, 8)); + } + let data = if length { + access_unit([&fixture.nal], true) + } else { + access_unit(fixture.sets.iter().chain([&fixture.nal]), false) + }; + let mut serial = NativeHevc::with_options(NativeOptions { threads: Some(1) }) + .open(&stream, &Limits::default())?; + let expected = serial.send_packet(&packet(&data, None, false)).unwrap_err(); + let mut scheduled = NativeHevc::with_options(NativeOptions { threads: Some(2) }) + .open(&stream, &Limits::default())?; + let actual = match scheduled.send_packet(&packet(&data, None, false)) { + Err(error) => error, + Ok(()) => { + assert!(scheduled.receive_frame()?.is_none()); + scheduled + .send_packet(&packet(&[0, 0, 0, 10, 2], None, false)) + .unwrap_err() + } + }; + assert_eq!(actual.kind, expected.kind); + assert_eq!(actual.to_string(), expected.to_string()); + terminal(&mut *scheduled); + } + Ok(()) +} + +#[test] +fn a_packet_after_eof_fails_and_keeps_validated_frames() -> Result<()> { let fixture = pcm::pcm_fixture(8, 8, false)?; let data = access_unit(fixture.sets.iter().chain([&fixture.nal]), false); let mut decoder = @@ -190,10 +236,512 @@ fn a_packet_after_eof_fails_and_drops_unreceived_frames() -> Result<()> { .send_packet(&packet(&data, Some(18), false)) .is_err() ); + assert_eq!( + decoder.receive_frame()?.ok_or("lost EOF frame")?.pts, + Some(17) + ); terminal(&mut *decoder); Ok(()) } +#[test] +fn checked_packet_refusal_settles_earlier_entropy_failure() -> Result<()> { + let mut fixture = pcm::pcm_fixture(8, 8, false)?; + fixture.nal.pop(); + let mut deferred = 0; + for length in [false, true] { + let mut stream = StreamConfig::new(Codec::Hevc); + if length { + stream.config = Some(configuration(&fixture.sets, 8)); + } + let data = if length { + access_unit([&fixture.nal], true) + } else { + access_unit(fixture.sets.iter().chain([&fixture.nal]), false) + }; + let limits = Limits { + max_packet_bytes: data.len() as u64 + 8, + ..Limits::default() + }; + let serial = NativeHevc::with_options(NativeOptions { threads: Some(1) }); + let mut serial = coreav_decode::open(&stream, &Policy::default(), &limits, &[&serial])?; + let expected = serial + .decoder + .send_packet(&packet(&data, None, false)) + .unwrap_err(); + assert_eq!(expected.kind, ErrorKind::InvalidData); + let oversized = vec![0; limits.max_packet_bytes as usize + 1]; + for threads in [1, 2, 4, 8] { + let native = NativeHevc::with_options(NativeOptions { + threads: Some(threads), + }); + let mut opened = coreav_decode::open(&stream, &Policy::default(), &limits, &[&native])?; + let actual = match opened.decoder.send_packet(&packet(&data, None, false)) { + Err(error) => error, + Ok(()) => { + deferred += 1; + assert!(opened.decoder.receive_frame()?.is_none()); + opened + .decoder + .send_packet(&packet(&oversized, None, false)) + .unwrap_err() + } + }; + assert_eq!(actual.kind, expected.kind); + assert_eq!(actual.to_string(), expected.to_string()); + terminal(&mut *opened.decoder); + } + } + if std::thread::available_parallelism().is_ok_and(|workers| workers.get() > 1) { + assert!(deferred > 0); + } + Ok(()) +} + +#[test] +fn terminal_failures_preserve_validated_frames_through_public_open() -> Result<()> { + let mut serial_errors = std::collections::BTreeMap::new(); + for (depth, length, receive_early) in [ + (8, false, false), + (8, false, true), + (8, true, false), + (8, true, true), + (10, false, false), + (10, false, true), + (10, true, false), + (10, true, true), + ] { + let fixture = pcm::pcm_fixture(depth, depth, false)?; + let sample_bytes = if depth == 8 { 1 } else { 2 }; + let mut stream = StreamConfig::new(Codec::Hevc); + if length { + stream.config = Some(configuration(&fixture.sets, depth)); + } + let first = if length { + access_unit([&fixture.nal], true) + } else { + access_unit(fixture.sets.iter().chain([&fixture.nal]), false) + }; + let next = access_unit([&fixture.nal], length); + for threads in [1, 2, 4, 8] { + for checked in [false, true] { + for fault in 0..5 { + let limits = Limits { + max_packet_bytes: first.len() as u64 + 8, + max_decoded_bytes: if fault == 2 { + 96 * sample_bytes as u64 + } else { + 4 << 30 + }, + ..Limits::default() + }; + let native = NativeHevc::with_options(NativeOptions { + threads: Some(threads), + }); + let mut decoder = if checked { + coreav_decode::open(&stream, &Policy::default(), &limits, &[&native])? + .decoder + } else { + native.open(&stream, &limits)? + }; + decoder.send_packet(&packet(&first, Some(11), false))?; + decoder.send_packet(&packet(&next, Some(22), false))?; + let mut frames = Vec::new(); + if receive_early { + let early = decoder.receive_frame()?; + if threads == 1 { + assert!(early.is_some(), "missing serial early frame"); + } + if let Some(frame) = early { + frames.push(frame); + } + } + if fault == 3 { + decoder.send_eof()?; + } + let bad = match fault { + 0 => access_unit([[0, 0, 0x80]], length), + 1 => vec![0; limits.max_packet_bytes as usize + 1], + 4 => access_unit([&fixture.nal[..fixture.nal.len() - 1]], length), + _ => next.clone(), + }; + let error = match decoder.send_packet(&packet(&bad, Some(33), false)) { + Err(error) => error, + Ok(()) if fault == 4 => decoder.send_eof().unwrap_err(), + Ok(()) => return Err("terminal fixture did not fail".into()), + }; + assert_eq!( + error.kind, + if matches!(fault, 1 | 2) { + ErrorKind::Limit + } else { + ErrorKind::InvalidData + } + ); + assert_eq!( + serial_errors + .entry((depth, length, fault)) + .or_insert_with(|| error.clone()), + &error + ); + let count = if matches!(fault, 3 | 4) { 2 } else { 1 }; + while frames.len() < count { + frames.push(decoder.receive_frame()?.ok_or("lost terminal frame")?); + } + for (frame, pts) in frames.into_iter().zip([11, 22]) { + assert_eq!(frame.pts, Some(pts)); + assert_eq!((frame.width(), frame.height()), (8, 8)); + for (plane, samples) in frame.to_cpu()?.planes().iter().zip(&fixture.planes) + { + let expected: Vec = samples + .iter() + .flat_map(|value| { + value.to_le_bytes().into_iter().take(sample_bytes) + }) + .collect(); + assert_eq!(plane.data(), expected); + } + } + assert_eq!(decoder.receive_frame().unwrap_err(), error); + assert_eq!(decoder.send_eof().unwrap_err(), error); + assert_eq!( + decoder + .send_packet(&packet(&next, None, false)) + .unwrap_err(), + error + ); + } + } + } + } + Ok(()) +} + +fn native_with_limits_for_test(limits: &Limits, threads: usize) -> Result { + let name = DECODER_NAME.to_owned(); + let output = Output::new( + limits.clone(), + limits.deadline(), + Default::default(), + name.capacity(), + )?; + let mut core = PictureDecoder::new(DecodeLimits { + max_memory_bytes: limits.max_memory_bytes as usize, + ..DecodeLimits::default() + }) + .with_threads(threads); + core.set_external_memory(output.retained_bytes()?)?; + Ok(NativeDecoder { + decoder: Some(core), + info: DecoderInfo { + backend: BackendKind::Native, + isolation: Isolation::InProcess, + decoder: name, + }, + framing: Framing::AnnexB, + output, + state: State::Active, + next_origin: 0, + }) +} + +#[test] +fn native_row_receives_are_bounded_serial_prefixes_for_every_short_cadence() -> Result<()> { + let fixture = pcm::pcm_fixture(8, 8, false)?; + let first = access_unit(fixture.sets.iter().chain([&fixture.nal]), false); + let next = access_unit([&fixture.nal], false); + for threads in [1usize, 2, 4, 8] { + let effective = threads.min(std::thread::available_parallelism().map_or(1, usize::from)); + for cadence in 0u32..64 { + let limits = Limits { + max_packet_bytes: first.len() as u64 + 8, + max_pending_bytes: 6 * 96, + max_memory_bytes: 128 << 20, + ..Limits::default() + }; + let mut decoder = native_with_limits_for_test(&limits, threads)?; + let mut returned = 0usize; + let mut serial_received = 0usize; + let mut ring_capacity = None; + for index in 0..6 { + decoder.send_packet(&packet( + if index == 0 { &first } else { &next }, + Some(index as i64), + false, + ))?; + assert_eq!( + decoder + .decoder + .as_ref() + .is_some_and(PictureDecoder::has_native_rows), + effective > 1 + ); + if effective > 1 { + let capacity = decoder.output.ready.capacity(); + assert!(capacity >= 6 + effective); + assert_eq!(*ring_capacity.get_or_insert(capacity), capacity); + } + if cadence & (1 << index) != 0 { + serial_received += usize::from(serial_received < index); + if let Some(frame) = decoder.receive_frame()? { + assert_eq!(frame.pts, Some(returned as i64)); + for (plane, samples) in frame.to_cpu()?.planes().iter().zip(&fixture.planes) + { + assert_eq!( + plane.data(), + samples.iter().map(|n| *n as u8).collect::>() + ); + } + returned += 1; + } + assert!(returned <= serial_received); + assert!( + serial_received - returned < effective, + "threads={threads}, cadence={cadence}, index={index}" + ); + } + } + decoder.send_eof()?; + while let Some(frame) = decoder.receive_frame()? { + assert_eq!(frame.pts, Some(returned as i64)); + returned += 1; + } + assert_eq!(returned, 6, "threads={threads}, cadence={cadence}"); + assert!(decoder.output.ready.is_empty()); + assert!(decoder.decoder.is_none()); + assert!(decoder.receive_frame()?.is_none()); + } + } + Ok(()) +} + +#[test] +fn later_geometry_can_reduce_future_lag_to_zero_and_retire_old_debt() -> Result<()> { + if std::thread::available_parallelism().map_or(1, usize::from) < 2 { + return Ok(()); + } + let tiny = pcm::pcm_fixture(8, 8, false)?; + let first = access_unit(tiny.sets.iter().chain([&tiny.nal]), false); + let next = access_unit([&tiny.nal], false); + let tall = raster::stream(false)?; + let grown = access_unit(&tall[..35], false); + let grown_next = access_unit(&tall[35..], false); + let mut zero_lag = false; + for memory in (12u64 << 20..28u64 << 20).step_by(256 << 10) { + let limits = Limits { + max_packet_bytes: grown.len() as u64, + max_pending_bytes: 4 * 16 * 512 * 3 / 2, + max_memory_bytes: memory, + ..Limits::default() + }; + let mut decoder = native_with_limits_for_test(&limits, 2)?; + decoder.send_packet(&packet(&first, Some(0), false))?; + decoder.send_packet(&packet(&next, Some(1), false))?; + if !decoder + .decoder + .as_ref() + .is_some_and(PictureDecoder::has_native_rows) + { + continue; + } + assert!(decoder.receive_frame()?.is_none()); + assert!(decoder.receive_frame()?.is_none()); + decoder.send_packet(&packet(&grown, Some(2), false))?; + for pts in [0, 1] { + let frame = decoder.receive_frame()?.ok_or("lost old-geometry output")?; + assert_eq!( + (frame.pts, frame.width(), frame.height()), + (Some(pts), 8, 8) + ); + for (plane, samples) in frame.to_cpu()?.planes().iter().zip(&tiny.planes) { + assert_eq!( + plane.data(), + samples.iter().map(|n| *n as u8).collect::>() + ); + } + } + assert!(decoder.receive_frame()?.is_none()); + decoder.send_packet(&packet(&grown_next, Some(3), false))?; + let retired = !decoder + .decoder + .as_ref() + .is_some_and(PictureDecoder::has_native_rows); + decoder.send_eof()?; + for picture in 0..2 { + let frame = decoder.receive_frame()?.ok_or("lost new-geometry output")?; + assert_eq!( + (frame.pts, frame.width(), frame.height()), + (Some(picture + 2), 16, 512) + ); + for (component, plane) in frame.to_cpu()?.planes().iter().enumerate() { + let band = if component == 0 { 256 } else { 64 }; + for (row, bytes) in plane.data().chunks_exact(band).enumerate() { + assert!( + bytes + .iter() + .all(|byte| *byte == 37 + picture as u8 * 5 + row as u8) + ); + } + } + } + assert!(decoder.receive_frame()?.is_none()); + if retired { + zero_lag = true; + break; + } + } + assert!(zero_lag, "no active continuation reduced lag to zero"); + Ok(()) +} + +#[test] +fn later_envelope_refusal_keeps_old_outputs_and_never_emits_the_refused_picture() -> Result<()> { + if std::thread::available_parallelism().map_or(1, usize::from) < 2 { + return Ok(()); + } + let tiny = pcm::pcm_fixture(8, 8, false)?; + let first = access_unit(tiny.sets.iter().chain([&tiny.nal]), false); + let next = access_unit([&tiny.nal], false); + let tall = raster::stream_geometry(false, 512, 16)?; + let grown = access_unit(&tall[..515], false); + let bytes = 16 * 8192 * 3 / 2; + let mut refused = false; + for memory in (12u64 << 20..32u64 << 20).step_by(64 << 10) { + let limits = Limits { + max_packet_bytes: grown.len() as u64, + max_pending_bytes: bytes + 192, + max_memory_bytes: memory, + ..Limits::default() + }; + let mut decoder = native_with_limits_for_test(&limits, 2)?; + decoder.send_packet(&packet(&first, Some(0), false))?; + decoder.send_packet(&packet(&next, Some(1), false))?; + if !decoder + .decoder + .as_ref() + .is_some_and(PictureDecoder::has_native_rows) + { + continue; + } + assert!(decoder.receive_frame()?.is_none()); + decoder.send_packet(&packet(&access_unit([&tall[1]], false), None, false))?; + decoder.send_packet(&packet(&access_unit([&tiny.sets[1]], false), None, false))?; + let Err(error) = decoder.send_packet(&packet(&grown, Some(2), false)) else { + continue; + }; + assert_eq!(error.kind, ErrorKind::Limit); + assert!(error.message.contains("row_admission"), "{error}"); + for pts in [0, 1] { + let frame = decoder + .receive_frame()? + .ok_or("lost pre-admission output")?; + assert_eq!( + (frame.pts, frame.width(), frame.height()), + (Some(pts), 8, 8) + ); + for (plane, samples) in frame.to_cpu()?.planes().iter().zip(&tiny.planes) { + assert_eq!( + plane.data(), + samples.iter().map(|n| *n as u8).collect::>() + ); + } + } + assert_eq!(decoder.receive_frame().unwrap_err(), error); + assert_eq!(decoder.send_eof().unwrap_err(), error); + let mut serial = native_with_limits_for_test(&limits, 1)?; + serial.send_packet(&packet(&first, Some(0), false))?; + serial.send_packet(&packet(&next, Some(1), false))?; + serial.send_packet(&packet(&grown, Some(2), false))?; + serial.send_eof()?; + for pts in 0..3 { + assert_eq!( + serial + .receive_frame()? + .ok_or("missing serial continuation")? + .pts, + Some(pts) + ); + } + assert!(serial.receive_frame()?.is_none()); + let mut without_debt = native_with_limits_for_test(&limits, 2)?; + without_debt.send_packet(&packet(&first, Some(0), false))?; + without_debt.send_packet(&packet(&next, Some(1), false))?; + without_debt.send_packet(&packet(&grown, Some(2), false))?; + without_debt.send_eof()?; + for pts in 0..3 { + assert_eq!( + without_debt + .receive_frame()? + .ok_or("missing zero-debt continuation")? + .pts, + Some(pts) + ); + } + assert!(without_debt.receive_frame()?.is_none()); + refused = true; + break; + } + assert!( + refused, + "no later continuation exceeded the admitted envelope" + ); + Ok(()) +} + +#[test] +fn terminal_handoff_moves_only_packed_frames_after_decoder_teardown() -> Result<()> { + let fixture = pcm::pcm_fixture(10, 10, false)?; + let data = access_unit(fixture.sets.iter().chain([&fixture.nal]), false); + let mut source = NativeHevc::new().open(&StreamConfig::new(Codec::Hevc), &Limits::default())?; + source.send_packet(&packet(&data, Some(51), false))?; + source.send_eof()?; + let frame = source.receive_frame()?.ok_or("missing packed frame")?; + let pixels = frame.to_cpu()?; + let memory = pixels.planes().iter().map(|plane| plane.data().len()).sum(); + let first_sample = pixels.planes()[0].data().as_ptr(); + let mut output = Output::new( + Limits::default(), + Limits::default().deadline(), + frame.color, + 0, + )?; + output + .ready + .push_back(coreav_decode::native_hevc_private::OutputFrame { + frame, + bytes: 192, + memory, + }); + let entries = output.ready.as_slices().0.as_ptr(); + let mut decoder = NativeDecoder { + decoder: Some(PictureDecoder::new(DecodeLimits::default())), + info: source.info().clone(), + framing: Framing::AnnexB, + output, + state: State::Active, + next_origin: 0, + }; + let error = Error::new(ErrorKind::Timeout, "terminal timeout"); + assert_eq!( + decoder.terminal_result::<()>(Err(error.clone())), + Err(error.clone()) + ); + assert!(decoder.decoder.is_none()); + let mut terminal = decoder + .take_native_hevc_terminal_output() + .ok_or("no handoff")?; + assert_eq!(terminal.frames.as_slices().0.as_ptr(), entries); + assert!(decoder.output.is_empty()); + drop(decoder); + assert_eq!(terminal.frames[0].bytes, 192); + assert_eq!(terminal.frames[0].memory, memory); + let frame = terminal.receive_frame()?.ok_or("lost packed frame")?; + assert_eq!(frame.to_cpu()?.planes()[0].data().as_ptr(), first_sample); + assert_eq!(frame.pts, Some(51)); + assert_eq!(terminal.receive_frame().unwrap_err(), error); + Ok(()) +} + #[test] fn lowered_resource_limits_cannot_be_bypassed_by_direct_backend_open() -> Result<()> { let fixture = pcm::pcm_fixture(8, 8, false)?; diff --git a/crates/coreav-backend-hevc/tests/memory.rs b/crates/coreav-backend-hevc/tests/memory.rs new file mode 100644 index 000000000..c0f5a5013 --- /dev/null +++ b/crates/coreav-backend-hevc/tests/memory.rs @@ -0,0 +1,225 @@ +//! Counts every native decoder thread on generated pictures, without an external oracle. +#![cfg(feature = "native-hevc")] + +use coreav_backend_hevc::{NativeHevc, NativeOptions}; +use coreav_counting_alloc::{CountingAllocator, measure_process, report}; +use coreav_decode::{Codec, Decoder, Error, ErrorKind, Limits, Packet, Policy, StreamConfig}; +use coreav_frame::Pixels; + +type Result = std::result::Result>; + +#[path = "../../coreav-hevc/test-support/pcm.rs"] +mod pcm; +#[path = "../../coreav-hevc/test-support/raster.rs"] +mod raster; +#[path = "../../coreav-hevc/test-support/writer.rs"] +mod writer; + +#[global_allocator] +static ALLOCATOR: CountingAllocator = CountingAllocator; + +const FRAME_BYTES: u64 = 16 * 512 * 3 / 2; +const HEAP_BOUND: u64 = 32 * 1024 * 1024; +const STACK_BYTES: u64 = 2 * 1024 * 1024; + +#[derive(Debug, Default, PartialEq, Eq)] +struct Snapshot { + frames: [Option<(Option, u64)>; 2], + error: Option<(ErrorKind, u64)>, +} + +fn digest(bytes: &[u8], seed: u64) -> u64 { + bytes.iter().fold(seed, |value, byte| { + value.wrapping_mul(257).wrapping_add(u64::from(*byte)) + }) +} + +fn annex_b(nals: &[Vec]) -> Vec { + let mut packet = Vec::new(); + for nal in nals { + packet.extend_from_slice(&[0, 0, 0, 1]); + packet.extend_from_slice(nal); + } + packet +} + +fn drain( + decoder: &mut dyn Decoder, + snapshot: &mut Snapshot, + geometry: (u32, u32, u8), +) -> std::result::Result<(), Error> { + while let Some(frame) = decoder.receive_frame()? { + let Pixels::Cpu(picture) = frame.pixels else { + return Err(Error::new( + ErrorKind::Backend, + "generated HEVC frame is not on CPU", + )); + }; + if (picture.width(), picture.height(), picture.format().depth()) != geometry { + return Err(Error::new( + ErrorKind::Backend, + "generated HEVC frame geometry changed", + )); + } + let hash = picture + .planes() + .iter() + .fold(0, |hash, plane| digest(plane.data(), hash)); + let entry = snapshot + .frames + .iter_mut() + .find(|entry| entry.is_none()) + .ok_or_else(|| Error::new(ErrorKind::Backend, "extra generated HEVC frame"))?; + *entry = Some((frame.pts, hash)); + } + Ok(()) +} + +fn decode( + packets: &[Vec], + threads: usize, + limits: &Limits, + eager: bool, + geometry: (u32, u32, u8), +) -> Snapshot { + let mut snapshot = Snapshot::default(); + let backend = NativeHevc::with_options(NativeOptions { + threads: Some(threads), + }); + let result = (|| { + let mut decoder = coreav_decode::open( + &StreamConfig::new(Codec::Hevc), + &Policy::default(), + limits, + &[&backend], + )? + .decoder; + let result = (|| { + for (index, data) in packets.iter().enumerate() { + decoder.send_packet(&Packet { + data, + pts: Some(index as i64), + dts: Some(index as i64), + keyframe: true, + discard: false, + })?; + if eager { + drain(&mut *decoder, &mut snapshot, geometry)?; + } + } + decoder.send_eof()?; + drain(&mut *decoder, &mut snapshot, geometry) + })(); + if result.is_err() { + let _ = drain(&mut *decoder, &mut snapshot, geometry); + } + result + })(); + if let Err(error) = result { + snapshot.error = Some((error.kind, digest(error.message.as_bytes(), 0))); + } + snapshot +} + +#[test] +fn generated_raster_native_open_bounds_all_thread_heap_and_preserves_cold_admission() -> Result<()> +{ + let tiny = pcm::pcm_fixture(8, 8, false)?; + let packets = [annex_b( + &tiny.sets.into_iter().chain([tiny.nal]).collect::>(), + )]; + let tiny_limits = Limits { + max_memory_bytes: HEAP_BOUND, + ..Limits::default() + }; + let (actual, counters) = + measure_process(|| decode(&packets, 1, &tiny_limits, false, (8, 8, 8))); + let hash = tiny.planes.iter().flatten().fold(0u64, |value, sample| { + value.wrapping_mul(257).wrapping_add(u64::from(*sample)) + }); + assert_eq!( + actual, + Snapshot { + frames: [Some((Some(0), hash)), None], + error: None + } + ); + report("hevc-generated-tiny-cold", &counters); + assert!(counters.calls() > 0); + assert_eq!(counters.trough_bytes, 0); + assert_eq!(counters.live_bytes, 0); + assert!(counters.peak_bytes <= HEAP_BOUND, "{counters}"); + for predicted in [false, true] { + let nals = raster::stream(predicted)?; + let packets = [annex_b(&nals[..35]), annex_b(&nals[35..])]; + let limits = Limits { + max_packet_bytes: packets.iter().map(Vec::len).max().unwrap_or(0) as u64, + max_frame_bytes: FRAME_BYTES, + max_pending_bytes: 4 * FRAME_BYTES, + max_memory_bytes: HEAP_BOUND, + ..Limits::default() + }; + let serial = decode(&packets, 1, &limits, false, (16, 512, 8)); + assert!(serial.error.is_none(), "{serial:?}"); + assert!(serial.frames.iter().all(Option::is_some)); + let (mut low, mut high) = (0, HEAP_BOUND); + while low + 1 < high { + let middle = low + (high - low) / 2; + if decode( + &packets, + 1, + &Limits { + max_memory_bytes: middle, + ..limits.clone() + }, + false, + (16, 512, 8), + ) + .error + .is_none() + { + high = middle; + } else { + low = middle; + } + } + for memory in [low, high, high + 1] { + let bounded = Limits { + max_memory_bytes: memory, + ..limits.clone() + }; + let expected = decode(&packets, 1, &bounded, false, (16, 512, 8)); + for threads in [2, 4, 8] { + assert_eq!( + decode(&packets, threads, &bounded, false, (16, 512, 8)), + expected, + "predicted={predicted} memory={memory} threads={threads}" + ); + } + } + for threads in [1usize, 2, 4, 8] { + for eager in [false, true] { + let stacks = (threads as u64 - 1) * STACK_BYTES; + let bounded = Limits { + max_memory_bytes: HEAP_BOUND + stacks, + ..limits.clone() + }; + let (actual, counters) = + measure_process(|| decode(&packets, threads, &bounded, eager, (16, 512, 8))); + report( + &format!("hevc-raster-predicted-{predicted}-threads-{threads}-eager-{eager}"), + &counters, + ); + assert_eq!(actual, serial); + assert!(counters.calls() > 0); + assert_eq!(counters.trough_bytes, 0); + assert_eq!(counters.live_bytes, 0); + assert!( + counters.peak_bytes + stacks <= bounded.max_memory_bytes, + "{counters}" + ); + } + } + } + Ok(()) +} diff --git a/crates/coreav-decode/Cargo.toml b/crates/coreav-decode/Cargo.toml index 5fd75f936..0bbfee58b 100644 --- a/crates/coreav-decode/Cargo.toml +++ b/crates/coreav-decode/Cargo.toml @@ -8,6 +8,9 @@ license.workspace = true repository.workspace = true publish = false +[features] +native-hevc-private = [] + [dependencies] coreav-frame = { path = "../coreav-frame" } diff --git a/crates/coreav-decode/src/lib.rs b/crates/coreav-decode/src/lib.rs index 6f2f5d177..e19afd5da 100644 --- a/crates/coreav-decode/src/lib.rs +++ b/crates/coreav-decode/src/lib.rs @@ -27,6 +27,10 @@ mod buffer; mod error; mod stream; +#[cfg(feature = "native-hevc-private")] +#[doc(hidden)] +pub mod native_hevc_private; + use std::time::{Duration, Instant}; pub use buffer::{ @@ -53,6 +57,18 @@ pub trait Decoder: Send { /// Which backend this is. fn info(&self) -> &DecoderInfo; + + #[cfg(feature = "native-hevc-private")] + #[doc(hidden)] + fn settle_native_hevc_refusal(&mut self, refusal: Error) -> Error { + refusal + } + + #[cfg(feature = "native-hevc-private")] + #[doc(hidden)] + fn take_native_hevc_terminal_output(&mut self) -> Option { + None + } } /// A kind of backend, in the default order of preference. @@ -378,6 +394,8 @@ pub fn open( info: decoder.info().clone(), decoder: Some(decoder), limits: limits.clone(), + #[cfg(feature = "native-hevc-private")] + terminal: None, }); return Ok(Opened { decoder, skipped }); } @@ -430,6 +448,8 @@ struct Checked { decoder: Option>, info: DecoderInfo, limits: Limits, + #[cfg(feature = "native-hevc-private")] + terminal: Option, } impl Checked { @@ -437,11 +457,30 @@ impl Checked { &mut self, operation: impl FnOnce(&mut dyn Decoder, &Limits) -> Result, ) -> Result { + #[cfg(feature = "native-hevc-private")] + if let Some(terminal) = &self.terminal { + return Err(terminal.error.clone()); + } let decoder = self .decoder .as_mut() .ok_or_else(|| Error::new(ErrorKind::InvalidData, "the decode request failed"))?; let result = operation(decoder.as_mut(), &self.limits); + #[cfg(feature = "native-hevc-private")] + let result = result.map_err(|error| { + if error.kind == ErrorKind::Limit { + decoder.settle_native_hevc_refusal(error) + } else { + error + } + }); + #[cfg(feature = "native-hevc-private")] + if result.is_err() { + self.terminal = decoder.take_native_hevc_terminal_output(); + if self.terminal.is_some() { + self.decoder = None; + } + } if result .as_ref() .is_err_and(|error| error.kind == ErrorKind::Limit) @@ -465,13 +504,29 @@ impl Decoder for Checked { } fn receive_frame(&mut self) -> Result, Error> { - self.call(|decoder, limits| { + #[cfg(feature = "native-hevc-private")] + if let Some(terminal) = &mut self.terminal { + return terminal.receive_frame(); + } + #[cfg(feature = "native-hevc-private")] + let mut frame_refused = false; + let result = self.call(|decoder, limits| { let frame = decoder.receive_frame()?; if let Some(frame) = &frame { - limits.check_frame(frame.width(), frame.height(), packed_bytes(frame))?; + let result = limits.check_frame(frame.width(), frame.height(), packed_bytes(frame)); + #[cfg(feature = "native-hevc-private")] + let result = result.inspect_err(|_| frame_refused = true); + result?; } Ok(frame) - }) + }); + #[cfg(feature = "native-hevc-private")] + if frame_refused { + if let Some(terminal) = &mut self.terminal { + terminal.frames = Default::default(); + } + } + result } fn info(&self) -> &DecoderInfo { diff --git a/crates/coreav-decode/src/native_hevc_private.rs b/crates/coreav-decode/src/native_hevc_private.rs new file mode 100644 index 000000000..a44d55123 --- /dev/null +++ b/crates/coreav-decode/src/native_hevc_private.rs @@ -0,0 +1,171 @@ +use std::collections::VecDeque; + +use crate::{Error, Frame}; + +pub struct OutputFrame { + pub frame: Frame, + pub bytes: u64, + pub memory: usize, +} + +pub struct TerminalOutput { + pub frames: VecDeque, + pub error: Error, +} + +impl TerminalOutput { + pub fn receive_frame(&mut self) -> Result, Error> { + match self.frames.pop_front() { + Some(entry) => Ok(Some(entry.frame)), + None => Err(self.error.clone()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{BackendKind, Checked, Decoder, DecoderInfo, ErrorKind, Isolation, Limits, Packet}; + use coreav_frame::{Chroma, Picture, PixelFormat, Pixels, Plane}; + use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }; + + struct Live { + info: DecoderInfo, + dropped: Arc, + terminal: Option, + frame: Option, + } + + impl Drop for Live { + fn drop(&mut self) { + self.dropped.store(true, Ordering::SeqCst); + } + } + + impl Decoder for Live { + fn send_packet(&mut self, _: &Packet<'_>) -> Result<(), Error> { + Err(self.terminal.as_ref().unwrap().error.clone()) + } + + fn send_eof(&mut self) -> Result<(), Error> { + panic!("live decoder used after handoff") + } + + fn receive_frame(&mut self) -> Result, Error> { + Ok(Some(self.frame.take().expect("unexpected live receive"))) + } + + fn info(&self) -> &DecoderInfo { + &self.info + } + + fn take_native_hevc_terminal_output(&mut self) -> Option { + self.terminal.take() + } + } + + fn packed_frame(pts: i64) -> OutputFrame { + let picture = Picture::new( + 2, + 2, + PixelFormat::Yuv { + chroma: Chroma::Cs420, + depth: 8, + }, + vec![ + Plane::new(vec![1; 4], 2), + Plane::new(vec![2], 1), + Plane::new(vec![3], 1), + ], + ) + .unwrap(); + OutputFrame { + frame: Frame { + pts: Some(pts), + color: Default::default(), + pixels: Pixels::Cpu(picture), + }, + bytes: 6, + memory: 6 + 3 * core::mem::size_of::(), + } + } + + #[test] + fn checked_terminal_handoff_drops_backend_before_returning_failure() { + for kind in [ErrorKind::Limit, ErrorKind::InvalidData, ErrorKind::Timeout] { + let info = DecoderInfo { + backend: BackendKind::Native, + isolation: Isolation::InProcess, + decoder: "terminal fixture".into(), + }; + let dropped = Arc::new(AtomicBool::new(false)); + let error = Error::new(kind, "latched failure"); + let output = TerminalOutput { + frames: [packed_frame(9)].into(), + error: error.clone(), + }; + let mut checked = Checked { + decoder: Some(Box::new(Live { + info: info.clone(), + dropped: Arc::clone(&dropped), + terminal: Some(output), + frame: None, + })), + info, + limits: Limits::default(), + terminal: None, + }; + let packet = Packet { + data: &[], + pts: None, + dts: None, + keyframe: false, + discard: false, + }; + assert_eq!(checked.send_packet(&packet), Err(error.clone())); + assert!(dropped.load(Ordering::SeqCst)); + assert!(checked.decoder.is_none()); + assert_eq!(checked.send_eof(), Err(error.clone())); + assert_eq!(checked.receive_frame().unwrap().unwrap().pts, Some(9)); + assert_eq!(checked.receive_frame().unwrap_err(), error); + assert_eq!(checked.info().decoder, "terminal fixture"); + } + } + + #[test] + fn wrapper_frame_refusal_discards_later_terminal_output() { + let limits = Limits { + max_width: 1, + ..Limits::default() + }; + let error = limits.check_frame(2, 2, 6).unwrap_err(); + let info = DecoderInfo { + backend: BackendKind::Native, + isolation: Isolation::InProcess, + decoder: "terminal fixture".into(), + }; + let dropped = Arc::new(AtomicBool::new(false)); + let mut checked = Checked { + decoder: Some(Box::new(Live { + info: info.clone(), + dropped: Arc::clone(&dropped), + frame: Some(packed_frame(1).frame), + terminal: Some(TerminalOutput { + frames: [packed_frame(2)].into(), + error: error.clone(), + }), + })), + info, + limits, + terminal: None, + }; + assert_eq!(checked.receive_frame().unwrap_err(), error); + assert!(dropped.load(Ordering::SeqCst)); + assert!(checked.decoder.is_none()); + assert!(checked.terminal.as_ref().unwrap().frames.is_empty()); + assert_eq!(checked.receive_frame().unwrap_err(), error); + } +} diff --git a/crates/coreav-hevc/Cargo.toml b/crates/coreav-hevc/Cargo.toml index 621122471..f1ac167c0 100644 --- a/crates/coreav-hevc/Cargo.toml +++ b/crates/coreav-hevc/Cargo.toml @@ -12,6 +12,7 @@ publish = false [features] default = [] native-hevc = ["dep:sha2", "dep:md-5", "dep:coreav-kernels-hevc", "coreav-kernels-hevc/native-hevc"] +native-hevc-rows-private = ["native-hevc"] scalar = ["coreav-kernels-hevc?/scalar"] test-support = [] diff --git a/crates/coreav-hevc/examples/common/counters.rs b/crates/coreav-hevc/examples/common/counters.rs new file mode 100644 index 000000000..fd44bc966 --- /dev/null +++ b/crates/coreav-hevc/examples/common/counters.rs @@ -0,0 +1,22 @@ +pub fn inter_stats(stats: coreav_hevc::InterStats) -> serde_json::Value { + serde_json::json!({"skip_units":stats.skip_units,"prediction_units":stats.prediction_units, + "p_units":stats.p_units,"b_units":stats.b_units,"bi_units":stats.bi_units, + "merge_units":stats.merge_units,"amvp_lists":stats.amvp_lists, + "temporal_units":stats.temporal_units,"weighted_units":stats.weighted_units, + "weighted_changed_samples":stats.weighted_changed_samples, + "fractional_luma_units":stats.fractional_luma_units, + "fractional_chroma_units":stats.fractional_chroma_units, + "negative_motion_units":stats.negative_motion_units,"long_term_units":stats.long_term_units, + "list_modification_units":stats.list_modification_units,"partitions":stats.partitions, + "boundary_strengths":stats.boundary_strengths,"boundary_changed_samples":stats.boundary_changed_samples}) +} + +pub fn header_stats(headers: coreav_hevc::Summary) -> serde_json::Value { + serde_json::json!({"nals":headers.nals,"pictures":headers.pictures, + "warnings":{"profile_bit_depth":headers.warnings.profile_bit_depth, + "invalid_sample_aspect_ratio":headers.warnings.invalid_sample_aspect_ratio, + "temporal_nesting_nal_type":headers.warnings.temporal_nesting_nal_type}, + "output_flag_pictures":headers.output_flag_pictures,"output_pictures":headers.output_pictures, + "independent_slices":headers.independent_slices,"dependent_slices":headers.dependent_slices, + "picture_hashes":headers.picture_hashes,"reserved_hashes":headers.reserved_hashes}) +} diff --git a/crates/coreav-hevc/examples/common/decoded.rs b/crates/coreav-hevc/examples/common/decoded.rs index 60b40a1df..1a2efd42f 100644 --- a/crates/coreav-hevc/examples/common/decoded.rs +++ b/crates/coreav-hevc/examples/common/decoded.rs @@ -1,21 +1,12 @@ use super::common; -use coreav_hevc::{DecodeLimits, DecodeSummary, DecodedPicture, Error, InterStats, Result}; +use coreav_hevc::{DecodeLimits, DecodeSummary, DecodedPicture, Error, Result}; use md5::{Digest, Md5}; use serde_json::{Value, json}; use std::io::Write; -pub(super) fn inter_stats(stats: InterStats) -> Value { - json!({"skip_units":stats.skip_units,"prediction_units":stats.prediction_units, - "p_units":stats.p_units,"b_units":stats.b_units,"bi_units":stats.bi_units, - "merge_units":stats.merge_units,"amvp_lists":stats.amvp_lists, - "temporal_units":stats.temporal_units,"weighted_units":stats.weighted_units, - "weighted_changed_samples":stats.weighted_changed_samples, - "fractional_luma_units":stats.fractional_luma_units, - "fractional_chroma_units":stats.fractional_chroma_units, - "negative_motion_units":stats.negative_motion_units,"long_term_units":stats.long_term_units, - "list_modification_units":stats.list_modification_units,"partitions":stats.partitions, - "boundary_strengths":stats.boundary_strengths,"boundary_changed_samples":stats.boundary_changed_samples}) -} +#[path = "counters.rs"] +mod counters; +pub(super) use counters::inter_stats; pub(super) fn picture(picture: DecodedPicture<'_>, stream: &mut Md5) -> Result<(Value, u64)> { let info = picture.info; @@ -207,15 +198,10 @@ impl<'a, W: Write> Output<'a, W> { json_line( self.writer, &json!({"type":"summary","complete":true, - "headers":{"nals":headers.nals,"pictures":headers.pictures, - "warnings":{"profile_bit_depth":headers.warnings.profile_bit_depth, - "invalid_sample_aspect_ratio":headers.warnings.invalid_sample_aspect_ratio, - "temporal_nesting_nal_type":headers.warnings.temporal_nesting_nal_type}, - "output_flag_pictures":headers.output_flag_pictures,"output_pictures":headers.output_pictures, - "independent_slices":headers.independent_slices,"dependent_slices":headers.dependent_slices, - "picture_hashes":headers.picture_hashes,"reserved_hashes":headers.reserved_hashes}, + "headers":counters::header_stats(headers), "decoded_pictures":summary.decoded_pictures,"skipped_pictures":summary.skipped_pictures, "parallel_pictures":summary.parallel_pictures,"max_workers":summary.max_workers, + "frame_parallel_pictures":summary.frame_parallel_pictures, "skipped_picture_hashes":summary.skipped_picture_hashes,"skipped_reserved_hashes":summary.skipped_reserved_hashes, "stats":common::reconstruction_stats(summary.reconstruction),"inter":inter_stats(summary.inter), "memory_bytes":summary.memory_bytes,"input_bytes":input.bytes,"input_capacity":input.capacity, diff --git a/crates/coreav-hevc/examples/decode.rs b/crates/coreav-hevc/examples/decode.rs index c5757c3e8..f393665d6 100644 --- a/crates/coreav-hevc/examples/decode.rs +++ b/crates/coreav-hevc/examples/decode.rs @@ -18,12 +18,26 @@ use decoded::Output; #[cfg(test)] use decoded::{inter_stats, picture}; +#[cfg(test)] +#[path = "../test-support/pcm.rs"] +mod pcm; +#[cfg(test)] +#[path = "../test-support/writer.rs"] +mod writer; + fn decode(source: impl Read, writer: &mut impl Write, limits: DecodeLimits) -> Result<()> { let mut decoder = PictureDecoder::new(limits); let mut output = Output::new(writer); let receipt = common::read_nals(source, limits.headers.max_nal_bytes, |_, bytes| { - decoder.read_nal(bytes, None, |picture| output.picture(picture)) - })?; + decoder.read_nal_scheduled(bytes, None, None, |picture, _| output.picture(picture)) + }); + let receipt = match receipt { + Ok(receipt) => receipt, + Err(error) => { + decoder.synchronize_pending()?; + return Err(error); + } + }; let summary = decoder.finish(|picture| output.picture(picture))?; output.finish(summary, receipt, limits) } @@ -87,6 +101,70 @@ mod tests { use super::*; use coreav_hevc::{IntraPlane, IntraStats, PictureInfo, Summary, Vui}; + #[test] + fn scheduled_input_read_errors_preserve_earlier_reconstruction_failures() -> Result<()> { + struct ReadFailure(std::io::Cursor>); + impl Read for ReadFailure { + fn read(&mut self, bytes: &mut [u8]) -> std::io::Result { + match self.0.read(bytes)? { + 0 => Err(std::io::Error::other("generated_read_failure")), + count => Ok(count), + } + } + } + let mut fixture = pcm::pcm_fixture(8, 8, false)?; + let mut complete = Vec::new(); + for nal in fixture.sets.iter().chain([&fixture.nal]) { + complete.extend([0, 0, 1]); + complete.extend(nal); + } + let mut output = Vec::new(); + decode( + complete.as_slice(), + &mut output, + DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }, + )?; + let mut expected = Md5::new(); + for sample in fixture.planes.iter().flatten() { + expected.update([*sample as u8]); + } + assert_eq!( + lines(&output)?.last().and_then(|row| row.get("output_md5")), + Some(&json!(format!("{:x}", expected.finalize()))) + ); + fixture.nal.pop(); + let mut damaged = Vec::new(); + for nal in fixture.sets.iter().chain([&fixture.nal]) { + damaged.extend([0, 0, 1]); + damaged.extend(nal); + } + damaged.extend([0, 0, 1]); + let mut expected = None; + for threads in [1, 2, 4, 8] { + let mut output = Vec::new(); + let actual = decode( + ReadFailure(std::io::Cursor::new(damaged.clone())), + &mut output, + DecodeLimits { + threads, + ..DecodeLimits::default() + }, + ); + assert!(actual.is_err()); + assert_ne!(actual, Err(Error::InvalidData("input_read"))); + assert!(output.is_empty()); + if let Some(expected) = &expected { + assert_eq!(&actual, expected); + } else { + expected = Some(actual); + } + } + Ok(()) + } + struct Fixture { info: PictureInfo, planes: [Vec; 3], @@ -408,6 +486,7 @@ mod tests { fn summary(count: u64) -> DecodeSummary { DecodeSummary { parallel_pictures: 0, + frame_parallel_pictures: 0, max_workers: 1, headers: Summary { warnings: coreav_hevc::Warnings { @@ -499,6 +578,9 @@ mod tests { ); for (key, value) in [ ("decoded_pictures", 3), + ("parallel_pictures", 0), + ("frame_parallel_pictures", 0), + ("max_workers", 1), ("skipped_pictures", 0), ("memory_bytes", 4096), ("input_bytes", 1234), diff --git a/crates/coreav-hevc/src/decoder.rs b/crates/coreav-hevc/src/decoder.rs index 780bce084..2e3b473a9 100644 --- a/crates/coreav-hevc/src/decoder.rs +++ b/crates/coreav-hevc/src/decoder.rs @@ -5,10 +5,59 @@ use crate::{ reconstruction::{Cache, Frame, Memory, ReclaimMemory}, }; +#[cfg(feature = "native-hevc-rows-private")] +mod rows; +mod scheduler; + +trait Delivery { + fn output(&mut self, picture: DecodedPicture<'_>, memory: &mut OutputMemory<'_>) -> Result<()>; + #[cfg(feature = "native-hevc-rows-private")] + fn row_sink(&mut self) -> Option<&mut dyn crate::native_rows::RowSink> { + None + } +} + +impl, &mut OutputMemory<'_>) -> Result<()>> Delivery for F { + fn output(&mut self, picture: DecodedPicture<'_>, memory: &mut OutputMemory<'_>) -> Result<()> { + self(picture, memory) + } +} + +#[cfg(feature = "native-hevc-rows-private")] +struct RowsOutput<'a>(&'a mut dyn crate::native_rows::RowSink); + +#[cfg(feature = "native-hevc-rows-private")] +impl Delivery for RowsOutput<'_> { + fn row_sink(&mut self) -> Option<&mut dyn crate::native_rows::RowSink> { + Some(self.0) + } + fn output(&mut self, picture: DecodedPicture<'_>, memory: &mut OutputMemory<'_>) -> Result<()> { + let offset = self.0.cost(0, 0)?.queued_frames; + if self.0.prepare( + crate::native_rows::OutputHeader { + index: picture.index, + poc: picture.poc, + timestamp: picture.timestamp, + origin: picture.origin, + info: picture.info, + }, + memory, + false, + )? { + let result = self.0.fill(offset, picture); + if result.is_err() { + self.0.truncate(offset); + } + result?; + } + Ok(()) + } +} + /// Independent admission, live-memory and entropy-work limits for a complete stream. #[derive(Clone, Copy, Debug)] pub struct DecodeLimits { - /// Requested reconstruction workers, bounded to one through sixteen. + /// Requested reconstruction workers, bounded to one through sixteen and available CPUs. pub threads: usize, /// Header admission, including the stream-wide coded-picture limit. pub headers: Limits, @@ -100,9 +149,11 @@ impl OutputMemory<'_> { /// Successful complete-stream accounting, including pictures not selected for output. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct DecodeSummary { - /// Pictures reconstructed using more than one successfully started worker. + /// Pictures whose reconstruction overlapped another whole-picture job. + pub frame_parallel_pictures: u64, + /// Pictures reconstructed with whole-picture overlap or multiple WPP workers. pub parallel_pictures: u64, - /// Greatest number of successfully started workers, including the caller. + /// Greatest executing picture-job count or WPP worker count, including the caller. pub max_workers: usize, /// Header syntax and presentation-order output counts. pub headers: Summary, @@ -134,6 +185,7 @@ enum State { struct Slots { active: Vec, cache: Cache, + scheduler: scheduler::Scheduler, } impl Slots { @@ -165,7 +217,24 @@ impl Slots { } } -struct Pictures { +#[expect( + dead_code, + reason = "The original typed accounting layout fixes cold admission independently of optional counters" +)] +struct SerialAccounting { + parallel_pictures: u64, + max_workers: usize, + headers: Summary, + decoded_pictures: u64, + skipped_pictures: u64, + skipped_picture_hashes: u64, + skipped_reserved_hashes: u64, + reconstruction: IntraStats, + inter: InterStats, + memory_bytes: usize, +} + +struct Pictures { limits: DecodeLimits, frame: Storage, timestamp: Option, @@ -174,12 +243,26 @@ struct Pictures { skipped: Option, stream_remaining: u64, picture_remaining: u64, - summary: DecodeSummary, + summary: Accounting, } -struct Session { +struct Session { core: StreamCore, - pictures: Pictures, + pictures: Pictures, +} + +impl Pictures { + fn synchronize(&mut self) -> Result<()> { + self.frame + .scheduler + .drain(&mut self.summary, &mut self.stream_remaining, None)?; + self.frame.scheduler.flush( + &mut self.frame.active, + &mut self.stream_remaining, + &mut self.picture_remaining, + )?; + Ok(()) + } } /// Sans-IO Main/Main10 scalar decoder with one DPB and borrowed output callbacks. @@ -189,11 +272,18 @@ pub struct PictureDecoder { state: State, } +fn worker_count(requested: usize, available: Option) -> usize { + requested + .clamp(1, 16) + .min(available.map_or(1, core::num::NonZeroUsize::get)) +} + impl PictureDecoder { /// Selects reconstruction workers without changing sample or output ordering. pub fn with_threads(mut self, threads: usize) -> Self { - self.limits.threads = threads.clamp(1, 16); + self.limits.threads = worker_count(threads, std::thread::available_parallelism().ok()); for session in &mut self.session { + session.pictures.frame.scheduler.join_pending(); session.pictures.limits.threads = self.limits.threads; session.pictures.frame.cache.clear(); } @@ -201,7 +291,8 @@ impl PictureDecoder { } /// Creates an opt-in decoder without allocating picture samples. - pub fn new(limits: DecodeLimits) -> Self { + pub fn new(mut limits: DecodeLimits) -> Self { + limits.threads = worker_count(limits.threads, std::thread::available_parallelism().ok()); Self { limits, session: Vec::new(), @@ -212,7 +303,7 @@ impl PictureDecoder { fn fixed_memory(&self) -> Result { core::mem::size_of::() .checked_add( - core::mem::size_of::>>() + core::mem::size_of::, SerialAccounting>>() .checked_mul(self.session.capacity().max(1) + 1) .ok_or(Error::Limit("reconstruction_memory"))?, ) @@ -280,24 +371,161 @@ impl PictureDecoder { self.read(bytes, timestamp, Some(origin), emit) } + /// Feeds a NAL while allowing complete-reference picture jobs to overlap. + /// + /// Unlike `read_nal`, reconstruction failures may be deferred until a required + /// reference, output, reservation or final drain synchronizes the serial prefix. + /// Callbacks run only on the caller, and `finish_with_output` is required. + pub fn read_nal_scheduled( + &mut self, + bytes: &[u8], + timestamp: Option, + origin: Option, + emit: impl FnMut(DecodedPicture<'_>, &mut OutputMemory<'_>) -> Result<()>, + ) -> Result<()> { + self.read_mode(bytes, timestamp, origin, true, emit) + } + + #[cfg(feature = "native-hevc-rows-private")] + #[doc(hidden)] + pub fn read_nal_rows( + &mut self, + bytes: &[u8], + timestamp: Option, + origin: Option, + sink: &mut impl crate::native_rows::RowSink, + ) -> Result<()> { + self.read_mode(bytes, timestamp, origin, true, RowsOutput(sink)) + } + + #[cfg(feature = "native-hevc-rows-private")] + #[doc(hidden)] + pub fn settle_rows(&mut self, sink: &mut impl crate::native_rows::RowSink) -> Result<()> { + self.settle_delivery(&mut RowsOutput(sink), true) + } + + #[cfg(feature = "native-hevc-rows-private")] + #[doc(hidden)] + pub fn has_native_rows(&self) -> bool { + self.session + .first() + .is_some_and(|session| session.pictures.frame.scheduler.rows().is_some()) + } + + #[cfg(all(feature = "native-hevc-rows-private", feature = "test-support"))] + #[doc(hidden)] + pub fn native_row_admission(&self) -> Option { + let rows = self.session.first()?.pictures.frame.scheduler.rows()?; + Some(crate::native_rows::Admission { + sps_id: rows.identity.0, + sps_hash: rows.identity.1, + ceiling: rows.plan.ceiling, + workers: rows.plan.workers, + lag: rows.plan.lag, + frame_bytes: rows.plan.frame_bytes, + reserved_bytes: rows.plan.reserved, + limit_bytes: self.limits.max_memory_bytes, + }) + } + + #[cfg(feature = "native-hevc-rows-private")] + #[doc(hidden)] + pub fn receive_rows(&mut self) -> Result> { + self.session + .first_mut() + .and_then(|session| session.pictures.frame.scheduler.rows_mut()) + .map(rows::Rows::receive) + .transpose() + } + + #[cfg(feature = "native-hevc-rows-private")] + #[doc(hidden)] + pub fn release_rows_memory(&mut self, bytes: usize) -> Result<()> { + let session = self + .session + .first_mut() + .ok_or(Error::Internal("row_owner"))?; + if bytes > session.pictures.external_memory { + return Err(Error::Internal("output_reservation_growth")); + } + session.pictures.external_memory = bytes; + Ok(()) + } + + /// Reconciles deferred reconstruction before reporting a caller-owned input error. + /// + /// This does not seal the current picture, emit output or validate end of stream. + /// Closed sessions have no pending work; an already returned error stays authoritative. + pub fn synchronize_pending(&mut self) -> Result<()> { + if self.state != State::Active { + return Ok(()); + } + let result = self + .session + .first_mut() + .map_or(Ok(()), |session| session.pictures.synchronize()); + if result.is_err() { + self.fail(); + } + result + } + fn read( &mut self, bytes: &[u8], timestamp: Option, origin: Option, - mut emit: impl FnMut(DecodedPicture<'_>, &mut OutputMemory<'_>) -> Result<()>, + emit: impl FnMut(DecodedPicture<'_>, &mut OutputMemory<'_>) -> Result<()>, + ) -> Result<()> { + self.read_mode(bytes, timestamp, origin, false, emit) + } + + fn read_mode( + &mut self, + bytes: &[u8], + timestamp: Option, + origin: Option, + scheduled: bool, + mut emit: impl Delivery, ) -> Result<()> { if self.state != State::Active { self.fail(); return Err(Error::InvalidData("picture_decoder_closed")); } let result = (|| { + #[cfg(hevc_thread_diagnostics)] + crate::thread_diagnostics::check_row_threads(self.limits.threads)?; self.initialize()?; let fixed = self.fixed_memory()?; let session = self .session .first_mut() .ok_or(Error::InvalidData("decoder_state"))?; + session.pictures.frame.scheduler.enabled = + scheduled && session.pictures.limits.threads > 1; + if !session.pictures.frame.scheduler.enabled { + session.pictures.synchronize()?; + } + session.core.unshare(); + let needed = fixed + + session.core.heap_bytes()? + + session + .pictures + .frame + .as_ref() + .map_or(Ok(0), Frame::heap_bytes)? + + session.pictures.external_memory + + session + .pictures + .frame + .scheduler + .bytes(Some(session.core.dpb()))? + + bytes.len().saturating_sub(2); + if needed > session.pictures.limits.max_memory_bytes { + crate::thread_diagnostic!(barrier MemoryInput); + session.pictures.synchronize()?; + session.core.unshare(); + } let frame_bytes = session .pictures .frame @@ -322,6 +550,11 @@ impl PictureDecoder { .max_memory_bytes .checked_sub(live) .ok_or(Error::Limit("reconstruction_memory"))?; + let available = if scheduled { + session.pictures.limits.max_memory_bytes + } else { + available + }; session.core.read_reconstructed( bytes, available, @@ -330,9 +563,21 @@ impl PictureDecoder { fixed, incoming_timestamp: timestamp, incoming_origin: origin, + #[cfg(feature = "native-hevc-rows-private")] + activation: None, emit: &mut emit, }, )?; + #[cfg(feature = "native-hevc-rows-private")] + Observer { + pictures: &mut session.pictures, + fixed, + incoming_timestamp: timestamp, + incoming_origin: origin, + activation: None, + emit: &mut emit, + } + .row_enforce(session.core.dpb(), 0)?; session.pictures.summary.memory_bytes = session .pictures .summary @@ -340,12 +585,58 @@ impl PictureDecoder { .max(session.pictures.frame.cache.peak()); Ok(()) })(); + let result = match result { + Err(error) => match self.settle_delivery(&mut emit, true) { + Err(earlier) => Err(earlier), + Ok(_) => Err(error), + }, + result => result, + }; if result.is_err() { self.fail(); } result } + fn settle_delivery(&mut self, emit: &mut impl Delivery, truncate: bool) -> Result<()> { + let fixed = self.fixed_memory()?; + let Some(session) = self.session.first_mut() else { + return Ok(()); + }; + #[cfg(feature = "native-hevc-rows-private")] + { + let mut observer = Observer { + pictures: &mut session.pictures, + fixed, + incoming_timestamp: None, + incoming_origin: None, + activation: None, + emit, + }; + let result = observer.row_settle(session.core.dpb(), 0); + if truncate { + if let Some(rows) = observer.pictures.frame.scheduler.rows() { + let keep = rows + .validated + .checked_sub(rows.returned) + .and_then(|n| usize::try_from(n).ok()); + if let Some(sink) = observer.emit.row_sink() { + sink.truncate(keep.unwrap_or(0)); + } + if keep.is_none() { + return Err(Error::Internal("output_ordinal")); + } + } + } + result + } + #[cfg(not(feature = "native-hevc-rows-private"))] + { + let _ = (fixed, emit, truncate); + session.pictures.synchronize() + } + } + /// Finalizes the last picture and drains output; success requires this call. /// Earlier callbacks remain valid if a later NAL, hash or callback fails. pub fn finish( @@ -358,8 +649,21 @@ impl PictureDecoder { /// Drains pictures with callback-scoped output reservations in the same account. pub fn finish_with_output( &mut self, - mut emit: impl FnMut(DecodedPicture<'_>, &mut OutputMemory<'_>) -> Result<()>, + emit: impl FnMut(DecodedPicture<'_>, &mut OutputMemory<'_>) -> Result<()>, + ) -> Result { + self.finish_mode(emit) + } + + #[cfg(feature = "native-hevc-rows-private")] + #[doc(hidden)] + pub fn finish_rows( + &mut self, + sink: &mut impl crate::native_rows::RowSink, ) -> Result { + self.finish_mode(RowsOutput(sink)) + } + + fn finish_mode(&mut self, mut emit: impl Delivery) -> Result { if self.state == State::Failed { return Err(Error::InvalidData("picture_decoder_failed")); } @@ -374,8 +678,23 @@ impl PictureDecoder { fixed, incoming_timestamp: None, incoming_origin: None, + #[cfg(feature = "native-hevc-rows-private")] + activation: None, emit: &mut emit, })?; + #[cfg(feature = "native-hevc-rows-private")] + Observer { + pictures: &mut session.pictures, + fixed, + incoming_timestamp: None, + incoming_origin: None, + activation: None, + emit: &mut emit, + } + .row_settle(session.core.dpb(), 0)?; + #[cfg(not(feature = "native-hevc-rows-private"))] + session.pictures.synchronize()?; + session.core.unshare(); session.pictures.summary.headers = headers; session.pictures.summary.memory_bytes = session .pictures @@ -385,6 +704,10 @@ impl PictureDecoder { session.pictures.frame.cache.clear(); Ok(session.pictures.summary) })(); + let result = match result { + Err(error) => self.settle_delivery(&mut emit, true).and(Err(error)), + result => result, + }; if result.is_err() { self.fail(); } else { @@ -406,6 +729,30 @@ impl PictureDecoder { .session .first_mut() .ok_or(Error::InvalidData("decoder_state"))?; + let optional = session + .pictures + .frame + .scheduler + .bytes(Some(session.core.dpb()))?; + let required = fixed + .checked_add(session.core.heap_bytes()?) + .and_then(|n| { + n.checked_add( + session + .pictures + .frame + .as_ref() + .map_or(Ok(0), Frame::heap_bytes) + .ok()?, + ) + }) + .and_then(|n| n.checked_add(optional)) + .and_then(|n| n.checked_add(bytes)) + .ok_or(Error::Limit("reconstruction_memory"))?; + if required > self.limits.max_memory_bytes { + session.pictures.synchronize()?; + session.core.unshare(); + } let frame = session .pictures .frame @@ -414,6 +761,16 @@ impl PictureDecoder { let live = fixed .checked_add(session.core.heap_bytes()?) .and_then(|n| n.checked_add(frame)) + .and_then(|n| { + n.checked_add( + session + .pictures + .frame + .scheduler + .bytes(Some(session.core.dpb())) + .ok()?, + ) + }) .ok_or(Error::Limit("reconstruction_memory"))?; let mut memory = Memory::with_reclaim( live, @@ -426,6 +783,18 @@ impl PictureDecoder { session.pictures.summary.memory_bytes.max(memory.peak()); Ok(()) })(); + let result = match result { + Err(error) => match self + .session + .first_mut() + .map(|session| session.pictures.synchronize()) + .transpose() + { + Err(earlier) => Err(earlier), + Ok(_) => Err(error), + }, + result => result, + }; if result.is_err() { self.fail(); } @@ -438,13 +807,78 @@ struct Observer<'a, F> { fixed: usize, incoming_timestamp: Option, incoming_origin: Option, + #[cfg(feature = "native-hevc-rows-private")] + activation: Option<(u32, [u8; 32])>, emit: &'a mut F, } -impl, &mut OutputMemory<'_>) -> Result<()>> PictureObserver - for Observer<'_, F> -{ - fn reserve_scratch(&mut self, scratch_bytes: usize, dpb: &Dpb) -> Result<()> { +impl PictureObserver for Observer<'_, F> { + #[cfg(feature = "native-hevc-rows-private")] + fn activated_sps(&mut self, identity: (u32, [u8; 32])) { + self.activation = Some(identity); + } + fn reconcile(&mut self) -> Result<()> { + crate::thread_diagnostic!(event Reconcile); + self.pictures.synchronize() + } + + fn before_picture( + &mut self, + dpb: &mut Dpb, + nal: crate::NalHeader, + sps: &crate::params::Sps, + ) -> Result<()> { + if dpb.picture_outputs(nal, sps) { + #[cfg(feature = "native-hevc-rows-private")] + if self + .pictures + .frame + .scheduler + .rows() + .is_some_and(|rows| rows.plan.lag != 0) + { + return Ok(()); + } + #[cfg(hevc_thread_diagnostics)] + if self.pictures.frame.scheduler.has_work() { + use crate::thread_diagnostics::{Barrier, barrier}; + for (active, reason) in dpb.picture_output_constraints(nal, sps).into_iter().zip([ + Barrier::OutputRandomAccess, + Barrier::OutputCapacity, + Barrier::OutputReorder, + Barrier::OutputLatency, + ]) { + if active { + barrier(reason); + } + } + } + self.pictures.synchronize()?; + dpb.unshare(); + } + Ok(()) + } + + fn reserve_scratch(&mut self, scratch_bytes: usize, dpb: &mut Dpb) -> Result<()> { + let optional = self.pictures.frame.scheduler.bytes(Some(dpb))?; + let current = self + .pictures + .frame + .as_ref() + .map_or(Ok(0), Frame::heap_bytes)?; + let needed = self + .fixed + .checked_add(dpb.heap_bytes()?) + .and_then(|n| n.checked_add(current)) + .and_then(|n| n.checked_add(self.pictures.external_memory)) + .and_then(|n| n.checked_add(scratch_bytes)) + .and_then(|n| n.checked_add(optional)) + .ok_or(Error::Limit("reconstruction_memory"))?; + if needed > self.pictures.limits.max_memory_bytes { + crate::thread_diagnostic!(barrier MemoryScratch); + self.pictures.synchronize()?; + dpb.unshare(); + } let frame = self .pictures .frame @@ -455,6 +889,7 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO .checked_add(dpb.heap_bytes()?) .and_then(|n| n.checked_add(frame)) .and_then(|n| n.checked_add(self.pictures.external_memory)) + .and_then(|n| n.checked_add(self.pictures.frame.scheduler.bytes(Some(dpb)).ok()?)) .ok_or(Error::Limit("reconstruction_memory"))?; let mut memory = Memory::with_reclaim( fixed, @@ -471,6 +906,55 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO let Some(mut frame) = self.pictures.frame.take() else { return Ok(()); }; + if self.pictures.frame.scheduler.has_input() { + self.pictures.frame.scheduler.seal( + frame, + dpb, + self.pictures.limits, + &mut self.pictures.summary, + &mut self.pictures.stream_remaining, + scheduler::JobIdentity { + prior_bins: self.pictures.limits.max_picture_bins + - self.pictures.picture_remaining, + timestamp: self.pictures.timestamp, + origin: self.pictures.origin, + }, + )?; + if dpb.resume_outputs() { + #[cfg(feature = "native-hevc-rows-private")] + if self + .pictures + .frame + .scheduler + .rows() + .is_some_and(|rows| rows.plan.lag != 0) + { + return Ok(()); + } + #[cfg(hevc_thread_diagnostics)] + { + use crate::thread_diagnostics::{Barrier, barrier}; + for (active, reason) in dpb + .resume_output_constraints() + .into_iter() + .zip([Barrier::OutputReorder, Barrier::OutputLatency]) + { + if active { + barrier(reason); + } + } + } + self.pictures.synchronize()?; + dpb.unshare(); + } + return Ok(()); + } + self.pictures.frame.scheduler.drain( + &mut self.pictures.summary, + &mut self.pictures.stream_remaining, + None, + )?; + dpb.unshare(); let fixed = self .fixed .checked_add(dpb.heap_bytes()?) @@ -495,10 +979,15 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO self.pictures .summary .add(frame.stats, frame.inter_stats())?; - dpb.attach(frame.retain(self.pictures.timestamp, self.pictures.origin)?) + #[cfg(feature = "native-hevc-rows-private")] + let index = frame.index(); + dpb.attach(frame.retain(self.pictures.timestamp, self.pictures.origin)?)?; + #[cfg(feature = "native-hevc-rows-private")] + self.pictures.frame.scheduler.confirm(index)?; + Ok(()) } - fn slice(&mut self, input: SliceInput<'_>, dpb: &Dpb) -> Result<()> { + fn slice(&mut self, input: SliceInput<'_>, dpb: &mut Dpb) -> Result<()> { if input.header.first { if self.incoming_origin.is_some() && self.incoming_origin == self.pictures.origin { return Err(Error::InvalidData("multiple_pictures_in_access_unit")); @@ -527,11 +1016,76 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO } return Ok(()); } - let fixed = self - .fixed - .checked_add(dpb.heap_bytes()?) - .and_then(|n| n.checked_add(self.pictures.external_memory)) - .ok_or(Error::Limit("reconstruction_memory"))?; + #[cfg(hevc_thread_diagnostics)] + if input.header.first { + crate::thread_diagnostics::row_picture( + input.index, + dpb.current_identity(input.index)?.poc, + input.picture, + )?; + } + crate::thread_diagnostic!(picture input.index); + #[cfg(feature = "native-hevc-rows-private")] + if input.header.first + && self.pictures.frame.scheduler.rows().is_some_and(|rows| { + rows.plan.lag == 0 + && rows.serial_received == rows.returned + && rows.pending.is_empty() + }) + { + self.pictures.frame.scheduler.retire_rows()?; + } + #[cfg(feature = "native-hevc-rows-private")] + if self.pictures.frame.scheduler.rows().is_some() + && (input.picture.tiles + || input.picture.wpp + || self + .pictures + .frame + .scheduler + .rows() + .is_some_and(|rows| rows.plan.lag == 0)) + { + self.row_settle(dpb, input.scratch_bytes)?; + dpb.unshare(); + self.pictures.frame.cache.clear(); + self.pictures.frame.scheduler.enabled = false; + } + #[cfg(hevc_thread_diagnostics)] + if input.header.first { + match input.header.slice_type { + Some(0) => crate::thread_diagnostic!(event PictureB), + Some(1) => crate::thread_diagnostic!(event PictureP), + Some(2) => crate::thread_diagnostic!(event PictureI), + _ => {} + } + } + if !self.pictures.frame.scheduler.enabled || input.picture.wpp { + if input.picture.wpp { + crate::thread_diagnostic!(barrier Wpp); + } else { + crate::thread_diagnostic!(barrier WorkersDisabled); + } + self.pictures.synchronize()?; + dpb.unshare(); + } + #[cfg(feature = "native-hevc-rows-private")] + let row_references = self.pictures.frame.scheduler.enabled + && self + .pictures + .frame + .scheduler + .rows() + .is_some_and(|rows| rows.plan.lag != 0); + #[cfg(not(feature = "native-hevc-rows-private"))] + let row_references = false; + if !row_references { + self.pictures.frame.scheduler.required_references( + dpb, + &mut self.pictures.summary, + &mut self.pictures.stream_remaining, + )?; + } if self.pictures.frame.is_none() { let bytes = (u64::from(input.picture.visible_width) * u64::from(input.picture.visible_height)) @@ -543,20 +1097,147 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO return Err(Error::Limit("output_frame_bytes")); } let identity = dpb.current_identity(input.index)?; + #[cfg(feature = "native-hevc-rows-private")] + { + self.row_admit( + &input, + dpb, + usize::try_from(bytes).map_err(|_| Error::Limit("output_frame_bytes"))?, + )?; + if self.pictures.frame.scheduler.rows().is_some_and(|rows| { + rows.plan.lag == 0 || input.picture.tiles || input.picture.wpp + }) { + self.pictures.frame.scheduler.enabled = false; + } + } + let fixed = self + .fixed + .checked_add(dpb.heap_bytes()?) + .and_then(|n| n.checked_add(self.pictures.external_memory)) + .ok_or(Error::Limit("reconstruction_memory"))?; self.pictures.frame.prepare()?; - let mut frame = Frame::new_full( + let optional = self.pictures.frame.scheduler.bytes(Some(dpb))?; + let create = Frame::new_full( &input, identity.poc, dpb.current_references(), - fixed, + fixed + .checked_add(optional) + .ok_or(Error::Limit("reconstruction_memory"))?, self.pictures.limits.max_memory_bytes, Some(&mut self.pictures.frame.cache), - )?; + ); + let mut frame = match create { + Err(Error::Limit("reconstruction_memory")) if optional != 0 => { + crate::thread_diagnostic!(barrier MemoryFrame); + self.pictures.synchronize()?; + dpb.unshare(); + Frame::new_full( + &input, + identity.poc, + dpb.current_references(), + self.fixed + dpb.heap_bytes()? + self.pictures.external_memory, + self.pictures.limits.max_memory_bytes, + Some(&mut self.pictures.frame.cache), + )? + } + result => result?, + }; frame.deadline = self.pictures.limits.deadline; self.pictures.frame.put(frame)?; self.pictures.picture_remaining = self.pictures.limits.max_picture_bins; } - let Slots { active, cache } = &mut self.pictures.frame; + if self.pictures.frame.scheduler.enabled && !input.picture.wpp { + self.pictures.frame.cache.clear(); + if self + .pictures + .frame + .scheduler + .full(self.pictures.limits.threads) + { + crate::thread_diagnostic!(barrier WindowFull); + self.pictures.frame.scheduler.drain( + &mut self.pictures.summary, + &mut self.pictures.stream_remaining, + None, + )?; + dpb.unshare(); + } + let fixed = self.fixed + + dpb.heap_bytes()? + + self.pictures.external_memory + + self.pictures.frame.scheduler.bytes(Some(dpb))? + - self.pictures.frame.scheduler.current_reservation() + + dpb.sharing_bytes(); + let Slots { + active, scheduler, .. + } = &mut self.pictures.frame; + let frame = active.first().ok_or(Error::Internal("picture_job_frame"))?; + if scheduler.queue( + input, + frame, + fixed, + self.pictures.limits.max_memory_bytes, + self.pictures.stream_remaining, + self.pictures.picture_remaining, + )? { + if !self.pictures.frame.scheduler.references( + dpb, + &mut self.pictures.summary, + &mut self.pictures.stream_remaining, + )? { + self.pictures.frame.scheduler.discard_current(); + } else { + self.pictures.summary.memory_bytes = self.pictures.summary.memory_bytes.max( + self.fixed + + dpb.heap_bytes()? + + self.pictures.external_memory + + self + .pictures + .frame + .as_ref() + .map_or(Ok(0), Frame::heap_bytes)? + + self.pictures.frame.scheduler.bytes(Some(dpb))?, + ); + return Ok(()); + } + } + self.pictures.synchronize()?; + dpb.unshare(); + } + let fixed = self.fixed + dpb.heap_bytes()? + self.pictures.external_memory; + #[cfg(feature = "native-hevc-rows-private")] + let fixed = if self.pictures.frame.scheduler.rows().is_some() { + fixed + .checked_add(self.pictures.frame.scheduler.bytes(Some(dpb))?) + .ok_or(Error::Limit("reconstruction_memory"))? + } else { + fixed + }; + let decode_limit = self.pictures.limits.max_memory_bytes; + let decode_threads = self.pictures.limits.threads; + #[cfg(feature = "native-hevc-rows-private")] + let decode_limit = match self.pictures.frame.scheduler.rows() { + Some(rows) => rows + .plan + .lag + .checked_mul(rows.plan.frame_bytes) + .and_then(|debt| rows.plan.reserved.checked_add(debt)) + .and_then(|limit| { + limit.checked_sub(rows.plan.ceiling.checked_mul(rows::ATOMIC_RUNTIME_BYTES)?) + }) + .filter(|limit| *limit <= decode_limit) + .ok_or(Error::Internal("row_admission"))?, + None => decode_limit, + }; + #[cfg(feature = "native-hevc-rows-private")] + let decode_threads = self + .pictures + .frame + .scheduler + .rows() + .map_or(decode_threads, |rows| decode_threads.min(rows.plan.ceiling)); + let Slots { active, cache, .. } = &mut self.pictures.frame; let frame = active .first_mut() .ok_or(Error::InvalidData("missing_working_picture"))?; @@ -571,14 +1252,18 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO dpb, &mut budget, fixed, - self.pictures.limits.max_memory_bytes, - self.pictures.limits.threads, + decode_limit, + decode_threads, Some(cache), ) }); let consumed = allowance - budget; self.pictures.stream_remaining -= consumed; self.pictures.picture_remaining -= consumed; + #[cfg(feature = "native-hevc-rows-private")] + if self.pictures.frame.scheduler.rows().is_some() { + self.pictures.frame.cache.clear(); + } match result { Ok(peak) => { self.pictures.summary.memory_bytes = self @@ -597,7 +1282,17 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO } fn sei(&mut self, info: SeiInfo, dpb: &Dpb, scratch_bytes: usize) -> Result<()> { - let Slots { active, cache } = &mut self.pictures.frame; + if info.picture_hash.is_none() && info.reserved_hashes == 0 { + return Ok(()); + } + if self.pictures.frame.scheduler.queue_hash(info) { + return Ok(()); + } + if self.pictures.frame.scheduler.has_input() { + crate::thread_diagnostic!(barrier DuplicateHash); + self.pictures.synchronize()?; + } + let Slots { active, cache, .. } = &mut self.pictures.frame; if let Some(frame) = active.first_mut() { let fixed = self .fixed @@ -637,8 +1332,22 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO } fn output(&mut self, emission: Emission<'_>, scratch_bytes: usize) -> Result<()> { + #[cfg(feature = "native-hevc-rows-private")] + if self.pictures.frame.scheduler.rows().is_some() { + return self.row_output(emission, scratch_bytes); + } + self.pictures.frame.scheduler.drain( + &mut self.pictures.summary, + &mut self.pictures.stream_remaining, + Some(emission.event.picture_index), + )?; let samples = emission .picture + .or_else(|| { + emission + .publication + .and_then(|publication| publication.picture()) + }) .ok_or(Error::InvalidData("missing_output_samples"))?; let picture = samples.view(); let frame = self @@ -648,9 +1357,18 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO .map_or(Ok(0), Frame::heap_bytes)?; let codec_bytes = self .fixed - .checked_add(emission.retained_bytes) + .checked_add(emission.dpb.heap_bytes()?) .and_then(|n| n.checked_add(frame)) .and_then(|n| n.checked_add(scratch_bytes)) + .and_then(|n| { + n.checked_add( + self.pictures + .frame + .scheduler + .bytes(Some(emission.dpb)) + .ok()?, + ) + }) .ok_or(Error::Limit("reconstruction_memory"))?; let mut memory = OutputMemory { codec_bytes, @@ -658,7 +1376,7 @@ impl, &mut OutputMemory<'_>) -> Result<()>> PictureO limit: self.pictures.limits.max_memory_bytes, reclaim: Some(&mut self.pictures.frame.cache), }; - (self.emit)( + self.emit.output( DecodedPicture { index: picture.index, poc: emission.event.poc, @@ -807,13 +1525,51 @@ fn add_counts(target: &mut [u64; N], source: [u64; N]) -> Result mod tests { use super::*; + #[test] + fn requested_workers_respect_the_cpu_cap_and_unavailable_cpu_fallback() { + for (requested, available, expected) in [ + (16, Some(4), 4), + (8, Some(4), 4), + (2, Some(4), 2), + (0, Some(4), 1), + (usize::MAX, Some(64), 16), + (16, Some(1), 1), + (16, None, 1), + (0, None, 1), + ] { + assert_eq!( + worker_count(requested, available.and_then(core::num::NonZeroUsize::new)), + expected + ); + } + } + + #[test] + fn public_worker_boundaries_normalize_limits_and_existing_sessions() -> Result<()> { + let expected = worker_count(usize::MAX, std::thread::available_parallelism().ok()); + let mut decoder = PictureDecoder::new(DecodeLimits { + threads: usize::MAX, + ..DecodeLimits::default() + }); + assert_eq!(decoder.limits.threads, expected); + decoder.initialize()?; + decoder = decoder.with_threads(0); + assert_eq!(decoder.limits.threads, 1); + assert_eq!(decoder.session[0].pictures.limits.threads, 1); + decoder = decoder.with_threads(usize::MAX); + assert_eq!(decoder.limits.threads, expected); + assert_eq!(decoder.session[0].pictures.limits.threads, expected); + Ok(()) + } + #[test] fn cached_storage_fits_the_unchanged_cold_state_envelope() -> Result<()> { let mut decoder = PictureDecoder::new(DecodeLimits::default()); let cold = decoder.fixed_memory()?; for stage in 0..3 { let capacity = decoder.session.capacity().max(1); - let reserved = (capacity + 1) * core::mem::size_of::>>(); + let reserved = + (capacity + 1) * core::mem::size_of::, SerialAccounting>>(); let physical = capacity * core::mem::size_of::(); let active = decoder .session @@ -873,6 +1629,91 @@ mod tests { Ok(()) } + #[test] + fn skipped_picture_scratch_preserves_cold_rbsp_and_entry_limits_with_idle_workers() -> Result<()> + { + fn seed_cache(sets: &[Vec; 3], cra: &[u8]) -> Result { + let mut core = StreamCore::new(Limits::default()); + for set in sets { + core.read_nal(set, |_| Ok(()))?; + } + let mut cache = Cache::default(); + core.read_selected( + cra, + |_| Ok(()), + Some(0), + usize::MAX, + |input| { + cache + .frames_for_test() + .push(Frame::new(&input, 0, usize::MAX)?); + Ok(()) + }, + )?; + assert_eq!(cache.frames_for_test().len(), 1); + Ok(cache) + } + fn prepare_decoder( + decoder: &mut PictureDecoder, + sets: &[Vec; 3], + cra: &[u8], + cache: Cache, + ) -> Result<()> { + let session = &mut decoder.session[0]; + session.core = StreamCore::new(Limits::default()); + for set in sets { + session.core.read_nal(set, |_| Ok(()))?; + } + session.core.read_nal(cra, |_| Ok(()))?; + session.pictures.frame.cache = cache; + Ok(()) + } + let sets = crate::tests::synthetic_parameter_nals_with_wpp(true); + let cra = crate::tests::intra_first_with_wpp(21, 1, Some(0)); + let retained = seed_cache(&sets, &cra)?.retained_bytes(0, usize::MAX)?; + assert!(retained > 0); + for entries in [0, 1] { + let skipped = crate::tests::intra_first_with_wpp(8, 0, Some(entries)); + let rbsp = crate::nal::rbsp(&skipped, &Limits::default())?; + let required = (skipped.len() - 2).max(rbsp.capacity() + entries as usize * 8); + for warm in [false, true] { + for extra in [0, retained] { + for below in [true, false] { + let cache = if warm { + seed_cache(&sets, &cra)? + } else { + Cache::default() + }; + let mut decoder = PictureDecoder::new(DecodeLimits::default()); + decoder.initialize()?; + prepare_decoder(&mut decoder, &sets, &cra, cache)?; + let fixed = decoder.fixed_memory()?; + let session = &mut decoder.session[0]; + let mandatory = fixed + session.core.heap_bytes()?; + session.pictures.limits.max_memory_bytes = + mandatory + required + extra - usize::from(below); + let result = decoder.read_nal(&skipped, None, |_| { + Err(Error::Internal("unexpected_output")) + }); + if below && extra == 0 { + assert_eq!(result, Err(Error::Limit("reconstruction_memory"))); + } else { + result?; + let session = &mut decoder.session[0]; + assert_eq!(session.pictures.summary.skipped_pictures, 1); + assert_eq!(session.pictures.skipped, Some(1)); + assert!(session.pictures.frame.is_none()); + let kept = + session.pictures.frame.cache.retained_bytes(0, usize::MAX)?; + assert_eq!(kept != 0, warm && extra == retained && !below); + } + } + } + } + } + Ok(()) + } + #[test] fn skipped_hash_messages_never_receive_checked_sample_credit() -> Result<()> { let mut decoder = PictureDecoder::new(DecodeLimits::default()); @@ -891,6 +1732,8 @@ mod tests { fixed, incoming_timestamp: None, incoming_origin: None, + #[cfg(feature = "native-hevc-rows-private")] + activation: None, emit: &mut emit, }; let mut dpb = Dpb::new_reconstruction(); diff --git a/crates/coreav-hevc/src/decoder/rows.rs b/crates/coreav-hevc/src/decoder/rows.rs new file mode 100644 index 000000000..277c392d1 --- /dev/null +++ b/crates/coreav-hevc/src/decoder/rows.rs @@ -0,0 +1,783 @@ +use super::{DecodeLimits, scheduler}; +use crate::{ + Error, PictureInfo, Result, + dpb::Publication, + native_rows::{OutputHeader, RowReceive, SinkCost}, + params::Sps, + reconstruction::continuation_capacity, +}; +use std::sync::Arc; + +pub(super) const ATOMIC_RUNTIME_BYTES: usize = 65536; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn virtual_receive_offsets_follow_unreturned_old_geometry_debt() -> Result<()> { + let mut owner = Rows::allocate( + Plan { + ceiling: 4, + workers: 4, + lag: 3, + frame_bytes: 384, + reserved: 1 << 20, + }, + (0, [0; 32]), + 0, + 3, + Vec::new(), + 0, + )?; + let rows = &mut owner[0]; + rows.validated = 0; + let first = rows.receive()?; + assert_eq!(first.consume, Some(0)); + assert!(!first.pop); + let second = rows.receive()?; + assert_eq!(second.consume, Some(1)); + assert!(!second.pop); + rows.validated = 1; + let third = rows.receive()?; + assert_eq!(third.consume, Some(2)); + assert!(third.pop); + assert_eq!(rows.debt()?, 2); + let empty = rows.receive()?; + assert!(empty.consume.is_none()); + assert!(!empty.pop); + rows.validated = 3; + for debt in [1, 0] { + let catch_up = rows.receive()?; + assert!(catch_up.consume.is_none()); + assert!(catch_up.pop); + assert_eq!(rows.debt()?, debt); + } + Ok(()) + } +} + +pub(super) struct Pending { + pub(super) header: OutputHeader, + pub(super) publication: Arc, + pub(super) fence: u64, +} + +#[derive(Clone, Copy)] +pub(super) struct Plan { + pub(super) ceiling: usize, + pub(super) workers: usize, + pub(super) lag: usize, + pub(super) frame_bytes: usize, + pub(super) reserved: usize, +} + +pub(super) struct Rows { + pub(super) identity: (u32, [u8; 32]), + pub(super) plan: Plan, + pub(super) pending: Vec, + pub(super) operation_limit: usize, + pub(super) emitted: u64, + pub(super) validated: u64, + pub(super) serial_received: u64, + pub(super) returned: u64, + pub(super) reconciled: u64, +} + +pub(super) struct Admission<'a> { + pub(super) info: PictureInfo, + pub(super) sps: &'a Sps, + pub(super) limits: DecodeLimits, + pub(super) fixed: usize, + pub(super) sink: SinkCost, + pub(super) allow_zero: bool, + pub(super) positive: bool, +} + +fn sum(values: &[usize]) -> Result { + values + .iter() + .try_fold(0usize, |total, value| total.checked_add(*value)) + .ok_or(Error::Limit("reconstruction_memory")) +} + +fn frame_bytes(info: PictureInfo) -> Result { + (info.visible_width as usize) + .checked_mul(info.visible_height as usize) + .and_then(|n| n.checked_mul(3)) + .and_then(|n| n.checked_div(2)) + .and_then(|n| n.checked_mul(if info.bit_depth == 10 { 2 } else { 1 })) + .filter(|n| *n != 0) + .ok_or(Error::Limit("reconstruction_memory")) +} + +impl Admission<'_> { + fn output_records(&self) -> Result { + let pending = usize::try_from(self.sink.max_pending_bytes) + .map_err(|_| Error::Limit("reconstruction_memory"))?; + let maximum = usize::try_from(self.limits.headers.max_pictures).unwrap_or(usize::MAX); + self.sink + .queued_frames + .checked_add((pending / frame_bytes(self.info)?).min(maximum)) + .and_then(|n| n.checked_add(self.limits.threads.clamp(1, 16))) + .map(|n| n.min(maximum)) + .ok_or(Error::Limit("reconstruction_memory")) + } + + pub(super) fn choose(&self, records: Option<(usize, usize)>) -> Result> { + let overflow = || Error::Limit("reconstruction_memory"); + let area = (self.info.width as usize) + .checked_mul(self.info.height as usize) + .ok_or_else(overflow)?; + let frame_bytes = frame_bytes(self.info)?; + let capacity = continuation_capacity(self.info, self.sps.ctu_count as usize)?; + let dpb = self + .sps + .max_dpb + .get(self.sps.sub_layers as usize) + .copied() + .filter(|n| *n != 0) + .ok_or(Error::Internal("row_dpb_capacity"))? as usize; + let ctus = self.sps.ctu_count as usize; + let nal = self + .limits + .headers + .max_nal_bytes + .min(self.sink.max_packet_bytes); + let entries = (self.limits.headers.max_ctus as usize) + .checked_sub(1) + .and_then(|n| n.checked_mul(core::mem::size_of::())) + .ok_or_else(overflow)?; + let parser = sum(&[ + nal.checked_sub(2).ok_or_else(overflow)?, + entries, + ctus.checked_mul(core::mem::size_of::>()) + .ok_or_else(overflow)?, + ])?; + let pending_bytes = usize::try_from(self.sink.max_pending_bytes).map_err(|_| overflow())?; + let ceiling = self.limits.threads.clamp(1, 16); + let count = self.output_records()?; + let queue = self + .sink + .queue_capacity + .max(if records.is_some() { 0 } else { count }); + let output = sum(&[ + pending_bytes, + queue + .checked_mul(self.sink.record_bytes) + .ok_or_else(overflow)?, + count + .checked_mul(self.sink.plane_records_bytes) + .ok_or_else(overflow)?, + self.sink.owner_bytes, + ])?; + let common = sum(&[ + self.fixed, + parser, + output, + core::mem::size_of::(), + 65536, + ceiling + .checked_mul(ATOMIC_RUNTIME_BYTES) + .ok_or_else(overflow)?, + ])?; + let job = + scheduler::continuation_job_bytes(self.info, &capacity, self.sink, dpb - 1, ctus, nal)?; + let owner = sum(&[ + capacity.retained, + Publication::record_bytes(), + crate::reconstruction::RowStore::record_bytes(), + 65536, + ])?; + for lag in (1..ceiling).rev().filter(|_| self.positive) { + for workers in (lag + 1..=ceiling).rev() { + let live = workers.checked_add(1).ok_or_else(overflow)?; + let owners = (dpb - 1) + .checked_add(live) + .and_then(|n| n.checked_add(workers)) + .ok_or_else(overflow)?; + let base = sum(&[ + common, + records.map_or_else( + || (ceiling + 1) * scheduler::job_record_bytes(), + |(_, jobs)| jobs, + ), + records.map_or(ceiling * core::mem::size_of::(), |(pending, _)| { + pending + }), + owners.checked_mul(owner).ok_or_else(overflow)?, + live.checked_mul(job).ok_or_else(overflow)?, + ])?; + let debt = lag.checked_mul(frame_bytes).ok_or_else(overflow)?; + let reserved = base + .checked_add(self.sink.debt_peak_bytes.saturating_sub(debt)) + .ok_or_else(overflow)?; + if reserved + .checked_add(debt) + .is_some_and(|bytes| bytes <= self.limits.max_memory_bytes) + { + return Ok(Some(Plan { + ceiling, + workers, + lag, + frame_bytes, + reserved, + })); + } + } + } + if self.allow_zero { + let reserved = sum(&[ + common, + dpb.checked_mul(capacity.retained).ok_or_else(overflow)?, + capacity.base, + area.checked_mul(2).ok_or_else(overflow)?, + self.sink.debt_peak_bytes, + ])?; + if reserved <= self.limits.max_memory_bytes { + return Ok(Some(Plan { + ceiling, + workers: 1, + lag: 0, + frame_bytes, + reserved, + })); + } + } + Ok(None) + } +} + +impl Rows { + pub(super) fn allocate( + plan: Plan, + identity: (u32, [u8; 32]), + current: u64, + queued: usize, + pending: Vec, + operation_limit: usize, + ) -> Result> { + let mut owner = Vec::new(); + owner + .try_reserve_exact(1) + .map_err(|_| Error::Limit("picture_allocation"))?; + let emitted = u64::try_from(queued).map_err(|_| Error::Limit("reconstruction_memory"))?; + owner.push(Self { + identity, + plan, + pending, + operation_limit, + emitted, + validated: emitted, + serial_received: 0, + returned: 0, + reconciled: current, + }); + owner + .into_boxed_slice() + .try_into() + .map_err(|_| Error::Internal("row_owner")) + } + + pub(super) fn debt(&self) -> Result { + self.serial_received + .checked_sub(self.returned) + .and_then(|n| usize::try_from(n).ok()) + .ok_or(Error::Internal("output_ordinal")) + } + + pub(super) fn heap_bytes(&self) -> Result { + let bytes = self + .pending + .capacity() + .checked_mul(core::mem::size_of::()) + .and_then(|n| n.checked_add(core::mem::size_of::())) + .ok_or(Error::Limit("reconstruction_memory"))?; + if bytes > self.plan.reserved { + return Err(Error::Internal("row_admission")); + } + Ok(bytes) + } + + pub(super) fn receive(&mut self) -> Result { + let consume = if self.serial_received < self.emitted { + let index = self.debt()?; + self.serial_received = self + .serial_received + .checked_add(1) + .ok_or(Error::Limit("decode_work_count"))?; + Some(index) + } else { + None + }; + let pop = self.returned < self.validated; + if pop { + self.returned = self + .returned + .checked_add(1) + .ok_or(Error::Limit("decode_work_count"))?; + } + Ok(RowReceive { consume, pop }) + } +} + +impl super::Observer<'_, F> { + pub(super) fn codec_bytes(&self, dpb: &crate::dpb::Dpb, scratch: usize) -> Result { + sum(&[ + self.fixed, + dpb.heap_bytes()?, + self.pictures + .frame + .as_ref() + .map_or(Ok(0), crate::reconstruction::Frame::heap_bytes)?, + scratch, + self.pictures.frame.scheduler.bytes(Some(dpb))?, + ]) + } + + pub(super) fn row_publish(&mut self, dpb: &crate::dpb::Dpb, scratch: usize) -> Result<()> { + loop { + let Some(rows) = self.pictures.frame.scheduler.rows() else { + return Ok(()); + }; + let Some(front) = rows.pending.first() else { + return Ok(()); + }; + if front.fence > rows.reconciled { + return Ok(()); + } + let header = front.header; + let publication = Arc::clone(&front.publication); + let offset = rows + .validated + .checked_sub(rows.returned) + .and_then(|n| usize::try_from(n).ok()) + .ok_or(Error::Internal("output_ordinal"))?; + let picture = publication + .picture() + .ok_or(Error::InvalidData("missing_output_samples"))?; + let view = picture.view(); + self.emit + .row_sink() + .ok_or(Error::Internal("row_sink"))? + .fill( + offset, + crate::DecodedPicture { + index: header.index, + poc: header.poc, + timestamp: header.timestamp, + origin: header.origin, + info: view.info, + planes: view.planes, + stats: view.stats, + inter: picture.inter_stats(), + }, + )?; + drop(publication); + let rows = self + .pictures + .frame + .scheduler + .rows_mut() + .ok_or(Error::Internal("row_owner"))?; + rows.pending.remove(0); + rows.validated = rows + .validated + .checked_add(1) + .ok_or(Error::Limit("decode_work_count"))?; + let bytes = self.codec_bytes(dpb, scratch)?; + self.pictures.summary.memory_bytes = self + .pictures + .summary + .memory_bytes + .max(bytes.saturating_add(self.pictures.external_memory)); + } + } + + pub(super) fn row_settle(&mut self, dpb: &crate::dpb::Dpb, scratch: usize) -> Result<()> { + self.row_publish(dpb, scratch)?; + while let Some(fence) = self + .pictures + .frame + .scheduler + .rows() + .and_then(|rows| rows.pending.first().map(|front| front.fence)) + { + let result = self.pictures.frame.scheduler.drain( + &mut self.pictures.summary, + &mut self.pictures.stream_remaining, + fence.checked_sub(1), + ); + self.row_publish(dpb, scratch)?; + result?; + if self.pictures.frame.scheduler.rows().is_some_and(|rows| { + rows.pending + .first() + .is_some_and(|front| front.fence == fence) + }) { + return Err(Error::Internal("output_fence")); + } + } + let result = self.pictures.synchronize(); + self.row_publish(dpb, scratch)?; + result + } + + pub(super) fn row_enforce(&mut self, dpb: &crate::dpb::Dpb, scratch: usize) -> Result<()> { + self.row_publish(dpb, scratch)?; + while self + .pictures + .frame + .scheduler + .rows() + .is_some_and(|rows| rows.pending.len() > rows.plan.lag) + { + let fence = self + .pictures + .frame + .scheduler + .rows() + .and_then(|rows| rows.pending.first()) + .map(|front| front.fence) + .ok_or(Error::Internal("output_fence"))?; + let result = self.pictures.frame.scheduler.drain( + &mut self.pictures.summary, + &mut self.pictures.stream_remaining, + fence.checked_sub(1), + ); + self.row_publish(dpb, scratch)?; + result?; + if self.pictures.frame.scheduler.rows().is_some_and(|rows| { + rows.pending + .first() + .is_some_and(|front| front.fence == fence) + }) { + return Err(Error::Internal("output_fence")); + } + } + Ok(()) + } + + pub(super) fn row_output( + &mut self, + emission: crate::dpb::Emission<'_>, + scratch: usize, + ) -> Result<()> { + self.row_publish(emission.dpb, scratch)?; + let immediate = emission.publication.is_none() + || self + .pictures + .frame + .scheduler + .rows() + .is_some_and(|rows| rows.plan.lag == 0); + if immediate { + self.row_settle(emission.dpb, scratch)?; + } + let samples = emission.picture.or_else(|| { + emission + .publication + .and_then(|publication| publication.picture()) + }); + let header = match samples { + Some(samples) => OutputHeader { + index: emission.event.picture_index, + poc: emission.event.poc, + timestamp: samples.timestamp(), + origin: samples.origin(), + info: samples.info(), + }, + None => self + .pictures + .frame + .scheduler + .header(emission.event.picture_index, emission.event.poc)?, + }; + let fence = self.pictures.frame.scheduler.fence()?; + if self + .pictures + .frame + .scheduler + .rows() + .is_none_or(|rows| frame_bytes(header.info).ok() != Some(rows.plan.frame_bytes)) + { + return Err(Error::Internal("row_geometry")); + } + let codec_bytes = self.codec_bytes(emission.dpb, scratch)?; + let mut memory = super::OutputMemory { + codec_bytes, + reserved: &mut self.pictures.external_memory, + limit: self.pictures.limits.max_memory_bytes, + reclaim: Some(&mut self.pictures.frame.cache), + }; + let created = self + .emit + .row_sink() + .ok_or(Error::Internal("row_sink"))? + .prepare(header, &mut memory, true)?; + if !created { + return Ok(()); + } + let rows = self + .pictures + .frame + .scheduler + .rows_mut() + .ok_or(Error::Internal("row_owner"))?; + let offset = rows + .emitted + .checked_sub(rows.returned) + .and_then(|n| usize::try_from(n).ok()) + .ok_or(Error::Internal("output_ordinal"))?; + rows.emitted = rows + .emitted + .checked_add(1) + .ok_or(Error::Limit("decode_work_count"))?; + if immediate { + let samples = samples.ok_or(Error::InvalidData("missing_output_samples"))?; + let view = samples.view(); + self.emit + .row_sink() + .ok_or(Error::Internal("row_sink"))? + .fill( + offset, + crate::DecodedPicture { + index: header.index, + poc: header.poc, + timestamp: header.timestamp, + origin: header.origin, + info: header.info, + planes: view.planes, + stats: view.stats, + inter: samples.inter_stats(), + }, + )?; + let rows = self + .pictures + .frame + .scheduler + .rows_mut() + .ok_or(Error::Internal("row_owner"))?; + rows.validated = rows + .validated + .checked_add(1) + .ok_or(Error::Limit("decode_work_count"))?; + } else { + if rows.pending.len() == rows.pending.capacity() { + return Err(Error::Internal("output_records")); + } + rows.pending.push(Pending { + header, + publication: Arc::clone( + emission + .publication + .ok_or(Error::Internal("output_publication"))?, + ), + fence, + }); + } + self.row_enforce(emission.dpb, scratch) + } + + pub(super) fn row_admit( + &mut self, + input: &crate::parser::SliceInput<'_>, + dpb: &mut crate::dpb::Dpb, + frame_bytes: usize, + ) -> Result<()> { + let Some(identity) = self.activation else { + return Ok(()); + }; + if self.emit.row_sink().is_none() || self.pictures.limits.threads == 1 { + return Ok(()); + } + if self + .pictures + .frame + .scheduler + .rows() + .is_some_and(|rows| rows.identity == identity) + { + return Ok(()); + } + if self.pictures.frame.scheduler.rows().is_none() { + let sink = self + .emit + .row_sink() + .ok_or(Error::Internal("row_sink"))? + .cost(0, frame_bytes)?; + let initial = Admission { + info: input.picture, + sps: input.sps, + limits: self.pictures.limits, + fixed: self.fixed, + sink, + allow_zero: false, + positive: true, + } + .choose(None); + match initial { + Ok(Some(_)) => {} + Ok(None) | Err(Error::Limit("reconstruction_memory")) => return Ok(()), + Err(error) => return Err(error), + } + } + self.row_settle(dpb, input.scratch_bytes)?; + dpb.unshare(); + self.pictures.frame.cache.clear(); + let debt = self + .pictures + .frame + .scheduler + .rows() + .map_or(Ok(0), Rows::debt)?; + if debt == 0 { + self.pictures.frame.scheduler.retire_rows()?; + } else if let Some(rows) = self.pictures.frame.scheduler.rows_mut() { + rows.pending = Vec::new(); + } + let sink = self + .emit + .row_sink() + .ok_or(Error::Internal("row_sink"))? + .cost(debt, frame_bytes)?; + self.pictures.frame.scheduler.release_row_jobs(); + let qualification_bytes = sum(&[ + self.codec_bytes(dpb, input.scratch_bytes)?, + self.pictures.external_memory, + ])?; + let available = self + .pictures + .limits + .max_memory_bytes + .saturating_sub(qualification_bytes); + let mut probe = crate::native_rows::CapacityProbe::new(available); + let qualified = + scheduler::qualify_capacities(input, self.pictures.limits, sink, &mut probe) + && self + .emit + .row_sink() + .ok_or(Error::Internal("row_sink"))? + .qualify(input.picture, &mut probe); + self.pictures.summary.memory_bytes = self + .pictures + .summary + .memory_bytes + .max(sum(&[qualification_bytes, probe.peak_bytes])?); + if !qualified && debt == 0 { + return Ok(()); + } + let admission = Admission { + info: input.picture, + sps: input.sps, + limits: self.pictures.limits, + fixed: self.fixed, + sink, + allow_zero: debt != 0, + positive: qualified, + }; + let mut pending = Vec::new(); + let mut plan = None; + let mut owner = None; + let reuse = self.pictures.frame.scheduler.rows().is_some(); + let operation_limit = (input.sps.ctu_count as usize).min(sink.max_nals); + let external = self.pictures.external_memory; + let mut peak = 0; + let reservation = (|| { + pending + .try_reserve_exact(self.pictures.limits.threads.clamp(1, 16)) + .map_err(|_| Error::Limit("picture_allocation"))?; + let pending_bytes = pending + .capacity() + .checked_mul(core::mem::size_of::()) + .ok_or(Error::Limit("reconstruction_memory"))?; + let jobs = self + .pictures + .frame + .scheduler + .reserve_row_jobs(self.pictures.limits.threads.clamp(1, 16))?; + let codec_bytes = self + .codec_bytes(dpb, input.scratch_bytes)? + .checked_add(pending_bytes) + .and_then(|n| { + n.checked_add(if reuse { + 0 + } else { + core::mem::size_of::() + }) + }) + .ok_or(Error::Limit("reconstruction_memory"))?; + let mut memory = super::OutputMemory { + codec_bytes, + reserved: &mut self.pictures.external_memory, + limit: self.pictures.limits.max_memory_bytes, + reclaim: None, + }; + self.emit + .row_sink() + .ok_or(Error::Internal("row_sink"))? + .reserve_queue(admission.output_records()?, &mut memory, &mut |capacity| { + peak = peak.max(sum(&[ + codec_bytes, + external, + if capacity == sink.queue_capacity { + 0 + } else { + capacity + .checked_mul(sink.record_bytes) + .ok_or(Error::Limit("reconstruction_memory"))? + }, + ])?); + let actual = Admission { + sink: SinkCost { + queue_capacity: capacity, + ..sink + }, + ..admission + }; + plan = actual.choose(Some((pending_bytes, jobs)))?; + if let Some(plan) = plan.filter(|_| !reuse) { + owner = Some(Rows::allocate( + plan, + identity, + input.index, + sink.queued_frames, + core::mem::take(&mut pending), + operation_limit, + )?); + } + Ok(plan.is_some()) + })?; + Ok(()) + })(); + self.pictures.summary.memory_bytes = self.pictures.summary.memory_bytes.max(peak); + match reservation { + Ok(()) => {} + Err(Error::Limit(_)) if debt == 0 => { + self.pictures.frame.scheduler.release_row_jobs(); + return Ok(()); + } + Err(error) => return Err(error), + } + let Some(plan) = plan else { + self.pictures.frame.scheduler.release_row_jobs(); + return if debt == 0 { + Ok(()) + } else { + Err(Error::Limit("row_admission")) + }; + }; + if reuse { + self.pictures.frame.scheduler.replace_row_plan( + identity, + plan, + pending, + operation_limit, + input.index, + )?; + } else { + self.pictures.frame.scheduler.rows = Some(owner.ok_or(Error::Internal("row_owner"))?); + } + Ok(()) + } +} diff --git a/crates/coreav-hevc/src/decoder/scheduler.rs b/crates/coreav-hevc/src/decoder/scheduler.rs new file mode 100644 index 000000000..1acc2677d --- /dev/null +++ b/crates/coreav-hevc/src/decoder/scheduler.rs @@ -0,0 +1,1382 @@ +use super::*; +use crate::{ + cabac::Cabac, + ctu::Layout, + dpb::{Publication, ReferenceIdentity, ReferenceProvider}, + params::{Pps, Sps}, + parser::SliceSyntax, + reconstruction::{ + RetainedPicture, RowPublisher, RowStore, + raster::{Completion, Producer}, + }, +}; +use std::sync::{ + Arc, Mutex, OnceLock, + atomic::{AtomicBool, Ordering}, +}; + +const STACK_BYTES: usize = 2 * 1024 * 1024; + +#[cfg(feature = "native-hevc-rows-private")] +pub(super) fn job_record_bytes() -> usize { + core::mem::size_of::() +} + +#[cfg(feature = "native-hevc-rows-private")] +pub(super) fn qualify_capacities( + input: &SliceInput<'_>, + limits: DecodeLimits, + sink: crate::native_rows::SinkCost, + probe: &mut crate::native_rows::CapacityProbe, +) -> bool { + let ctus = input.sps.ctu_count as usize; + let nal = limits.headers.max_nal_bytes.min(sink.max_packet_bytes); + let Some(&references) = input.sps.max_dpb.get(input.sps.sub_layers as usize) else { + return false; + }; + let references = references as usize; + crate::reconstruction::qualify_capacities(input.picture, ctus, probe) + && [ + input.original.len(), + input.rbsp.len(), + nal, + nal.saturating_sub(2), + ] + .into_iter() + .all(|count| probe.check::(count)) + && [ + input.entry_lengths.len(), + limits.headers.max_ctus.saturating_sub(1) as usize, + ] + .into_iter() + .all(|count| probe.check::(count)) + && probe.check::>(ctus) + && probe.check::(ctus.min(sink.max_nals)) + && (0..references).all(|count| probe.check::<(ReferenceIdentity, Arc)>(count)) + && probe.check::(1) +} + +#[cfg(feature = "native-hevc-rows-private")] +pub(super) fn continuation_job_bytes( + info: PictureInfo, + capacity: &crate::reconstruction::ContinuationCapacity, + sink: crate::native_rows::SinkCost, + references: usize, + ctus: usize, + nal: usize, +) -> Result { + let overflow = || Error::Limit("reconstruction_memory"); + let operations = ctus.min(sink.max_nals); + let sources = sink + .max_packet_bytes + .checked_mul(2) + .and_then(|n| n.checked_add(operations.checked_mul(core::mem::size_of::())?)) + .ok_or_else(overflow)?; + let area = (info.width as usize) + .checked_mul(info.height as usize) + .ok_or_else(overflow)?; + let fallback = area + .checked_mul(4) + .and_then(|n| n.checked_add(nal)) + .and_then(|n| n.checked_add(4096 + 32)) + .ok_or_else(overflow)?; + let records = references + .checked_mul(2) + .and_then(|n| n.checked_mul(core::mem::size_of::<(ReferenceIdentity, Arc)>())) + .and_then(|n| n.checked_add(core::mem::size_of::())) + .and_then(|n| { + n.checked_add( + 4 * core::mem::size_of::() + core::mem::size_of::(), + ) + }) + .ok_or_else(overflow)?; + [ + capacity.base, + capacity.row_extra.max(fallback), + sources, + records, + STACK_BYTES + 65536, + ] + .into_iter() + .try_fold(0usize, |total, bytes| total.checked_add(bytes)) + .ok_or_else(overflow) +} + +#[cfg(test)] +#[derive(Default)] +pub(super) struct TestHook { + entered: Mutex, + ready: std::sync::Condvar, + overlap: bool, + completed: std::sync::atomic::AtomicU64, + raster_conversions: std::sync::atomic::AtomicUsize, + dependent_rows: bool, + fault: Option, +} + +#[cfg(test)] +enum TestFault { + Panic(u64), + ReverseErrors, + Deadline, +} + +#[cfg(test)] +#[derive(Clone, Copy)] +enum AdmissionFault { + Input, + Snapshot, + Record, + Spawn, + SpawnAfterOne, +} + +#[cfg(test)] +impl TestHook { + fn enter(&self, index: u64) -> Result<()> { + let mut entered = self + .entered + .lock() + .unwrap_or_else(|error| error.into_inner()); + *entered |= 1 << index; + self.ready.notify_all(); + if self.overlap && matches!(index, 1 | 2) { + let (entered, _) = self + .ready + .wait_timeout_while(entered, std::time::Duration::from_secs(3), |entered| { + *entered & 6 != 6 + }) + .unwrap_or_else(|error| error.into_inner()); + if *entered & 6 != 6 { + return Err(Error::Internal("picture_jobs_did_not_overlap")); + } + } else { + drop(entered); + } + if matches!(self.fault, Some(TestFault::ReverseErrors)) && index == 1 { + let entered = self + .entered + .lock() + .unwrap_or_else(|error| error.into_inner()); + let (_entered, result) = self + .ready + .wait_timeout_while(entered, std::time::Duration::from_secs(3), |_| { + self.completed.load(Ordering::Acquire) & 4 == 0 + }) + .unwrap_or_else(|error| error.into_inner()); + if result.timed_out() { + return Err(Error::Internal("picture_job_completion_order")); + } + return Err(Error::InvalidData("earlier_picture_job")); + } + match self.fault { + Some(TestFault::Panic(fault)) if fault == index => panic!("picture job panic"), + Some(TestFault::ReverseErrors) if index == 2 => { + Err(Error::InvalidData("later_picture_job")) + } + Some(TestFault::Deadline) if matches!(index, 1 | 2) => { + Err(Error::Limit("decode_deadline")) + } + _ => Ok(()), + } + } + + fn complete(&self, index: u64) { + let _entered = self + .entered + .lock() + .unwrap_or_else(|error| error.into_inner()); + self.completed.fetch_or(1 << index, Ordering::Release); + self.ready.notify_all(); + } +} + +struct OwnedSlice { + index: u64, + hash: Option, + header: crate::SliceHeader, + picture: PictureInfo, + original: Vec, + rbsp: Vec, + entries: Vec, + syntax: SliceSyntax, + sps: Sps, + pps: Pps, + layout: Layout, +} + +impl OwnedSlice { + fn new(input: SliceInput<'_>, memory: &mut Memory<'_>) -> Result { + fn copy(input: &[T], memory: &mut Memory<'_>) -> Result> { + memory.charge( + input + .len() + .checked_mul(core::mem::size_of::()) + .ok_or(Error::Limit("reconstruction_memory"))?, + )?; + let mut output = Vec::new(); + output + .try_reserve_exact(input.len()) + .map_err(|_| Error::Limit("picture_allocation"))?; + memory.charge( + (output.capacity() - input.len()) + .checked_mul(core::mem::size_of::()) + .ok_or(Error::Limit("reconstruction_memory"))?, + )?; + output.extend_from_slice(input); + Ok(output) + } + Ok(Self { + index: input.index, + hash: None, + header: input.header, + picture: input.picture, + original: copy(input.original, memory)?, + rbsp: copy(input.rbsp, memory)?, + entries: copy(input.entry_lengths, memory)?, + syntax: input.syntax, + sps: *input.sps, + pps: *input.pps, + layout: *input.layout, + }) + } + + fn bytes(&self) -> usize { + self.original.capacity() + + self.rbsp.capacity() + + self.entries.capacity() * core::mem::size_of::() + } + + fn input(&self) -> SliceInput<'_> { + SliceInput { + index: self.index, + header: self.header, + picture: self.picture, + original: &self.original, + rbsp: &self.rbsp, + entry_lengths: &self.entries, + syntax: self.syntax, + sps: &self.sps, + pps: &self.pps, + layout: &self.layout, + scratch_bytes: self.rbsp.capacity() + + self.entries.capacity() * core::mem::size_of::(), + } + } +} + +#[derive(Default)] +pub(super) struct ReadyReferences { + entries: Vec<(ReferenceIdentity, Arc)>, + #[cfg(test)] + hook: Option<(u64, Arc)>, +} + +impl ReferenceProvider for ReadyReferences { + fn reference(&self, identity: ReferenceIdentity) -> Result<&RetainedPicture> { + self.publication(identity)? + .picture() + .ok_or(Error::InvalidData("unavailable reference samples")) + } + + fn reference_info(&self, identity: ReferenceIdentity) -> Result { + self.publication(identity)?.reference_info(identity) + } + + fn read_reference( + &self, + identity: ReferenceIdentity, + support: crate::reconstruction::ReadSupport, + ) -> Result> { + self.publication(identity)? + .read_reference(identity, support) + } +} + +impl ReadyReferences { + fn publication(&self, identity: ReferenceIdentity) -> Result<&Publication> { + #[cfg(test)] + if let Some((index, hook)) = &self.hook { + hook.enter(*index)?; + } + self.entries + .iter() + .find(|(entry, _)| entry.index == identity.index && entry.poc == identity.poc) + .map(|(_, publication)| publication.as_ref()) + .ok_or(Error::InvalidData("unavailable reference samples")) + } +} + +struct RasterReferences<'a> { + ready: &'a ReadyReferences, + consumer: u64, +} + +impl RasterReferences<'_> { + fn publication(&self, identity: ReferenceIdentity) -> Result<&Publication> { + let publication = self.ready.publication(identity)?; + if publication.rows.is_some() && identity.index >= self.consumer { + return Err(Error::Internal("reference_row_order")); + } + Ok(publication) + } +} + +impl ReferenceProvider for RasterReferences<'_> { + fn reference(&self, identity: ReferenceIdentity) -> Result<&RetainedPicture> { + self.publication(identity)? + .picture() + .ok_or(Error::InvalidData("unavailable reference samples")) + } + fn reference_info(&self, identity: ReferenceIdentity) -> Result { + self.publication(identity)?.reference_info(identity) + } + fn read_reference( + &self, + identity: ReferenceIdentity, + support: crate::reconstruction::ReadSupport, + ) -> Result> { + self.publication(identity)? + .read_reference(identity, support) + } +} + +#[derive(Default)] +struct Execution { + active: [Option>; 16], + maximum: usize, +} + +#[expect( + clippy::large_enum_variant, + reason = "Keep the whole-picture fallback inline without another allocation" +)] +enum PictureWork { + Whole(Frame), + Raster(Producer, RowPublisher), +} + +#[expect( + clippy::large_enum_variant, + reason = "Keep the whole-picture fallback inline without another allocation" +)] +enum PictureOutcome { + Whole(Frame, Result<()>), + Raster(Result<(Completion, RowPublisher)>), +} + +struct Work { + index: u64, + picture: PictureWork, + operations: Vec, + references: Arc, + budget: u64, + reservation: usize, + finalize: bool, + execution: Arc>, + overlap: Arc, + #[cfg(all(feature = "native-hevc-rows-private", feature = "test-support"))] + row_mode: bool, +} + +pub(super) struct Outcome { + picture: PictureOutcome, + pub(super) consumed: u64, + overlap: bool, + #[cfg(all(feature = "native-hevc-rows-private", feature = "test-support"))] + row_overlap: bool, +} + +impl Outcome { + fn result(&self, stream_remaining: u64) -> Result<()> { + let result = match &self.picture { + PictureOutcome::Whole(_, result) => *result, + PictureOutcome::Raster(result) => result.as_ref().map(|_| ()).map_err(|error| *error), + }; + match result { + Err(Error::Limit("cabac_bin_budget")) if stream_remaining == 0 => { + Err(Error::Limit("stream_cabac_bin_budget")) + } + result => result, + } + } +} + +impl Work { + fn run(self) -> Outcome { + { + let mut execution = self + .execution + .lock() + .unwrap_or_else(|error| error.into_inner()); + let count = execution.active.iter().flatten().count(); + if count != 0 { + self.overlap.store(true, Ordering::Relaxed); + for active in execution.active.iter().flatten() { + active.store(true, Ordering::Relaxed); + } + } + let Some(slot) = execution.active.iter_mut().find(|slot| slot.is_none()) else { + return Outcome { + picture: match self.picture { + PictureWork::Whole(frame) => PictureOutcome::Whole( + frame, + Err(Error::Internal("picture_worker_limit")), + ), + PictureWork::Raster(_, publisher) => { + publisher.fail(); + PictureOutcome::Raster(Err(Error::Internal("picture_worker_limit"))) + } + }, + consumed: 0, + overlap: false, + #[cfg(all(feature = "native-hevc-rows-private", feature = "test-support"))] + row_overlap: false, + }; + }; + *slot = Some(Arc::clone(&self.overlap)); + execution.maximum = execution.maximum.max(count + 1); + } + let mut budget = self.budget; + let fixed = self + .operations + .capacity() + .checked_mul(core::mem::size_of::()) + .and_then(|fixed| { + self.operations + .iter() + .try_fold(fixed, |bytes, slice| bytes.checked_add(slice.bytes())) + }); + let picture = match self.picture { + PictureWork::Whole(mut frame) => { + let result = crate::reconstruction::caught(|| { + let fixed = fixed.ok_or(Error::Limit("reconstruction_memory"))?; + for slice in &self.operations { + frame.decode_full( + slice.input(), + self.references.as_ref(), + &mut budget, + fixed, + self.reservation, + 1, + None, + )?; + if let Some(hash) = slice.hash { + frame.hash_reclaiming(hash, fixed, self.reservation, None)?; + } + } + if self.finalize { + frame.finalize(fixed, self.reservation)?; + } + Ok(()) + }); + PictureOutcome::Whole(frame, result) + } + PictureWork::Raster(mut producer, mut publisher) => { + let result = crate::reconstruction::caught(|| { + let references = RasterReferences { + ready: self.references.as_ref(), + consumer: self.index, + }; + let fixed = fixed + .ok_or(Error::Limit("reconstruction_memory"))? + .checked_add(publisher.bytes()?) + .ok_or(Error::Limit("reconstruction_memory"))?; + #[cfg(test)] + if self + .references + .hook + .as_ref() + .is_some_and(|(index, hook)| *index == 0 && hook.dependent_rows) + { + publisher.wait_for_reader()?; + } + for slice in &self.operations { + producer.decode( + slice.input(), + &references, + &mut budget, + fixed, + self.reservation, + &mut publisher, + )?; + if let Some(hash) = slice.hash { + publisher.hash(&mut producer, hash)?; + } + } + Ok((producer.finish()?, publisher)) + }); + PictureOutcome::Raster(result) + } + }; + #[cfg(test)] + if let Some((index, hook)) = &self.references.hook { + hook.complete(*index); + } + { + let mut execution = self + .execution + .lock() + .unwrap_or_else(|error| error.into_inner()); + for active in &mut execution.active { + if active + .as_ref() + .is_some_and(|active| Arc::ptr_eq(active, &self.overlap)) + { + *active = None; + } + } + } + #[cfg(all(feature = "native-hevc-rows-private", feature = "test-support"))] + let row_overlap = self.row_mode + && matches!(&picture, PictureOutcome::Raster(_)) + && self.overlap.load(Ordering::Relaxed); + Outcome { + picture, + consumed: self.budget - budget, + overlap: self.overlap.load(Ordering::Relaxed), + #[cfg(all(feature = "native-hevc-rows-private", feature = "test-support"))] + row_overlap, + } + } +} + +struct Job { + index: u64, + #[cfg(feature = "native-hevc-rows-private")] + info: PictureInfo, + publication: Arc, + references: Arc, + handle: Option>>, + outcome: Option>, + reservation: usize, + budget: u64, + prior_bins: u64, + timestamp: Option, + origin: Option, +} + +pub(super) struct JobIdentity { + pub(super) prior_bins: u64, + pub(super) timestamp: Option, + pub(super) origin: Option, +} + +#[derive(Default)] +pub(super) struct Scheduler { + pub(super) enabled: bool, + operations: Vec, + references: ReadyReferences, + jobs: Vec, + bound: u64, + reserved_bins: u64, + reservation: usize, + scratch: usize, + execution: Option>>, + failure: Option, + #[cfg(feature = "native-hevc-rows-private")] + pub(super) rows: Option>, + #[cfg(test)] + pub(super) hook: Option>, + #[cfg(test)] + admission_fault: Option, +} + +impl Drop for Scheduler { + fn drop(&mut self) { + for job in &self.jobs { + if let Some(rows) = &job.publication.rows { + rows.fail(); + } + } + for job in &mut self.jobs { + if let Some(handle) = job.handle.take() { + let _ = handle.join(); + } + } + } +} + +impl Scheduler { + #[cfg(feature = "native-hevc-rows-private")] + pub(super) fn reserve_row_jobs(&mut self, ceiling: usize) -> Result { + if !self.jobs.is_empty() { + return Err(Error::Internal("row_job_storage")); + } + self.jobs = Vec::new(); + self.jobs + .try_reserve_exact(ceiling + 1) + .map_err(|_| Error::Limit("picture_allocation"))?; + self.jobs + .capacity() + .checked_mul(core::mem::size_of::()) + .ok_or(Error::Limit("reconstruction_memory")) + } + + #[cfg(feature = "native-hevc-rows-private")] + pub(super) fn release_row_jobs(&mut self) { + if self.jobs.is_empty() { + self.jobs = Vec::new(); + } + } + + #[cfg(feature = "native-hevc-rows-private")] + pub(super) fn retire_rows(&mut self) -> Result<()> { + if !self.jobs.is_empty() || self.rows().is_some_and(|rows| !rows.pending.is_empty()) { + return Err(Error::Internal("row_retirement")); + } + self.rows = None; + self.release_row_jobs(); + Ok(()) + } + + #[cfg(feature = "native-hevc-rows-private")] + pub(super) fn replace_row_plan( + &mut self, + identity: (u32, [u8; 32]), + plan: super::rows::Plan, + mut pending: Vec, + operation_limit: usize, + current: u64, + ) -> Result<()> { + if !self.jobs.is_empty() { + return Err(Error::Internal("row_job_storage")); + } + if plan.lag == 0 { + pending = Vec::new(); + self.release_row_jobs(); + } + let rows = self.rows_mut().ok_or(Error::Internal("row_owner"))?; + rows.identity = identity; + rows.plan = plan; + rows.pending = pending; + rows.operation_limit = operation_limit; + rows.reconciled = rows.reconciled.max(current); + Ok(()) + } + #[cfg(feature = "native-hevc-rows-private")] + pub(super) fn rows(&self) -> Option<&super::rows::Rows> { + self.rows.as_deref().and_then(|rows| rows.first()) + } + + #[cfg(feature = "native-hevc-rows-private")] + pub(super) fn rows_mut(&mut self) -> Option<&mut super::rows::Rows> { + self.rows.as_deref_mut().and_then(|rows| rows.first_mut()) + } + + #[cfg(feature = "native-hevc-rows-private")] + pub(super) fn fence(&self) -> Result { + let issued = self + .jobs + .last() + .map(|job| { + job.index + .checked_add(1) + .ok_or(Error::Limit("picture_count")) + }) + .transpose()? + .unwrap_or(0); + Ok(issued.max(self.rows().map_or(0, |rows| rows.reconciled))) + } + + #[cfg(feature = "native-hevc-rows-private")] + pub(super) fn confirm(&mut self, index: u64) -> Result<()> { + if let Some(rows) = self.rows_mut() { + rows.reconciled = rows + .reconciled + .max(index.checked_add(1).ok_or(Error::Limit("picture_count"))?); + } + Ok(()) + } + + #[cfg(feature = "native-hevc-rows-private")] + pub(super) fn header(&self, index: u64, poc: i32) -> Result { + let job = self + .jobs + .iter() + .find(|job| job.index == index) + .ok_or(Error::InvalidData("missing_output_samples"))?; + Ok(crate::native_rows::OutputHeader { + index, + poc, + timestamp: job.timestamp, + origin: job.origin, + info: job.info, + }) + } + pub(super) fn join_pending(&mut self) { + for job in &mut self.jobs { + if let Some(handle) = job.handle.take() { + job.outcome = Some( + handle + .join() + .unwrap_or(Err(Error::Internal("reconstruction_panic"))), + ); + } + } + } + + pub(super) fn has_input(&self) -> bool { + !self.operations.is_empty() + } + + #[cfg(hevc_thread_diagnostics)] + pub(super) fn has_work(&self) -> bool { + self.has_input() || !self.jobs.is_empty() + } + + pub(super) fn queue_hash(&mut self, info: SeiInfo) -> bool { + let Some(slice) = self.operations.last_mut() else { + return false; + }; + if slice.hash.is_some() { + return false; + } + slice.hash = Some(info); + true + } + + pub(super) fn full(&self, threads: usize) -> bool { + #[cfg(feature = "native-hevc-rows-private")] + let threads = self.rows().map_or(threads, |rows| rows.plan.workers); + self.jobs.len() >= threads + } + + pub(super) fn current_reservation(&self) -> usize { + self.reservation + } + + pub(super) fn required_references( + &mut self, + dpb: &Dpb, + summary: &mut DecodeSummary, + remaining: &mut u64, + ) -> Result<()> { + for identity in dpb.current_references().entries.iter().flatten() { + if dpb.reference(*identity).is_err() { + crate::thread_diagnostic!(barrier ReferenceUnavailable); + self.drain(summary, remaining, Some(identity.index))?; + } + dpb.reference(*identity)?; + } + Ok(()) + } + + pub(super) fn bytes(&self, dpb: Option<&Dpb>) -> Result { + let mut bytes = self.jobs.capacity() * core::mem::size_of::(); + bytes = bytes + .checked_add(self.reservation) + .ok_or(Error::Limit("reconstruction_memory"))?; + for job in &self.jobs { + bytes = bytes + .checked_add(job.reservation) + .ok_or(Error::Limit("reconstruction_memory"))?; + } + #[cfg(feature = "native-hevc-rows-private")] + if let Some(rows) = self.rows() { + bytes = bytes + .checked_add(rows.heap_bytes()?) + .ok_or(Error::Limit("reconstruction_memory"))?; + } + #[cfg(feature = "native-hevc-rows-private")] + let pending = self.rows().into_iter().flat_map(|rows| { + rows.pending + .iter() + .map(|pending| (pending.header.index, &pending.publication)) + }); + #[cfg(not(feature = "native-hevc-rows-private"))] + let pending = core::iter::empty(); + let mut seen = [None; 304]; + let mut count = 0; + for (index, publication) in self + .references + .entries + .iter() + .chain( + self.jobs + .iter() + .flat_map(|job| job.references.entries.iter()), + ) + .map(|(identity, publication)| (identity.index, publication)) + .chain(self.jobs.iter().map(|job| (job.index, &job.publication))) + .chain(pending) + { + if dpb + .and_then(|dpb| dpb.publication(index)) + .is_some_and(|owner| Arc::ptr_eq(owner, publication)) + { + continue; + } + let pointer = Arc::as_ptr(publication); + if seen.iter().take(count).any(|entry| *entry == Some(pointer)) { + continue; + } + if count == seen.len() { + return Err(Error::Internal("reference_owners")); + } + *seen + .get_mut(count) + .ok_or(Error::Internal("reference_owners"))? = Some(pointer); + count += 1; + bytes = bytes + .checked_add(publication.bytes()?) + .ok_or(Error::Limit("reconstruction_memory"))?; + } + Ok(bytes) + } + + pub(super) fn drain( + &mut self, + summary: &mut DecodeSummary, + remaining: &mut u64, + through: Option, + ) -> Result<()> { + if let Some(error) = self.failure { + return Err(error); + } + let result = self.drain_inner(summary, remaining, through); + if let Err(error) = result { + self.failure = Some(error); + } + result + } + + fn drain_inner( + &mut self, + summary: &mut DecodeSummary, + remaining: &mut u64, + through: Option, + ) -> Result<()> { + while !self.jobs.is_empty() { + if through.is_some_and(|index| self.jobs.first().is_some_and(|job| job.index > index)) { + break; + } + let mut job = self.jobs.remove(0); + let outcome = match job.outcome.take() { + Some(outcome) => outcome?, + None => job + .handle + .take() + .ok_or(Error::Internal("picture_job_handle"))? + .join() + .map_err(|_| Error::Internal("reconstruction_panic"))??, + }; + self.reserved_bins -= job.budget; + *remaining = remaining + .checked_sub(outcome.consumed) + .ok_or(Error::Internal("picture_job_budget"))?; + outcome.result(*remaining)?; + #[cfg(all(feature = "native-hevc-rows-private", feature = "test-support"))] + if outcome.row_overlap { + crate::native_rows::record_row_overlap(); + } + summary.frame_parallel_pictures += u64::from(outcome.overlap); + summary.parallel_pictures += u64::from(outcome.overlap); + if let Some(execution) = &self.execution { + summary.max_workers = summary.max_workers.max( + execution + .lock() + .unwrap_or_else(|error| error.into_inner()) + .maximum, + ); + } + match outcome.picture { + PictureOutcome::Whole(mut frame, _) => { + frame.stats.bins = job.prior_bins + outcome.consumed; + frame.stats.memory_bytes = summary.memory_bytes; + summary.add(frame.stats, frame.inter_stats())?; + job.publication + .samples + .set(frame.retain(job.timestamp, job.origin)?) + .map_err(|_| Error::Internal("picture_job_publication"))?; + } + PictureOutcome::Raster(result) => { + let (mut completion, publisher) = result?; + completion.stats.bins = job.prior_bins + outcome.consumed; + completion.stats.memory_bytes = summary.memory_bytes; + summary.add(completion.stats, completion.inter_stats)?; + publisher.complete(completion, job.timestamp, job.origin)?; + } + } + #[cfg(feature = "native-hevc-rows-private")] + self.confirm(job.index)?; + } + if self.jobs.is_empty() { + #[cfg(feature = "native-hevc-rows-private")] + let retain = self.rows().is_some(); + #[cfg(not(feature = "native-hevc-rows-private"))] + let retain = false; + if !retain { + self.jobs = Vec::new(); + } + self.execution = None; + } + Ok(()) + } + + pub(super) fn references( + &mut self, + dpb: &mut Dpb, + summary: &mut DecodeSummary, + remaining: &mut u64, + ) -> Result { + if !self.references.entries.is_empty() { + return Ok(true); + } + let references = dpb.current_references(); + for identity in references.entries.iter().flatten() { + #[cfg(feature = "native-hevc-rows-private")] + if self.rows().is_some_and(|rows| rows.plan.lag != 0) + && dpb.publication(identity.index).is_some_and(|publication| { + publication.rows.is_some() && publication.reference_info(*identity).is_ok() + }) + { + continue; + } + if dpb.reference(*identity).is_err() { + crate::thread_diagnostic!(barrier ReferenceUnavailable); + self.drain(summary, remaining, Some(identity.index))?; + } + } + let mut entries = Vec::new(); + #[cfg(test)] + if matches!(self.admission_fault, Some(AdmissionFault::Snapshot)) { + return Ok(false); + } + if entries + .try_reserve_exact(references.entries.iter().flatten().count()) + .is_err() + { + crate::thread_diagnostic!(barrier SnapshotAllocation); + return Ok(false); + } + for identity in references.entries.into_iter().flatten() { + #[cfg(feature = "native-hevc-rows-private")] + if self.rows().is_some_and(|rows| rows.plan.lag != 0) { + let consumer = self + .operations + .first() + .ok_or(Error::Internal("picture_job_input"))? + .index; + entries.push((identity, dpb.share_rows(identity, consumer)?)); + continue; + } + entries.push((identity, dpb.share(identity)?)); + } + self.references = ReadyReferences { + entries, + #[cfg(test)] + hook: None, + }; + Ok(true) + } + + pub(super) fn discard_current(&mut self) { + self.operations = Vec::new(); + self.references = ReadyReferences::default(); + self.reservation = 0; + self.scratch = 0; + self.bound = 0; + } + + pub(super) fn queue( + &mut self, + input: SliceInput<'_>, + frame: &Frame, + fixed: usize, + limit: usize, + remaining: u64, + picture_remaining: u64, + ) -> Result { + #[cfg(feature = "native-hevc-rows-private")] + if self + .rows() + .is_some_and(|rows| self.operations.len() >= rows.operation_limit) + { + return Ok(false); + } + let bound = Cabac::max_bins(input.rbsp.len()) + .saturating_add((input.entry_lengths.len() as u64).saturating_mul(128)) + .min(picture_remaining.saturating_sub(self.bound)); + if bound + > remaining + .saturating_sub(self.reserved_bins) + .saturating_sub(self.bound) + { + crate::thread_diagnostic!(barrier QueueBudget); + return Ok(false); + } + let extra = input + .original + .len() + .checked_add(input.rbsp.len()) + .and_then(|n| n.checked_add(input.entry_lengths.len() * 8)) + .ok_or(Error::Limit("reconstruction_memory"))?; + let scratch = (input.picture.width as usize) + .checked_mul(input.picture.height as usize) + .and_then(|n| n.checked_mul(4)) + .and_then(|n| n.checked_add(input.scratch_bytes)) + .and_then(|n| n.checked_add((input.entry_lengths.len() + 1) * 32)) + .and_then(|n| n.checked_add(4096)) + .ok_or(Error::Limit("reconstruction_memory"))? + .max(self.scratch); + let requested = self + .reservation + .checked_add(extra) + .and_then(|n| n.checked_add(2 * core::mem::size_of::())) + .and_then(|n| n.checked_add(core::mem::size_of::())) + .and_then(|n| n.checked_add(scratch)) + .and_then(|n| n.checked_add(STACK_BYTES + 65536 + 4 * core::mem::size_of::())) + .ok_or(Error::Limit("reconstruction_memory"))?; + let Some(available) = limit + .checked_sub(fixed) + .and_then(|n| n.checked_sub(frame.heap_bytes().ok()?)) + else { + crate::thread_diagnostic!(barrier QueueMemory); + return Ok(false); + }; + if requested > available { + crate::thread_diagnostic!(barrier QueueMemory); + return Ok(false); + } + #[cfg(test)] + if matches!(self.admission_fault, Some(AdmissionFault::Input)) { + return Ok(false); + } + let mut memory = Memory::new(0, available)?; + memory.charge(requested - extra)?; + let capacity = self.jobs.capacity(); + #[cfg(test)] + let refuse = matches!(self.admission_fault, Some(AdmissionFault::Record)); + #[cfg(not(test))] + let refuse = false; + #[cfg(feature = "native-hevc-rows-private")] + let bounded = self.rows().is_some(); + #[cfg(not(feature = "native-hevc-rows-private"))] + let bounded = false; + if bounded && self.jobs.len() == self.jobs.capacity() { + return Err(Error::Internal("row_job_storage")); + } + if refuse || (!bounded && self.jobs.try_reserve_exact(1).is_err()) { + crate::thread_diagnostic!(barrier QueueMemory); + return Ok(false); + } + let records = (self.jobs.capacity() - capacity) * core::mem::size_of::(); + if memory + .charge(records.saturating_sub(core::mem::size_of::())) + .is_err() + { + crate::thread_diagnostic!(barrier QueueMemory); + return Ok(false); + } + let slice = match OwnedSlice::new(input, &mut memory) { + Ok(slice) => slice, + Err(Error::Limit(_)) => { + crate::thread_diagnostic!(barrier QueueMemory); + return Ok(false); + } + Err(error) => return Err(error), + }; + let prior_capacity = self.operations.capacity(); + #[cfg(feature = "native-hevc-rows-private")] + let operations = self + .rows() + .filter(|_| self.operations.capacity() == 0) + .map_or(1, |rows| rows.operation_limit); + #[cfg(not(feature = "native-hevc-rows-private"))] + let operations = 1; + if (!bounded || self.operations.capacity() == 0) + && self.operations.try_reserve_exact(operations).is_err() + { + crate::thread_diagnostic!(barrier QueueMemory); + return Ok(false); + } + let actual = self.operations.capacity() * core::mem::size_of::() + + self.operations.iter().map(OwnedSlice::bytes).sum::() + + slice.bytes(); + let reservation = actual + .checked_add(scratch) + .and_then(|n| n.checked_add(STACK_BYTES + 65536 + 4 * core::mem::size_of::())) + .ok_or(Error::Limit("reconstruction_memory"))?; + if Memory::new(reservation + records, available).is_err() { + crate::thread_diagnostic!(barrier QueueMemory); + if self.operations.is_empty() { + self.operations = Vec::new(); + } else { + self.reservation += (self.operations.capacity() - prior_capacity) + * core::mem::size_of::(); + } + return Ok(false); + } + self.operations.push(slice); + crate::thread_diagnostic!(event QueueAdmitted); + self.reservation = reservation; + self.scratch = scratch; + self.bound += bound; + Ok(true) + } + + pub(super) fn run_current(&mut self, frame: Frame, finalize: bool, budget: u64) -> Outcome { + crate::thread_diagnostic!(event CallerJob); + let execution = self + .execution + .get_or_insert_with(|| Arc::new(Mutex::new(Execution::default()))); + Work { + index: frame.index(), + reservation: self.reservation + frame.heap_bytes().unwrap_or(0), + picture: PictureWork::Whole(frame), + operations: core::mem::take(&mut self.operations), + references: Arc::new(core::mem::take(&mut self.references)), + budget, + finalize, + execution: Arc::clone(execution), + overlap: Arc::new(AtomicBool::new(false)), + #[cfg(all(feature = "native-hevc-rows-private", feature = "test-support"))] + row_mode: false, + } + .run() + } + + pub(super) fn flush( + &mut self, + active: &mut Vec, + remaining: &mut u64, + picture_remaining: &mut u64, + ) -> Result<()> { + if let Some(error) = self.failure { + return Err(error); + } + if !self.has_input() { + return Ok(()); + } + crate::thread_diagnostic!(event ForcedFlush); + let frame = active.pop().ok_or(Error::Internal("picture_job_frame"))?; + let outcome = self.run_current(frame, false, (*remaining).min(*picture_remaining)); + self.bound = 0; + self.reservation = 0; + self.scratch = 0; + if self.jobs.is_empty() { + self.execution = None; + #[cfg(feature = "native-hevc-rows-private")] + let retain = self.rows().is_some(); + #[cfg(not(feature = "native-hevc-rows-private"))] + let retain = false; + if !retain { + self.jobs = Vec::new(); + } + } + *remaining -= outcome.consumed; + *picture_remaining -= outcome.consumed; + let result = outcome.result(*remaining); + let PictureOutcome::Whole(frame, _) = outcome.picture else { + return Err(Error::Internal("picture_partial_owner")); + }; + active.push(frame); + if let Err(error) = result { + self.failure = Some(error); + } + result + } + + pub(super) fn seal( + &mut self, + frame: Frame, + dpb: &mut Dpb, + limits: DecodeLimits, + summary: &mut DecodeSummary, + remaining: &mut u64, + identity: JobIdentity, + ) -> Result<()> { + if self.full(limits.threads) { + crate::thread_diagnostic!(barrier WindowFull); + self.drain(summary, remaining, None)?; + } + #[cfg(feature = "native-hevc-rows-private")] + let bounded = self.rows().is_some(); + #[cfg(not(feature = "native-hevc-rows-private"))] + let bounded = false; + if bounded && self.jobs.len() == self.jobs.capacity() { + return Err(Error::Internal("row_job_storage")); + } + if !bounded && self.jobs.try_reserve_exact(1).is_err() { + self.drain(summary, remaining, None)?; + let outcome = self.run_current(frame, true, self.bound); + self.discard_current(); + *remaining -= outcome.consumed; + outcome.result(*remaining)?; + let PictureOutcome::Whole(mut frame, _) = outcome.picture else { + return Err(Error::Internal("picture_partial_owner")); + }; + frame.stats.bins = identity.prior_bins + outcome.consumed; + frame.stats.memory_bytes = summary.memory_bytes; + summary.max_workers = summary.max_workers.max(1); + summary.add(frame.stats, frame.inter_stats())?; + self.execution = None; + #[cfg(feature = "native-hevc-rows-private")] + let index = frame.index(); + dpb.attach(frame.retain(identity.timestamp, identity.origin)?)?; + #[cfg(feature = "native-hevc-rows-private")] + self.confirm(index)?; + return Ok(()); + } + let index = frame.index(); + let info = frame.info(); + let reservation = self + .reservation + .checked_add(frame.heap_bytes()?) + .ok_or(Error::Limit("reconstruction_memory"))?; + let sources = self + .operations + .capacity() + .checked_mul(core::mem::size_of::()) + .and_then(|fixed| { + self.operations + .iter() + .try_fold(fixed, |bytes, slice| bytes.checked_add(slice.bytes())) + }) + .ok_or(Error::Limit("reconstruction_memory"))?; + let allowance = (info.width as usize) + .checked_mul(info.height as usize) + .and_then(|samples| samples.checked_mul(core::mem::size_of::())) + .ok_or(Error::Limit("reconstruction_memory"))?; + let row_limit = sources + .checked_add(frame.heap_bytes()?) + .and_then(|bytes| bytes.checked_add(allowance)) + .ok_or(Error::Limit("reconstruction_memory"))?; + let fixed = sources + .checked_add(RowStore::record_bytes()) + .ok_or(Error::Limit("reconstruction_memory"))?; + let deadline = frame.deadline; + let (picture, rows) = match Producer::try_from_full(frame, fixed, row_limit) { + Ok((producer, backing)) => { + let backing_bytes = backing.heap_bytes()?; + let fixed = sources + .checked_add(producer.heap_bytes()?) + .and_then(|bytes| bytes.checked_add(backing_bytes)) + .ok_or(Error::Limit("reconstruction_memory"))?; + let mut memory = Memory::new(fixed, row_limit)?; + let (rows, publisher) = RowStore::new( + dpb.current_identity(index)?, + index, + backing, + deadline, + &mut memory, + )?; + (PictureWork::Raster(producer, publisher), Some(rows)) + } + Err((frame, _)) => (PictureWork::Whole(frame), None), + }; + #[cfg(test)] + if rows.is_some() { + if let Some(hook) = &self.hook { + hook.raster_conversions.fetch_add(1, Ordering::Relaxed); + } + } + let retained_reservation = match &rows { + Some(rows) => reservation + .checked_sub(rows.bytes()?) + .ok_or(Error::Internal("picture_job_reservation"))?, + None => reservation, + }; + #[cfg(all(feature = "native-hevc-rows-private", feature = "test-support"))] + let row_mode = self.rows().is_some_and(|rows| rows.plan.lag > 0); + let execution = self + .execution + .get_or_insert_with(|| Arc::new(Mutex::new(Execution::default()))); + let publication = Arc::new(Publication { + samples: OnceLock::new(), + rows, + }); + dpb.seal(index, info, Arc::clone(&publication))?; + #[cfg(test)] + { + self.references.hook = self.hook.as_ref().map(|hook| (index, Arc::clone(hook))); + } + let references = Arc::new(core::mem::take(&mut self.references)); + let work = Work { + index, + picture, + operations: core::mem::take(&mut self.operations), + references: Arc::clone(&references), + budget: self.bound, + reservation, + finalize: true, + execution: Arc::clone(execution), + overlap: Arc::new(AtomicBool::new(false)), + #[cfg(all(feature = "native-hevc-rows-private", feature = "test-support"))] + row_mode, + }; + let budget = self.bound; + self.reserved_bins += budget; + self.bound = 0; + self.reservation = 0; + self.scratch = 0; + let background = self.jobs.iter().filter(|job| job.handle.is_some()).count(); + let mut job = Job { + index, + #[cfg(feature = "native-hevc-rows-private")] + info, + publication, + references, + handle: None, + outcome: None, + reservation: retained_reservation, + budget, + prior_bins: identity.prior_bins, + timestamp: identity.timestamp, + origin: identity.origin, + }; + if background < limits.threads.saturating_sub(1) { + let owner = Arc::new(Mutex::new(Some(work))); + let worker = Arc::clone(&owner); + let spawn = || { + std::thread::Builder::new() + .stack_size(STACK_BYTES) + .spawn(move || { + let work = worker + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + .ok_or(Error::Internal("picture_job_owner"))?; + Ok(work.run()) + }) + }; + #[cfg(test)] + let started = if matches!(self.admission_fault, Some(AdmissionFault::Spawn)) + || matches!(self.admission_fault, Some(AdmissionFault::SpawnAfterOne)) + && self.jobs.iter().any(|job| job.handle.is_some()) + { + Err(std::io::Error::other("picture_job_start")) + } else { + spawn() + }; + #[cfg(not(test))] + let started = spawn(); + match started { + Ok(handle) => { + crate::thread_diagnostic!(event SpawnedJob); + job.handle = Some(handle); + } + Err(_) => { + crate::thread_diagnostic!(barrier WorkerSpawn); + crate::thread_diagnostic!(event CallerJob); + let work = owner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + .ok_or(Error::Internal("picture_job_owner"))?; + job.outcome = Some(Ok(work.run())); + } + } + } else { + crate::thread_diagnostic!(event CallerJob); + job.outcome = Some(Ok(work.run())); + } + self.jobs.push(job); + Ok(()) + } +} + +#[cfg(test)] +#[path = "scheduler_tests.rs"] +mod tests; diff --git a/crates/coreav-hevc/src/decoder/scheduler_tests.rs b/crates/coreav-hevc/src/decoder/scheduler_tests.rs new file mode 100644 index 000000000..be379b9eb --- /dev/null +++ b/crates/coreav-hevc/src/decoder/scheduler_tests.rs @@ -0,0 +1,1705 @@ +use super::*; +use crate::tests::writer; +use writer::Writer; + +#[path = "../../test-support/pcm.rs"] +#[expect( + clippy::duplicate_mod, + reason = "Standalone generated fixtures retain their existing test-only visibility" +)] +mod pcm; + +fn profile(writer: &mut Writer) { + writer.u(2, 0); + writer.flag(false); + writer.u(5, 1); + writer.u(32, 0); + writer.u(4, 0); + writer.u(32, 0); + writer.u(12, 0); + writer.u(8, 120); +} + +fn stream(chain: bool) -> Result>> { + stream_kind(chain, 1) +} + +#[path = "../../test-support/raster.rs"] +mod raster; +use raster::stream as raster_stream; + +#[cfg(feature = "native-hevc-rows-private")] +use crate::native_rows::{OutputHeader, RowSink, SinkCost}; +#[cfg(feature = "native-hevc-rows-private")] +struct NativeRowsSink { + headers: Vec, + output: Vec, + packet: usize, +} +#[cfg(feature = "native-hevc-rows-private")] +impl RowSink for NativeRowsSink { + fn prepare( + &mut self, + header: OutputHeader, + memory: &mut OutputMemory<'_>, + bounded: bool, + ) -> Result { + if bounded { + assert!(self.headers.len() < self.headers.capacity()); + } + memory.reserve( + (header.info.visible_width as usize * header.info.visible_height as usize * 3) + + core::mem::size_of::(), + )?; + self.headers.push(header); + Ok(true) + } + fn reserve_queue( + &mut self, + records: usize, + memory: &mut OutputMemory<'_>, + admit: &mut dyn FnMut(usize) -> Result, + ) -> Result { + let mut headers = Vec::new(); + let mut output = Vec::new(); + headers + .try_reserve_exact(records) + .map_err(|_| Error::Limit("picture_allocation"))?; + output + .try_reserve_exact(records) + .map_err(|_| Error::Limit("picture_allocation"))?; + if headers.capacity() != output.capacity() || !admit(headers.capacity())? { + return Ok(false); + } + memory.reserve( + headers.capacity() + * (core::mem::size_of::() + core::mem::size_of::()), + )?; + while !self.headers.is_empty() { + headers.push(self.headers.remove(0)); + } + while !self.output.is_empty() { + output.push(self.output.remove(0)); + } + self.headers = headers; + self.output = output; + Ok(true) + } + fn qualify(&self, _: PictureInfo, _: &mut crate::native_rows::CapacityProbe) -> bool { + true + } + fn fill(&mut self, offset: usize, picture: DecodedPicture<'_>) -> Result<()> { + assert_eq!(self.headers[offset].index, picture.index); + assert_eq!(offset, self.output.len()); + capture(picture, &mut self.output); + Ok(()) + } + fn truncate(&mut self, keep: usize) { + self.headers.truncate(keep); + self.output.truncate(keep); + } + fn cost(&self, debt: usize, _: usize) -> Result { + assert_eq!(debt, 0); + Ok(SinkCost { + max_packet_bytes: self.packet, + max_nals: 4096, + max_pending_bytes: 1 << 20, + queue_capacity: self.headers.capacity(), + queued_frames: self.headers.len(), + record_bytes: core::mem::size_of::() + core::mem::size_of::(), + plane_records_bytes: 3 * core::mem::size_of::>(), + owner_bytes: core::mem::size_of::(), + debt_peak_bytes: 0, + }) + } +} + +#[cfg(feature = "native-hevc-rows-private")] +#[test] +fn dependent_raster_job_reads_a_building_predecessor_before_its_first_publication() -> Result<()> { + let nals = raster_stream(true)?; + let limits = DecodeLimits { + threads: 2, + max_memory_bytes: 128 << 20, + ..DecodeLimits::default() + }; + let serial = decode_raster(&nals, false, limits, None); + let expected = semantic(serial.0?); + #[cfg(feature = "test-support")] + crate::native_rows::begin_evidence(); + let hook = Arc::new(TestHook { + dependent_rows: true, + ..TestHook::default() + }); + let mut decoder = test_decoder(limits); + decoder.initialize()?; + decoder.session[0].pictures.frame.scheduler.hook = Some(Arc::clone(&hook)); + let mut sink = NativeRowsSink { + headers: Vec::new(), + output: Vec::new(), + packet: nals.iter().map(|nal| nal.len() + 4).sum(), + }; + for (index, nal) in nals.iter().enumerate() { + let origin = if index < 3 { + 0 + } else { + 77 + ((index - 3) / 32).min(1) as u64 + }; + decoder.read_nal_rows(nal, Some(origin as i64), Some(origin), &mut sink)?; + } + assert!(decoder.has_native_rows()); + let receive = decoder + .receive_rows()? + .ok_or(Error::Internal("row_owner"))?; + assert!(receive.consume.is_none()); + assert!(!receive.pop); + assert_eq!(hook.completed.load(Ordering::Acquire), 0); + let summary = decoder.finish_rows(&mut sink)?; + assert_eq!(sink.output, serial.1); + assert_eq!(semantic(summary), expected); + assert_eq!(summary.frame_parallel_pictures, 2); + #[cfg(feature = "test-support")] + assert_eq!(crate::native_rows::take_row_overlap(), Some(2)); + assert_eq!(summary.max_workers, 2); + assert_eq!(hook.completed.load(Ordering::Acquire) & 3, 3); + assert_eq!(*hook.entered.lock().unwrap() & 2, 2); + assert_eq!(hook.raster_conversions.load(Ordering::Relaxed), 2); + let mut failed = stream_kind_reorder(false, 1, 2)?; + let mut hash = vec![132, 13, 2]; + hash.extend([0; 12]); + hash.push(0x80); + failed.insert(5, pcm::escaped(40, &hash)); + let serial = decode(&failed, false, limits, None); + assert!(serial.0.is_err()); + assert!(serial.1.is_empty()); + let mut decoder = test_decoder(limits); + let mut sink = NativeRowsSink { + headers: Vec::new(), + output: Vec::new(), + packet: failed.iter().map(|nal| nal.len() + 4).sum(), + }; + let actual = (|| { + for (index, nal) in failed.iter().enumerate() { + decoder.read_nal_rows(nal, Some(index as i64), Some(index as u64), &mut sink)?; + } + assert!(decoder.has_native_rows()); + decoder.finish_rows(&mut sink) + })(); + assert_eq!(actual.map(semantic), serial.0.map(semantic)); + assert_eq!(sink.output, serial.1); + Ok(()) +} + +#[cfg(feature = "native-hevc-rows-private")] +#[test] +fn zero_lag_and_retirement_release_optional_record_capacities() -> Result<()> { + use super::super::rows::{Pending, Plan, Rows}; + for reduce_first in [false, true] { + let plan = Plan { + ceiling: 4, + workers: 4, + lag: 3, + frame_bytes: 384, + reserved: 1 << 20, + }; + let pending = || -> Result> { + let mut records = Vec::new(); + records + .try_reserve_exact(4) + .map_err(|_| Error::Limit("picture_allocation"))?; + Ok(records) + }; + let mut scheduler = Scheduler::default(); + assert!( + scheduler + .rows + .replace(Rows::allocate(plan, (0, [0; 32]), 0, 0, pending()?, 4)?) + .is_none() + ); + let jobs = scheduler.reserve_row_jobs(4)?; + assert!(jobs > 0); + assert!(scheduler.bytes(None)? > core::mem::size_of::() + jobs); + if reduce_first { + scheduler.replace_row_plan( + (1, [1; 32]), + Plan { + workers: 1, + lag: 0, + ..plan + }, + pending()?, + 4, + 1, + )?; + let rows = scheduler.rows().ok_or(Error::Internal("row_owner"))?; + assert_eq!(rows.plan.lag, 0); + assert_eq!(rows.pending.capacity(), 0); + assert_eq!(scheduler.jobs.capacity(), 0); + assert_eq!(scheduler.bytes(None)?, core::mem::size_of::()); + } + scheduler.retire_rows()?; + assert!(scheduler.rows().is_none()); + assert_eq!(scheduler.jobs.capacity(), 0); + assert_eq!(scheduler.bytes(None)?, 0); + } + Ok(()) +} + +#[cfg(feature = "native-hevc-rows-private")] +#[test] +fn capacity_probe_mismatch_preserves_legacy_frames_and_summary() -> Result<()> { + let nals = raster_stream(true)?; + let limits = DecodeLimits { + threads: 2, + max_memory_bytes: 128 << 20, + ..DecodeLimits::default() + }; + let serial = decode_raster(&nals, false, limits, None); + crate::native_rows::FORCE_CAPACITY_MISMATCH.set(true); + let fallback: Result<()> = (|| { + let mut decoder = test_decoder(limits); + let mut sink = NativeRowsSink { + headers: Vec::new(), + output: Vec::new(), + packet: nals.iter().map(|nal| nal.len() + 4).sum(), + }; + for (index, nal) in nals.iter().enumerate() { + let origin = if index < 3 { + 0 + } else { + 77 + ((index - 3) / 32).min(1) as u64 + }; + decoder.read_nal_rows(nal, Some(origin as i64), Some(origin), &mut sink)?; + assert!(!decoder.has_native_rows()); + } + let summary = decoder.finish_rows(&mut sink)?; + assert_eq!(sink.output, serial.1); + assert_eq!(semantic(summary), semantic(serial.0?)); + Ok(()) + })(); + crate::native_rows::FORCE_CAPACITY_MISMATCH.set(false); + fallback?; + Ok(()) +} + +fn decode_raster( + nals: &[Vec], + scheduled: bool, + limits: DecodeLimits, + hook: Option>, +) -> (Result, Vec) { + let mut decoder = test_decoder(limits); + let mut output = Vec::new(); + let result = (|| { + decoder.initialize()?; + decoder.session[0].pictures.frame.scheduler.hook = hook; + for (index, nal) in nals.iter().enumerate() { + let origin = if index < 3 { + 0 + } else { + 77 + ((index - 3) / 32).min(1) as u64 + }; + let mut emit = |picture: DecodedPicture<'_>, _: &mut OutputMemory<'_>| { + capture(picture, &mut output); + Ok(()) + }; + if scheduled { + decoder.read_nal_scheduled(nal, Some(origin as i64), Some(origin), &mut emit)?; + } else { + decoder.read_nal_with_origin(nal, Some(origin as i64), origin, &mut emit)?; + } + decoder.set_external_memory(0)?; + } + decoder.finish(|picture| { + capture(picture, &mut output); + Ok(()) + }) + })(); + (result, output) +} + +#[test] +fn raster_reference_snapshots_reject_self_dependencies_before_any_wait() -> Result<()> { + let info = PictureInfo { + width: 8, + height: 8, + visible_width: 8, + visible_height: 8, + bit_depth: 8, + profile_idc: 1, + ctb_log2: 4, + tiles: false, + wpp: false, + conformance_window: [0; 4], + vui: Default::default(), + }; + let backing = RetainedPicture::for_test(0, info, None); + let identity = ReferenceIdentity { + index: 0, + poc: 0, + long_term: false, + }; + let mut memory = Memory::new(backing.heap_bytes()?, usize::MAX)?; + let (rows, _publisher) = RowStore::new( + identity, + 1, + backing, + Some(std::time::Instant::now() + std::time::Duration::from_millis(20)), + &mut memory, + )?; + let ready = ReadyReferences { + entries: vec![( + identity, + Arc::new(Publication { + samples: OnceLock::new(), + rows: Some(rows), + }), + )], + ..Default::default() + }; + let cycle = RasterReferences { + ready: &ready, + consumer: 0, + }; + assert_eq!( + cycle.reference_info(identity), + Err(Error::Internal("reference_row_order")) + ); + assert!(matches!( + cycle.read_reference( + identity, + crate::reconstruction::ReadSupport::Motion { row: 0 } + ), + Err(Error::Internal("reference_row_order")) + )); + let ordered = RasterReferences { + ready: &ready, + consumer: 1, + }; + assert_eq!(ordered.reference_info(identity)?, info); + Ok(()) +} + +#[test] +fn admitted_raster_jobs_preserve_serial_frames_hash_failures_and_cold_boundaries() -> Result<()> { + raster_job_case(false)?; + raster_job_case(true) +} + +fn raster_job_case(predicted: bool) -> Result<()> { + let nals = raster_stream(predicted)?; + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + let serial = decode_raster(&nals, false, limits, None); + assert_eq!(serial.1.len(), 2); + if predicted { + assert!(serial.1[1].inter.skip_units > 0); + } + let expected = semantic(serial.0?); + for threads in [1, 2, 4, 8] { + let limits = DecodeLimits { threads, ..limits }; + let hook = Arc::new(TestHook::default()); + let actual = decode_raster(&nals, true, limits, Some(Arc::clone(&hook))); + assert_eq!(actual.1, serial.1, "threads={threads}"); + assert_eq!(semantic(actual.0?), expected); + assert_eq!( + hook.raster_conversions.load(Ordering::Relaxed), + if threads == 1 { 0 } else { 2 } + ); + if predicted && threads > 1 { + assert_ne!(*hook.entered.lock().unwrap(), 0); + } + } + let mut low = 0; + let mut high = 8 * 1024 * 1024; + while low + 1 < high { + let middle = low + (high - low) / 2; + if decode_raster( + &nals, + false, + DecodeLimits { + max_memory_bytes: middle, + ..limits + }, + None, + ) + .0 + .is_ok() + { + high = middle; + } else { + low = middle; + } + } + for memory in [ + low, + high, + high + 1, + high + STACK_BYTES, + high + 2 * STACK_BYTES, + 8 * 1024 * 1024, + ] { + let limits = DecodeLimits { + max_memory_bytes: memory, + ..limits + }; + let direct = decode_raster(&nals, false, limits, None); + let scheduled = decode_raster(&nals, true, limits, None); + assert_eq!(scheduled.1, direct.1, "memory={memory}"); + assert_eq!( + scheduled.0.map(semantic), + direct.0.map(semantic), + "memory={memory}" + ); + } + let mut failed = nals; + let mut hash = vec![132, 13, 2]; + hash.extend([0; 12]); + hash.push(0x80); + failed.push(pcm::escaped(40, &hash)); + for threads in [1, 2, 4, 8] { + let limits = DecodeLimits { threads, ..limits }; + let direct = decode_raster(&failed, false, limits, None); + let scheduled = decode_raster(&failed, true, limits, None); + assert!(direct.0.is_err()); + assert_eq!(direct.1.len(), if predicted { 0 } else { 1 }); + assert_eq!(scheduled.1, direct.1); + assert_eq!(scheduled.0.map(semantic), direct.0.map(semantic)); + } + Ok(()) +} + +fn stream_kind(chain: bool, kind: u32) -> Result>> { + stream_kind_reorder(chain, kind, 3) +} + +fn stream_kind_reorder(chain: bool, kind: u32, reorder: u32) -> Result>> { + let fixture = pcm::pcm_fixture(8, 8, false)?; + assert_eq!(fixture.planes.map(|plane| plane.len()), [64, 16, 16]); + let mut vps = Writer::default(); + vps.u(4, 0); + vps.flag(true); + vps.flag(true); + vps.u(6, 0); + vps.u(3, 0); + vps.flag(true); + vps.u(16, 0xffff); + profile(&mut vps); + vps.flag(false); + for value in [3, reorder, 0] { + vps.ue(value); + } + vps.u(6, 0); + vps.ue(0); + vps.flag(false); + vps.flag(false); + let mut sps = Writer::default(); + sps.u(4, 0); + sps.u(3, 0); + sps.flag(true); + profile(&mut sps); + for value in [0, 1, 8, 8] { + sps.ue(value); + } + sps.flag(false); + sps.ue(0); + sps.ue(0); + sps.ue(0); + sps.flag(false); + for value in [3, reorder, 0, 0, 1, 0, 1, 0, 0] { + sps.ue(value); + } + for flag in [false, false, false, true] { + sps.flag(flag); + } + sps.u(4, 7); + sps.u(4, 7); + sps.ue(0); + sps.ue(0); + sps.flag(true); + sps.ue(0); + for _ in 0..5 { + sps.flag(false); + } + let mut nals = vec![ + pcm::escaped(32, &vps.rbsp()), + pcm::escaped(33, &sps.rbsp()), + fixture.sets[2].clone(), + fixture.nal, + ]; + for poc in 1..=2 { + let mut header = Writer::default(); + header.flag(true); + header.ue(0); + header.ue(kind); + header.u(4, poc); + header.flag(false); + header.ue(1); + header.ue(0); + header.ue(if chain { 0 } else { poc - 1 }); + header.flag(true); + header.flag(false); + if kind == 0 { + header.flag(false); + } + header.ue(4); + header.se(0); + let mut rbsp = header.rbsp(); + rbsp.extend([0xfe, 0xe0]); + nals.push(pcm::escaped(1, &rbsp)); + } + Ok(nals) +} + +#[derive(Debug, PartialEq)] +struct Output { + index: u64, + poc: i32, + timestamp: Option, + origin: Option, + info: PictureInfo, + planes: [Vec; 3], + stats: crate::IntraStats, + inter: InterStats, +} + +fn capture(picture: DecodedPicture<'_>, output: &mut Vec) { + let mut stats = picture.stats; + stats.memory_bytes = 0; + output.push(Output { + index: picture.index, + poc: picture.poc, + timestamp: picture.timestamp, + origin: picture.origin, + info: picture.info, + planes: picture.planes.map(|plane| plane.samples.to_vec()), + stats, + inter: picture.inter, + }); +} + +fn decode( + nals: &[Vec], + scheduled: bool, + limits: DecodeLimits, + hook: Option>, +) -> (Result, Vec) { + decode_configuration(nals, scheduled, limits, hook, None) +} + +fn decode_configuration( + nals: &[Vec], + scheduled: bool, + limits: DecodeLimits, + hook: Option>, + fault: Option, +) -> (Result, Vec) { + let mut decoder = test_decoder(limits); + let mut output = Vec::new(); + let result = (|| { + decoder.initialize()?; + decoder.session[0].pictures.frame.scheduler.hook = hook; + decoder.session[0].pictures.frame.scheduler.admission_fault = fault; + for (index, nal) in nals.iter().enumerate() { + let mut emit = |picture: DecodedPicture<'_>, _: &mut OutputMemory<'_>| { + capture(picture, &mut output); + Ok(()) + }; + if scheduled { + decoder.read_nal_scheduled( + nal, + Some(index as i64), + Some(index as u64), + &mut emit, + )?; + } else { + decoder.read_nal_with_origin(nal, Some(index as i64), index as u64, &mut emit)?; + } + decoder.set_external_memory(0)?; + } + decoder.finish(|picture| { + capture(picture, &mut output); + Ok(()) + }) + })(); + (result, output) +} + +fn test_decoder(limits: DecodeLimits) -> PictureDecoder { + let mut decoder = PictureDecoder::new(limits); + decoder.limits.threads = limits.threads.clamp(1, 16); + decoder +} + +fn semantic(mut summary: DecodeSummary) -> DecodeSummary { + summary.parallel_pictures = 0; + summary.frame_parallel_pictures = 0; + summary.max_workers = 0; + summary.memory_bytes = 0; + summary.reconstruction.memory_bytes = 0; + summary +} + +#[test] +fn shared_anchor_siblings_enter_actual_scalar_prediction_before_either_completes() -> Result<()> { + let nals = stream(false)?; + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + let serial = decode(&nals, false, limits, None); + let hook = Arc::new(TestHook { + overlap: true, + ..TestHook::default() + }); + let parallel = decode(&nals, true, limits, Some(Arc::clone(&hook))); + assert_eq!(parallel.1, serial.1); + let summary = parallel.0?; + assert_eq!(*hook.entered.lock().unwrap(), 6); + assert_eq!( + ( + summary.frame_parallel_pictures, + summary.parallel_pictures, + summary.max_workers + ), + (2, 2, 2) + ); + assert_eq!(semantic(summary), semantic(serial.0?)); + assert_eq!(summary.inter.skip_units, 2); + Ok(()) +} + +#[cfg(all(feature = "native-hevc-rows-private", feature = "test-support"))] +#[test] +fn row_overlap_evidence_excludes_complete_reference_jobs() -> Result<()> { + crate::native_rows::begin_evidence(); + let hook = Arc::new(TestHook { + overlap: true, + ..TestHook::default() + }); + let result = decode( + &stream(false)?, + true, + DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }, + Some(hook), + ); + assert_eq!(result.0?.frame_parallel_pictures, 2); + assert_eq!(crate::native_rows::take_row_overlap(), Some(0)); + Ok(()) +} + +#[cfg(hevc_thread_diagnostics)] +#[test] +fn diagnostics_attribute_real_reference_waits_without_changing_overlap() -> Result<()> { + for chain in [false, true] { + crate::thread_diagnostics::take(); + let hook = (!chain).then(|| { + Arc::new(TestHook { + overlap: true, + ..TestHook::default() + }) + }); + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + let result = decode(&stream(chain)?, true, limits, hook); + let summary = result.0?; + let counters = crate::thread_diagnostics::take(); + let count = |key| { + counters + .iter() + .filter(|(name, _)| *name == key) + .map(|(_, count)| *count) + .sum::() + }; + assert_eq!(count("event_picture_i"), 1); + assert_eq!(count("event_picture_p"), 2); + assert_eq!(count("event_parameter_set"), 3); + assert_eq!(count("event_queue_admitted"), 3); + assert_eq!( + count("barrier_reference_unavailable"), + if chain { 2 } else { 1 } + ); + assert_eq!(count("barrier_output_reorder"), 0); + assert_eq!(count("barrier_output_latency"), 0); + assert_eq!(summary.frame_parallel_pictures, if chain { 0 } else { 2 }); + } + Ok(()) +} + +#[test] +fn picture_budget_caps_entropy_reservations_without_losing_overlap_or_error_order() -> Result<()> { + let nals = stream(false)?; + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + let serial = decode(&nals, false, limits, None); + let picture_bins = serial + .1 + .iter() + .map(|picture| picture.stats.bins) + .max() + .unwrap(); + assert!(picture_bins > 0); + assert!( + nals[3..] + .iter() + .all(|nal| Cabac::max_bins(nal.len()) > picture_bins) + ); + let limits = DecodeLimits { + max_picture_bins: picture_bins, + ..limits + }; + let hook = Arc::new(TestHook { + overlap: true, + ..TestHook::default() + }); + let parallel = decode(&nals, true, limits, Some(Arc::clone(&hook))); + assert_eq!(parallel.1, serial.1); + let summary = parallel.0?; + assert_eq!(*hook.entered.lock().unwrap(), 6); + assert_eq!(summary.frame_parallel_pictures, 2); + assert_eq!(semantic(summary), semantic(serial.0?)); + for threads in [1, 2, 4, 8] { + for stream_bins in [picture_bins - 1, picture_bins, picture_bins + 1, 1 << 20] { + for max_picture_bins in [0, 1, picture_bins - 1, picture_bins, picture_bins + 1] { + let limits = DecodeLimits { + threads, + max_stream_bins: stream_bins, + max_picture_bins, + ..limits + }; + let serial = decode(&nals, false, limits, None); + let parallel = decode(&nals, true, limits, None); + assert_eq!(parallel.1, serial.1, "{limits:?}"); + assert_eq!( + parallel.0.map(semantic), + serial.0.map(semantic), + "{limits:?}" + ); + } + } + } + Ok(()) +} + +#[cfg(hevc_thread_diagnostics)] +#[test] +fn diagnostics_count_output_bumps_that_reconcile_at_picture_sealing() -> Result<()> { + let nals = stream_kind_reorder(false, 1, 0)?; + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + let serial = decode(&nals, false, limits, None); + crate::thread_diagnostics::take(); + let parallel = decode(&nals, true, limits, None); + assert_eq!(parallel.1, serial.1); + let summary = parallel.0?; + assert_eq!(summary.frame_parallel_pictures, 0); + assert_eq!(semantic(summary), semantic(serial.0?)); + let counters = crate::thread_diagnostics::take(); + assert_eq!( + counters + .iter() + .find(|(name, _)| *name == "barrier_output_reorder") + .unwrap() + .1, + 3 + ); + Ok(()) +} + +#[cfg(hevc_thread_diagnostics)] +#[test] +fn header_only_dependency_replay_distinguishes_chains_and_siblings() -> Result<()> { + for (chain, reorder) in [(false, 0), (false, 3), (true, 0), (true, 3)] { + let mut parser = crate::HeaderParser::new(crate::Limits::default()); + let mut references = Vec::new(); + for nal in stream_kind_reorder(chain, 1, reorder)? { + if matches!( + parser.read_nal(&nal, |_| Ok(()))?, + crate::NalInfo::Slice { .. } + ) { + let snapshot = parser.diagnostic_dependencies()?; + references.push( + snapshot + .active + .into_iter() + .flatten() + .map(|value| value.0) + .collect::>(), + ); + } + } + assert_eq!(references, [vec![], vec![0], vec![u64::from(chain)]]); + assert_eq!(parser.finish(|_| Ok(()))?.pictures, 3); + } + Ok(()) +} + +#[test] +fn strict_p_chain_waits_for_complete_generation_identified_references() -> Result<()> { + let nals = stream(true)?; + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + let serial = decode(&nals, false, limits, None); + let hook = Arc::new(TestHook::default()); + let parallel = decode(&nals, true, limits, Some(Arc::clone(&hook))); + assert_eq!(parallel.1, serial.1); + let summary = parallel.0?; + assert_eq!(*hook.entered.lock().unwrap(), 6); + assert_eq!( + ( + summary.frame_parallel_pictures, + summary.parallel_pictures, + summary.max_workers + ), + (0, 0, 1) + ); + assert_eq!(semantic(summary), semantic(serial.0?)); + Ok(()) +} + +#[cfg(hevc_thread_diagnostics)] +#[test] +fn header_envelope_uses_the_activated_sps_ordering_limits() -> Result<()> { + let mut parser = crate::HeaderParser::new(crate::Limits::default()); + assert_eq!( + parser.diagnostic_sps_envelope(), + Err(Error::InvalidData("dependency_picture")) + ); + let mut pictures = 0; + for nal in stream(false)? { + if let crate::NalInfo::Slice { + header, picture, .. + } = parser.read_nal(&nal, |_| Ok(()))? + { + if header.first { + assert_eq!(parser.diagnostic_sps_envelope()?, (4, 0)); + assert_eq!( + (picture.width, picture.height, picture.bit_depth), + (8, 8, 8) + ); + pictures += 1; + } + } + } + assert_eq!(pictures, 3); + Ok(()) +} + +#[cfg(hevc_thread_diagnostics)] +#[test] +fn row_dependency_trace_refuses_parallel_decode_and_observed_workers() -> Result<()> { + crate::thread_diagnostics::begin_rows()?; + let mut decoder = test_decoder(DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }); + assert_eq!( + decoder.read_nal(&[], None, |_| Ok(())), + Err(Error::Unsupported("row_trace_workers")) + ); + assert!(matches!( + crate::thread_diagnostics::take_rows(1), + Err(Error::Unsupported("row_trace_workers")) + )); + crate::thread_diagnostics::begin_rows()?; + assert!(matches!( + crate::thread_diagnostics::take_rows(2), + Err(Error::Unsupported("row_trace_workers")) + )); + crate::thread_diagnostics::begin_rows()?; + assert!(crate::thread_diagnostics::take_rows(1)?.is_empty()); + Ok(()) +} + +#[cfg(hevc_thread_diagnostics)] +#[test] +fn row_dependency_trace_observes_generated_predictions() -> Result<()> { + let nals = stream(false)?; + let baseline = decode(&nals, false, DecodeLimits::default(), None); + crate::thread_diagnostics::begin_rows()?; + let observed = decode(&nals, false, DecodeLimits::default(), None); + let rows = crate::thread_diagnostics::take_rows( + observed.0.as_ref().map_or(0, |summary| summary.max_workers), + )?; + assert_eq!(baseline.0.map(semantic), observed.0.map(semantic)); + assert_eq!(baseline.1, observed.1); + assert_eq!(rows.len(), 3); + for (index, picture) in rows.into_iter().enumerate() { + assert_eq!(picture.index, index as u64); + assert_eq!(picture.rows.len(), 1); + let row = picture.rows.first().ok_or(Error::Internal("row_test"))?; + if index == 0 { + assert_eq!(row.sample_calls, 0); + assert!(row.references.iter().all(Option::is_none)); + } else { + assert!(row.sample_calls >= 3); + let references: Vec<_> = row.references.iter().flatten().collect(); + assert_eq!(references.len(), 1); + assert_eq!( + ( + references[0].index, + references[0].poc, + references[0].sample_row + ), + (0, 0, Some(0)) + ); + } + } + Ok(()) +} + +#[cfg(hevc_thread_diagnostics)] +#[test] +fn header_output_fences_match_serial_calls_across_reorder_and_sequence_end() -> Result<()> { + std::thread::Builder::new() + .stack_size(16 << 20) + .spawn(header_output_fences_match_serial_calls_inner) + .expect("test thread starts") + .join() + .expect("test thread completes") +} + +#[cfg(hevc_thread_diagnostics)] +fn header_output_fences_match_serial_calls_inner() -> Result<()> { + for reorder in [0, 1, 3] { + for aud in [false, true] { + let base = stream_kind_reorder(false, 1, reorder)?; + let mut nals = base.clone(); + if aud { + nals.insert(5, vec![70, 1, 0x10]); + } + nals.push(vec![72, 1]); + nals.extend(base); + let mut parser = crate::HeaderParser::new(crate::Limits::default()); + parser.diagnostic_trace_outputs()?; + let mut decoder = PictureDecoder::new(DecodeLimits::default()); + let mut serial = Vec::new(); + let mut count = 0; + for nal in nals { + decoder.read_nal(&nal, None, |picture| { + serial.push((picture.index, picture.poc, count)); + Ok(()) + })?; + if matches!( + parser.read_nal(&nal, |_| Ok(()))?, + crate::NalInfo::Slice { header, .. } if header.first + ) { + count += 1; + } + } + decoder.finish(|picture| { + serial.push((picture.index, picture.poc, count)); + Ok(()) + })?; + parser.finish(|_| Ok(()))?; + let observed: Vec<_> = parser + .diagnostic_outputs()? + .into_iter() + .flatten() + .map(|output| { + ( + output.event.picture_index, + output.event.poc, + output.before_picture, + ) + }) + .collect(); + assert_eq!(observed, serial, "reorder={reorder}, aud={aud}"); + assert_eq!(observed.len(), 6); + } + } + Ok(()) +} + +#[test] +fn bump_after_failed_in_flight_job_keeps_serial_output_prefix() -> Result<()> { + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + let mut nals = stream(false)?; + let mut header = Writer::default(); + header.flag(true); + header.ue(0); + header.ue(1); + header.u(4, 3); + header.flag(false); + header.ue(1); + header.ue(0); + header.ue(2); + header.flag(true); + header.flag(false); + header.ue(4); + header.se(0); + let mut rbsp = header.rbsp(); + rbsp.extend([0xfe, 0xe0]); + nals.push(pcm::escaped(1, &rbsp)); + let serial = decode(&nals, false, limits, None); + let scheduled = decode(&nals, true, limits, None); + assert_eq!(scheduled.1, serial.1); + assert_eq!(semantic(scheduled.0?), semantic(serial.0?)); + + nals[4].pop(); + let serial = decode(&nals, false, limits, None); + let scheduled = decode(&nals, true, limits, None); + assert!(serial.0.is_err()); + assert!(serial.1.is_empty()); + assert_eq!(scheduled.1, serial.1); + assert_eq!(scheduled.0.map(semantic), serial.0.map(semantic)); + Ok(()) +} + +#[test] +fn independent_cold_memory_and_entropy_boundaries_keep_serial_admission() -> Result<()> { + let nals = stream(false)?; + let mut low = 0; + let mut high = 4 * 1024 * 1024; + while low + 1 < high { + let middle = low + (high - low) / 2; + let limits = DecodeLimits { + threads: 2, + max_memory_bytes: middle, + ..DecodeLimits::default() + }; + if decode(&nals, false, limits, None).0.is_ok() { + high = middle; + } else { + low = middle; + } + } + for memory in [ + low, + high, + high + 1, + high + STACK_BYTES, + high + 2 * STACK_BYTES, + 8 * 1024 * 1024, + ] { + let limits = DecodeLimits { + threads: 2, + max_memory_bytes: memory, + ..DecodeLimits::default() + }; + let serial = decode(&nals, false, limits, None); + let scheduled = decode(&nals, true, limits, None); + assert_eq!(scheduled.1, serial.1, "memory={memory}"); + assert_eq!( + scheduled.0.map(semantic), + serial.0.map(semantic), + "memory={memory}" + ); + } + for stream_bins in [0, 1, 2, 3, 4, 5, 6, 7, 8, 4095, 4096, 8192, 1 << 20] { + for picture_bins in [0, 1, 2, 3, 4, 1 << 20] { + let limits = DecodeLimits { + threads: 2, + max_stream_bins: stream_bins, + max_picture_bins: picture_bins, + ..DecodeLimits::default() + }; + let serial = decode(&nals, false, limits, None); + let scheduled = decode(&nals, true, limits, None); + assert_eq!( + scheduled.1, serial.1, + "stream={stream_bins} picture={picture_bins}" + ); + assert_eq!( + scheduled.0.map(semantic), + serial.0.map(semantic), + "stream={stream_bins} picture={picture_bins}" + ); + } + } + Ok(()) +} + +#[test] +fn earlier_slice_errors_precede_later_headers_without_early_boundary_finalization() -> Result<()> { + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + for damaged in [3, 4, 5] { + for bad_header in [false, true] { + let mut nals = stream(false)?; + nals[damaged].pop(); + if bad_header { + nals.push(vec![2, 1, 0x80]); + } + let serial = decode(&nals, false, limits, None); + let scheduled = decode(&nals, true, limits, None); + assert!(serial.0.is_err()); + assert_eq!( + scheduled, serial, + "damaged={damaged} bad_header={bad_header}" + ); + } + } + let mut nals = stream(false)?; + nals.push(vec![2, 1, 0x80]); + assert_eq!( + decode(&nals, true, limits, None), + decode(&nals, false, limits, None) + ); + let mut nals = stream(false)?; + nals[4].extend([0x12, 0x34]); + nals[5].pop(); + nals.push(vec![2, 1, 0x80]); + assert_eq!( + decode(&nals, true, limits, None), + decode(&nals, false, limits, None) + ); + Ok(()) +} + +#[test] +fn reverse_parallel_errors_and_panics_join_all_started_jobs() -> Result<()> { + let nals = stream(false)?; + for fault in [ + TestFault::ReverseErrors, + TestFault::Panic(1), + TestFault::Panic(2), + TestFault::Deadline, + ] { + let expected = if matches!(fault, TestFault::ReverseErrors) { + Error::InvalidData("earlier_picture_job") + } else if matches!(fault, TestFault::Deadline) { + Error::Limit("decode_deadline") + } else { + Error::Internal("reconstruction_panic") + }; + let hook = Arc::new(TestHook { + overlap: true, + fault: Some(fault), + ..TestHook::default() + }); + let (result, _) = decode( + &nals, + true, + DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }, + Some(Arc::clone(&hook)), + ); + assert_eq!(result, Err(expected)); + assert_eq!(hook.completed.load(Ordering::Acquire) & 6, 6); + assert_eq!(Arc::strong_count(&hook), 1); + } + Ok(()) +} + +fn suffix(planes: &[Vec; 3], bad: bool) -> Vec { + use md5::{Digest, Md5}; + let mut rbsp = vec![132, 49, 0]; + for plane in planes { + let mut hash = Md5::new(); + for sample in plane { + hash.update([*sample as u8]); + } + rbsp.extend(hash.finalize()); + } + if bad { + rbsp[3] ^= 1; + } + rbsp.push(0x80); + pcm::escaped(40, &rbsp) +} + +#[test] +fn suffix_hashes_and_sequence_transitions_keep_the_same_output_prefix() -> Result<()> { + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + for bad in [false, true] { + let mut nals = stream(false)?; + let serial = decode(&nals, false, limits, None); + let hash = suffix(&serial.1[0].planes, bad); + nals.insert(4, hash.clone()); + nals.insert(6, hash.clone()); + nals.push(hash); + let serial = decode(&nals, false, limits, None); + let scheduled = decode(&nals, true, limits, None); + assert_eq!(scheduled.1, serial.1); + assert_eq!(scheduled.0.map(semantic), serial.0.map(semantic)); + } + for end in [36, 37] { + let mut nals = stream(false)?; + nals.push(vec![end << 1, 1]); + if end == 36 { + nals.extend(stream(false)?); + } + let serial = decode(&nals, false, limits, None); + let scheduled = decode(&nals, true, limits, None); + assert_eq!(scheduled.1, serial.1); + assert_eq!(scheduled.0.map(semantic), serial.0.map(semantic)); + } + Ok(()) +} + +#[test] +fn per_picture_sei_keeps_real_overlap_and_ordered_hash_failures() -> Result<()> { + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + let base = stream(false)?; + let reference = decode(&base, false, limits, None); + let hash = suffix(&reference.1[0].planes, false); + let mut nals = base[..3].to_vec(); + for nal in &base[3..] { + nals.push(pcm::escaped(39, &[1, 0, 0x80])); + nals.push(nal.clone()); + nals.push(pcm::escaped(40, &[5, 0, 0x80])); + nals.push(hash.clone()); + } + let serial = decode(&nals, false, limits, None); + let hook = Arc::new(TestHook { + overlap: true, + ..TestHook::default() + }); + let scheduled = decode(&nals, true, limits, Some(Arc::clone(&hook))); + assert_eq!(scheduled.1, serial.1); + let summary = scheduled.0?; + assert_eq!(summary.frame_parallel_pictures, 2); + assert_eq!(*hook.entered.lock().unwrap(), 6); + assert_eq!(semantic(summary), semantic(serial.0?)); + for slot in [6, 10, 14] { + let mut invalid = nals.clone(); + invalid[slot] = suffix(&reference.1[0].planes, true); + let serial = decode(&invalid, false, limits, None); + let scheduled = decode(&invalid, true, limits, None); + assert!(serial.0.is_err()); + assert_eq!(scheduled.1, serial.1); + assert_eq!(scheduled.0.map(semantic), serial.0.map(semantic)); + } + for extra in [ + hash, + pcm::escaped(40, &[132, 1, 3, 0x80]), + pcm::escaped(40, &[5, 3, 0x80]), + ] { + let mut invalid = nals.clone(); + invalid.insert(11, extra); + let serial = decode(&invalid, false, limits, None); + let scheduled = decode(&invalid, true, limits, None); + assert!(serial.0.is_err()); + assert_eq!(scheduled.1, serial.1); + assert_eq!(scheduled.0.map(semantic), serial.0.map(semantic)); + } + let mut invalid = nals; + invalid[8].pop(); + invalid.push(pcm::escaped(40, &[5, 3, 0x80])); + let serial = decode(&invalid, false, limits, None); + let scheduled = decode(&invalid, true, limits, None); + assert!(serial.0.is_err()); + assert_eq!(scheduled.1, serial.1); + assert_eq!(scheduled.0.map(semantic), serial.0.map(semantic)); + Ok(()) +} + +fn two_row_stream(wpp: bool, complete: bool) -> Vec> { + let mut vps = Writer::default(); + vps.u(4, 0); + vps.flag(true); + vps.flag(true); + vps.u(6, 0); + vps.u(3, 0); + vps.flag(true); + vps.u(16, 0xffff); + profile(&mut vps); + vps.flag(false); + for value in [3, 3, 0] { + vps.ue(value); + } + vps.u(6, 0); + vps.ue(0); + vps.flag(false); + vps.flag(false); + let mut sps = Writer::default(); + sps.u(4, 0); + sps.u(3, 0); + sps.flag(true); + profile(&mut sps); + for value in [0, 1, 16, 32] { + sps.ue(value); + } + sps.flag(false); + sps.ue(0); + sps.ue(0); + sps.ue(0); + sps.flag(false); + for value in [3, 3, 0, 1, 0, 0, 1, 0, 0] { + sps.ue(value); + } + for flag in [false, false, false, true] { + sps.flag(flag); + } + sps.u(4, 7); + sps.u(4, 7); + sps.ue(1); + sps.ue(0); + sps.flag(true); + sps.ue(0); + for _ in 0..5 { + sps.flag(false); + } + let mut pps = Writer::default(); + pps.ue(0); + pps.ue(0); + pps.flag(false); + pps.flag(false); + pps.u(3, 0); + pps.flag(false); + pps.flag(false); + pps.ue(0); + pps.ue(0); + pps.se(0); + for _ in 0..3 { + pps.flag(false); + } + pps.se(0); + pps.se(0); + for flag in [false, false, false, false, false, wpp, false] { + pps.flag(flag); + } + pps.flag(true); + pps.flag(false); + pps.flag(true); + pps.flag(false); + pps.flag(false); + pps.ue(0); + pps.flag(false); + pps.flag(false); + let mut header = Writer::default(); + header.flag(true); + header.flag(false); + header.ue(0); + header.ue(2); + header.se(0); + if wpp { + header.ue(u32::from(complete)); + if complete { + header.ue(8); + header.u(9, 387); + } + } + let mut rbsp = header.rbsp(); + rbsp.extend([0x86, 0x80]); + rbsp.extend(core::iter::repeat_n(34, 384)); + rbsp.extend([if complete { 0xfd } else { 0xfe }, 0x80]); + if complete { + rbsp.extend([0x86, 0x80]); + rbsp.extend(core::iter::repeat_n(51, 384)); + rbsp.extend([0xfe, 0x80]); + } + vec![ + pcm::escaped(32, &vps.rbsp()), + pcm::escaped(33, &sps.rbsp()), + pcm::escaped(34, &pps.rbsp()), + pcm::escaped(19, &rbsp), + ] +} + +#[test] +fn session_synchronization_and_polling_preserve_warm_wavefront_workers() -> Result<()> { + let nals = two_row_stream(true, true); + for scheduled in [false, true] { + let mut decoder = test_decoder(DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }); + for nal in &nals { + if scheduled { + decoder.read_nal_scheduled(nal, None, None, |_, _| Ok(()))?; + } else { + decoder.read_nal(nal, None, |_| Ok(()))?; + } + decoder.set_external_memory(0)?; + } + let before = decoder.session[0].pictures.frame.cache.hits(); + assert!( + decoder.session[0] + .pictures + .frame + .cache + .retained_bytes(0, usize::MAX)? + > 0 + ); + if scheduled { + decoder.read_nal_scheduled(&nals[3], None, None, |_, _| Ok(()))?; + } else { + decoder.read_nal(&nals[3], None, |_| Ok(()))?; + } + assert_eq!(decoder.session[0].pictures.frame.cache.hits(), before + 1); + let summary = decoder.finish(|_| Ok(()))?; + assert_eq!( + ( + summary.parallel_pictures, + summary.frame_parallel_pictures, + summary.max_workers + ), + (2, 0, 2) + ); + } + Ok(()) +} + +#[test] +fn changing_worker_count_joins_without_committing_an_ordered_pending_error() -> Result<()> { + let mut nals = stream(false)?; + nals[4].extend([0x12, 0x34]); + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + let expected = decode(&nals, false, limits, None).0.unwrap_err(); + let mut decoder = test_decoder(limits); + for nal in &nals { + decoder.read_nal_scheduled(nal, None, None, |_, _| Ok(()))?; + } + decoder = decoder.with_threads(1); + assert!( + decoder.session[0] + .pictures + .frame + .scheduler + .jobs + .iter() + .all(|job| job.handle.is_none()) + ); + assert_eq!(decoder.finish(|_| Ok(())), Err(expected)); + assert!(decoder.session.is_empty()); + Ok(()) +} + +#[test] +fn external_overflow_and_caller_validation_preserve_the_first_pending_slice_error() -> Result<()> { + let mut nals = stream(false)?; + nals.truncate(4); + nals[3].pop(); + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + let expected = decode(&nals, false, limits, None).0.unwrap_err(); + for external in [false, true] { + let mut decoder = test_decoder(limits); + for nal in &nals { + decoder.read_nal_scheduled(nal, None, None, |_, _| Ok(()))?; + } + let result = if external { + decoder.set_external_memory(usize::MAX) + } else { + decoder.synchronize_pending() + }; + assert_eq!(result, Err(expected)); + assert!(decoder.session.is_empty()); + } + Ok(()) +} + +#[test] +fn cold_layout_receipt_uses_the_original_typed_allowance() -> Result<()> { + let decoder = PictureDecoder::new(DecodeLimits::default()); + println!( + "fixed={} summary={} serial_summary={} dpb={} core={} serial_session={} frame={} slots={}", + decoder.fixed_memory()?, + core::mem::size_of::(), + core::mem::size_of::(), + core::mem::size_of::(), + core::mem::size_of::(), + core::mem::size_of::, SerialAccounting>>(), + core::mem::size_of::(), + core::mem::size_of::() + ); + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + assert_eq!(decoder.fixed_memory()?, 634320); + Ok(()) +} + +#[test] +fn next_header_validation_precedes_prior_incomplete_picture_finalization() { + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + for bad_header in [false, true] { + let mut nals = two_row_stream(false, false); + nals.push(if bad_header { + vec![2, 1, 0x80] + } else { + nals[3].clone() + }); + let serial = decode(&nals, false, limits, None); + let scheduled = decode(&nals, true, limits, None); + assert!(serial.0.is_err()); + if bad_header { + assert_ne!( + serial.0, + Err(Error::InvalidData("selected_picture_incomplete")) + ); + } else { + assert_eq!( + serial.0, + Err(Error::InvalidData("selected_picture_incomplete")) + ); + } + assert_eq!(scheduled, serial); + } +} + +#[test] +fn biprediction_can_read_one_pinned_publication_twice_without_a_reference_lock() -> Result<()> { + let nals = stream_kind(false, 0)?; + let limits = DecodeLimits { + threads: 2, + ..DecodeLimits::default() + }; + let serial = decode(&nals, false, limits, None); + let hook = Arc::new(TestHook { + overlap: true, + ..TestHook::default() + }); + let scheduled = decode(&nals, true, limits, Some(hook)); + assert_eq!(scheduled.1, serial.1); + let summary = scheduled.0?; + assert_eq!( + (summary.inter.bi_units, summary.frame_parallel_pictures), + (2, 2) + ); + assert_eq!(semantic(summary), semantic(serial.0?)); + Ok(()) +} + +#[test] +fn optional_input_snapshot_record_and_start_refusals_keep_serial_results() -> Result<()> { + let nals = stream(false)?; + let limits = DecodeLimits { + threads: 3, + ..DecodeLimits::default() + }; + let serial = decode(&nals, false, limits, None); + for fault in [ + AdmissionFault::Input, + AdmissionFault::Snapshot, + AdmissionFault::Record, + AdmissionFault::Spawn, + ] { + let scheduled = decode_configuration(&nals, true, limits, None, Some(fault)); + assert_eq!(scheduled.1, serial.1); + assert_eq!(scheduled.0.map(semantic), serial.0.map(semantic)); + } + let hook = Arc::new(TestHook { + overlap: true, + ..TestHook::default() + }); + let partial = decode_configuration( + &nals, + true, + limits, + Some(Arc::clone(&hook)), + Some(AdmissionFault::SpawnAfterOne), + ); + assert_eq!(partial.1, serial.1); + let summary = partial.0?; + assert_eq!( + (summary.frame_parallel_pictures, summary.max_workers), + (2, 2) + ); + assert_eq!(semantic(summary), semantic(serial.0?)); + assert_eq!(Arc::strong_count(&hook), 1); + Ok(()) +} + +fn reserved_output( + nals: &[Vec], + scheduled: bool, + bytes: usize, + hook: Option>, +) -> (Result, Vec) { + let mut decoder = test_decoder(DecodeLimits { + threads: 2, + max_memory_bytes: 8 * 1024 * 1024, + ..DecodeLimits::default() + }); + let mut output = Vec::new(); + let caller = std::thread::current().id(); + let result = (|| { + decoder.initialize()?; + decoder.session[0].pictures.frame.scheduler.hook = hook; + for nal in nals { + if scheduled { + decoder.read_nal_scheduled(nal, None, None, |_, _| { + Err(Error::Internal("early_output")) + })?; + } else { + decoder.read_nal(nal, None, |_| Err(Error::Internal("early_output")))?; + } + decoder.set_external_memory(0)?; + } + decoder.finish_with_output(|picture, memory| { + assert_eq!(std::thread::current().id(), caller); + memory.reserve(bytes)?; + memory.release(bytes)?; + capture(picture, &mut output); + Ok(()) + }) + })(); + if result.is_err() { + assert!(decoder.session.is_empty()); + } else { + assert_eq!( + decoder.session[0] + .pictures + .frame + .scheduler + .bytes(Some(decoder.session[0].core.dpb())) + .unwrap(), + 0 + ); + assert!(!decoder.session[0].pictures.frame.scheduler.has_input()); + } + (result, output) +} + +#[test] +fn output_reservations_retire_pending_jobs_and_pins_before_cold_admission() -> Result<()> { + let nals = stream(false)?; + let mut low = 0; + let mut high = 8 * 1024 * 1024; + while low + 1 < high { + let middle = low + (high - low) / 2; + if reserved_output(&nals, false, middle, None).0.is_ok() { + low = middle; + } else { + high = middle; + } + } + for bytes in [0, low, high] { + let serial = reserved_output(&nals, false, bytes, None); + let hook = Arc::new(TestHook { + overlap: true, + ..TestHook::default() + }); + let scheduled = reserved_output(&nals, true, bytes, Some(Arc::clone(&hook))); + assert_eq!(scheduled.1, serial.1); + assert_eq!(scheduled.0.map(semantic), serial.0.map(semantic)); + assert_eq!(hook.completed.load(Ordering::Acquire) & 6, 6); + assert_eq!(Arc::strong_count(&hook), 1); + } + Ok(()) +} diff --git a/crates/coreav-hevc/src/dpb.rs b/crates/coreav-hevc/src/dpb.rs index bb2f562c6..657810821 100644 --- a/crates/coreav-hevc/src/dpb.rs +++ b/crates/coreav-hevc/src/dpb.rs @@ -1,11 +1,121 @@ use crate::{ Error, Limits, NalHeader, Result, SliceHeader, params::Sps, reconstruction::RetainedPicture, }; +use std::sync::{Arc, OnceLock}; + +pub(crate) trait ReferenceProvider: Sync { + fn reference(&self, identity: ReferenceIdentity) -> Result<&RetainedPicture>; + fn reference_info(&self, identity: ReferenceIdentity) -> Result { + self.reference(identity).map(RetainedPicture::info) + } + fn read_reference( + &self, + identity: ReferenceIdentity, + support: crate::reconstruction::ReadSupport, + ) -> Result> { + self.reference(identity) + .map(|picture| crate::reconstruction::ReferenceRead::new(picture, support)) + } + fn dpb(&self) -> Option<&Dpb> { + None + } +} + +pub(crate) struct Publication { + pub(crate) samples: OnceLock, + pub(crate) rows: Option>, +} + +impl Publication { + pub(crate) const fn record_bytes() -> usize { + core::mem::size_of::() + 2 * core::mem::size_of::() + } + pub(crate) fn picture(&self) -> Option<&RetainedPicture> { + self.rows + .as_ref() + .map_or_else(|| self.samples.get(), |rows| rows.picture()) + } + + fn take_picture(&mut self) -> Option { + match &mut self.rows { + Some(rows) => Arc::get_mut(rows)?.take_picture(), + None => self.samples.take(), + } + } + + pub(crate) fn reference_info(&self, identity: ReferenceIdentity) -> Result { + match &self.rows { + Some(rows) => rows.info(identity, rows.generation()), + None => self + .samples + .get() + .map(RetainedPicture::info) + .ok_or(Error::InvalidData("unavailable reference samples")), + } + } + + pub(crate) fn read_reference( + &self, + identity: ReferenceIdentity, + support: crate::reconstruction::ReadSupport, + ) -> Result> { + match &self.rows { + Some(rows) => rows.read(identity, rows.generation(), support), + None => self + .samples + .get() + .map(|picture| crate::reconstruction::ReferenceRead::new(picture, support)) + .ok_or(Error::InvalidData("unavailable reference samples")), + } + } + + pub(crate) fn bytes(&self) -> Result { + let samples = match &self.rows { + Some(rows) => rows.bytes()?, + None => self + .samples + .get() + .map_or(Ok(0), RetainedPicture::heap_bytes)?, + }; + Self::record_bytes() + .checked_add(samples) + .ok_or(Error::Limit("reconstruction_memory")) + } +} + +#[expect( + clippy::large_enum_variant, + reason = "Inline legacy storage preserves the cold decoder allocation envelope" +)] +enum Samples { + None, + Owned(RetainedPicture), + Shared(Arc), +} + +impl Samples { + fn as_ref(&self) -> Option<&RetainedPicture> { + match self { + Self::None => None, + Self::Owned(samples) => Some(samples), + Self::Shared(publication) => publication.picture(), + } + } + + fn is_none(&self) -> bool { + matches!(self, Self::None) + } + + fn is_some(&self) -> bool { + !self.is_none() + } +} pub(crate) struct Emission<'a> { pub(crate) event: OutputEvent, pub(crate) picture: Option<&'a RetainedPicture>, - pub(crate) retained_bytes: usize, + pub(crate) publication: Option<&'a Arc>, + pub(crate) dpb: &'a Dpb, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -66,7 +176,16 @@ struct Picture { temporal_id: u8, unit_type: u8, generated: bool, - samples: Option, + samples: Samples, +} + +impl Picture { + fn matches_reference(&self, identity: ReferenceIdentity) -> bool { + !self.generated + && self.reference != Reference::Unused + && self.event.picture_index == identity.index + && self.event.poc == identity.poc + } } struct Pending { @@ -195,22 +314,18 @@ impl Dpb { self.pictures .iter() .flatten() - .find(|p| { - !p.generated - && p.reference != Reference::Unused - && p.event.picture_index == identity.index - && p.event.poc == identity.poc - }) + .find(|picture| picture.matches_reference(identity)) .and_then(|p| p.samples.as_ref()) .ok_or(Error::InvalidData("unavailable reference samples")) } pub(crate) fn heap_bytes(&self) -> Result { self.pictures.iter().flatten().try_fold(0usize, |total, p| { - let bytes = p - .samples - .as_ref() - .map_or(Ok(0), RetainedPicture::heap_bytes)?; + let bytes = match &p.samples { + Samples::None => 0, + Samples::Owned(samples) => samples.heap_bytes()?, + Samples::Shared(publication) => publication.bytes()?, + }; total .checked_add(bytes) .ok_or(Error::Limit("reconstruction_memory")) @@ -250,7 +365,160 @@ impl Dpb { { return Err(Error::InvalidData("DPB picture attachment")); } - picture.samples = Some(samples); + picture.samples = Samples::Owned(samples); + Ok(()) + } + + pub(crate) fn publication(&self, index: u64) -> Option<&Arc> { + self.pictures.iter().flatten().find_map(|picture| { + if picture.event.picture_index != index { + return None; + } + match &picture.samples { + Samples::Shared(publication) => Some(publication), + _ => None, + } + }) + } + + pub(crate) fn sharing_bytes(&self) -> usize { + self.current_references + .entries + .iter() + .flatten() + .filter(|identity| self.publication(identity.index).is_none()) + .count() + * (core::mem::size_of::() + 2 * core::mem::size_of::()) + } + + pub(crate) fn unshare(&mut self) { + for picture in self.pictures.iter_mut().flatten() { + let owned = match &mut picture.samples { + Samples::Shared(publication) => { + Arc::get_mut(publication).and_then(Publication::take_picture) + } + _ => None, + }; + if let Some(owned) = owned { + picture.samples = Samples::Owned(owned); + } + } + } + + pub(crate) fn resume_outputs(&self) -> bool { + self.resume_output_constraints() + .into_iter() + .any(|cause| cause) + } + + pub(crate) fn resume_output_constraints(&self) -> [bool; 2] { + self.pending.as_ref().map_or([false; 2], |pending| { + self.bump_constraints(pending.reorder, pending.latency) + }) + } + + pub(crate) fn picture_outputs(&self, nal: NalHeader, sps: &Sps) -> bool { + self.picture_output_constraints(nal, sps) + .into_iter() + .any(|cause| cause) + } + + pub(crate) fn picture_output_constraints(&self, nal: NalHeader, sps: &Sps) -> [bool; 4] { + let tid = self.highest_tid.unwrap_or(sps.sub_layers) as usize; + [ + (16..=21).contains(&nal.unit_type), + self.fullness() >= sps.max_dpb.get(tid).copied().unwrap_or(0) as usize, + self.pictures + .iter() + .flatten() + .filter(|picture| picture.output) + .count() + + 1 + > sps.max_reorder.get(tid).copied().unwrap_or(0) as usize, + sps.max_latency + .get(tid) + .copied() + .is_some_and(|latency| latency != 0), + ] + } + + #[cfg(feature = "native-hevc-rows-private")] + pub(crate) fn share_rows( + &mut self, + identity: ReferenceIdentity, + consumer: u64, + ) -> Result> { + if identity.index >= consumer { + return Err(Error::Internal("reference_row_order")); + } + if let Some(picture) = self + .pictures + .iter() + .flatten() + .find(|picture| picture.matches_reference(identity)) + { + if let Samples::Shared(publication) = &picture.samples { + if publication.rows.is_some() { + publication.reference_info(identity)?; + return Ok(Arc::clone(publication)); + } + } + } + self.share(identity) + } + + pub(crate) fn share(&mut self, identity: ReferenceIdentity) -> Result> { + self.reference(identity)?; + let picture = self + .pictures + .iter_mut() + .flatten() + .find(|picture| picture.matches_reference(identity)) + .ok_or(Error::InvalidData("unavailable reference samples"))?; + if let Samples::Owned(_) = &picture.samples { + let Samples::Owned(samples) = core::mem::replace(&mut picture.samples, Samples::None) + else { + return Err(Error::Internal("reference_publication")); + }; + picture.samples = Samples::Shared(Arc::new(Publication { + samples: OnceLock::from(samples), + rows: None, + })); + } + match &picture.samples { + Samples::Shared(publication) => Ok(Arc::clone(publication)), + _ => Err(Error::Internal("reference_publication")), + } + } + + pub(crate) fn seal( + &mut self, + index: u64, + info: crate::PictureInfo, + publication: Arc, + ) -> Result<()> { + let pending = self + .pending + .as_ref() + .ok_or(Error::InvalidData("no pending DPB picture"))?; + let picture = self + .pictures + .iter_mut() + .flatten() + .find(|picture| picture.event.picture_index == index && !picture.generated) + .ok_or(Error::InvalidData("pending DPB picture"))?; + if pending.index != index + || pending.coded_size != [info.width, info.height] + || pending.conformance_window != info.conformance_window + || picture.event.width != info.visible_width + || picture.event.height != info.visible_height + || picture.event.bit_depth != info.bit_depth + || picture.samples.is_some() + || publication.picture().is_some() + { + return Err(Error::InvalidData("DPB picture attachment")); + } + picture.samples = Samples::Shared(publication); Ok(()) } @@ -355,6 +623,8 @@ impl Dpb { if self.state != State::Reading { return Err(Error::InvalidData("DPB session ended")); } + #[cfg(hevc_thread_diagnostics)] + crate::thread_diagnostics::output_finish(); let result = self.drain(&mut emit); if result.is_ok() { self.state = State::Finished; @@ -539,7 +809,7 @@ impl Dpb { temporal_id: nal.temporal_id, unit_type: nal.unit_type, generated: true, - samples: None, + samples: Samples::None, })?; } } @@ -579,7 +849,7 @@ impl Dpb { temporal_id: nal.temporal_id, unit_type: nal.unit_type, generated: false, - samples: None, + samples: Samples::None, })?; if self.reconstruction { self.pending = Some(Pending { @@ -590,7 +860,11 @@ impl Dpb { latency, }); } + #[cfg(hevc_thread_diagnostics)] + crate::thread_diagnostics::output_picture(Some(index)); self.additional_bumps(reorder, latency, emit)?; + #[cfg(hevc_thread_diagnostics)] + crate::thread_diagnostics::output_picture(None); if irap { self.irap = Some(Irap { poc, @@ -932,13 +1206,21 @@ impl Dpb { } fn must_bump(&self, reorder: usize, latency: Option) -> bool { - self.pictures.iter().flatten().filter(|p| p.output).count() > reorder - || latency.is_some_and(|limit| { + self.bump_constraints(reorder, latency) + .into_iter() + .any(|cause| cause) + } + + fn bump_constraints(&self, reorder: usize, latency: Option) -> [bool; 2] { + [ + self.pictures.iter().flatten().filter(|p| p.output).count() > reorder, + latency.is_some_and(|limit| { self.pictures .iter() .flatten() .any(|p| p.output && p.latency >= limit) - }) + }), + ] } fn remove_unused(&mut self) { @@ -997,11 +1279,10 @@ impl Dpb { fn bump(&mut self, emit: &mut impl FnMut(Emission<'_>) -> Result<()>) -> Result<()> { let index = self.next_output()?; - let retained_bytes = self.heap_bytes()?; let picture = self .pictures - .get_mut(index) - .and_then(Option::as_mut) + .get(index) + .and_then(Option::as_ref) .ok_or(Error::InvalidData("DPB output picture"))?; if self .pending @@ -1017,10 +1298,26 @@ impl Dpb { emit(Emission { event: picture.event, picture: picture.samples.as_ref(), - retained_bytes, + dpb: self, + publication: match &picture.samples { + Samples::Shared(publication) => Some(publication), + _ => None, + }, })?; + #[cfg(hevc_thread_diagnostics)] + crate::thread_diagnostics::output_event( + picture.event, + self.count, + self.pictures.iter().flatten().any(|candidate| { + !candidate.generated && candidate.event.picture_index + 1 == self.count + }), + )?; self.last_output = Some(picture.event.poc); - picture.output = false; + self.pictures + .get_mut(index) + .and_then(Option::as_mut) + .ok_or(Error::Internal("DPB output picture"))? + .output = false; self.remove_unused(); Ok(()) } @@ -1055,11 +1352,104 @@ impl Dpb { } } +impl ReferenceProvider for Dpb { + fn reference(&self, identity: ReferenceIdentity) -> Result<&RetainedPicture> { + self.reference(identity) + } + + fn dpb(&self) -> Option<&Dpb> { + Some(self) + } +} + #[cfg(test)] mod tests { use super::*; use crate::{LongTermRef, rps::Rps, vui::Vui}; + #[test] + fn scheduled_publications_preserve_the_original_inline_dpb_storage_layout() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::>() + ); + assert_eq!( + core::mem::align_of::(), + core::mem::align_of::>() + ); + } + + #[test] + fn sharing_real_cra_samples_skips_generated_same_index_references() -> Result<()> { + let s = sps(); + let mut dpb = Dpb::new_reconstruction(); + push(&mut dpb, 21, &header(8, &[(-3, false)]), &s)?; + complete(&mut dpb, 0, &s)?; + push(&mut dpb, 7, &header(7, &[(-2, false), (1, true)]), &s)?; + let identity = dpb.current_identity(0)?; + assert_eq!(identity.poc, 8); + let pointer = dpb.reference(identity)?.view().planes[0].samples.as_ptr(); + for (index, poc) in [(0, 5), (1, 8)] { + assert!( + dpb.share(ReferenceIdentity { + index, + poc, + long_term: false, + }) + .is_err() + ); + } + let publication = dpb.share(identity)?; + let samples = publication + .samples + .get() + .ok_or(Error::Internal("test_publication"))?; + assert_eq!(samples.index(), 0); + assert_eq!(samples.view().planes[0].samples.as_ptr(), pointer); + assert!(Arc::ptr_eq(&publication, &dpb.share(identity)?)); + assert!(dpb.pictures.iter().flatten().any(|picture| { + picture.generated + && picture.event.picture_index == 0 + && picture.event.poc == 5 + && picture.samples.is_none() + })); + dpb.fail(); + assert_eq!(samples.view().planes[0].samples.as_ptr(), pointer); + Ok(()) + } + + #[test] + fn immutable_publications_pin_samples_after_normative_removal_and_poc_reuse() -> Result<()> { + let mut s = sps(); + s.max_reorder = [0; 7]; + let mut dpb = Dpb::new_reconstruction(); + push(&mut dpb, 19, &header(0, &[]), &s)?; + complete(&mut dpb, 0, &s)?; + let identity = dpb.current_identity(0)?; + let retained = dpb.share(identity)?; + let weak = Arc::downgrade(&retained); + let original = retained + .samples + .get() + .ok_or(Error::Internal("test_publication"))?; + let pointer = original.view().planes[0].samples.as_ptr(); + let mut next = header(0, &[]); + next.no_output_of_prior_pics = true; + push(&mut dpb, 19, &next, &s)?; + assert!(dpb.reference(identity).is_err()); + complete(&mut dpb, 1, &s)?; + assert_eq!(original.index(), 0); + assert_eq!(original.view().planes[0].samples.as_ptr(), pointer); + let current = dpb.reference(dpb.current_identity(1)?)?; + assert_eq!(current.index(), 1); + assert_ne!(current.view().planes[0].samples.as_ptr(), pointer); + dpb.fail(); + assert!(weak.upgrade().is_some()); + drop(retained); + assert!(weak.upgrade().is_none()); + Ok(()) + } + fn sps() -> Sps { Sps { id: 0, @@ -1340,6 +1730,92 @@ mod tests { Ok(()) } + #[cfg(hevc_thread_diagnostics)] + #[test] + fn header_output_trace_matches_latency_bursts_and_eof_after_a_skipped_picture() -> Result<()> { + std::thread::Builder::new() + .stack_size(16 << 20) + .spawn(|| { + use crate::thread_diagnostics::{DependencyOutputs, capture_outputs}; + for latency in [0, 1] { + for skipped in [false, true] { + let mut s = sps(); + s.sub_layers = 1; + s.max_latency = [latency; 7]; + let mut headers = Dpb::new(); + let mut decoded = Dpb::new_reconstruction(); + headers.highest_tid = Some(0); + decoded.highest_tid = Some(0); + let mut trace = DependencyOutputs::default(); + let mut serial = Vec::new(); + for (index, (kind, poc)) in + [(19, 0), (1, 5), (1, 3), (1, 1)].into_iter().enumerate() + { + if index != 0 { + serial.extend( + complete(&mut decoded, index as u64 - 1, &s)? + .into_iter() + .map(|event| { + (event.picture_index, event.poc, index as u64) + }), + ); + } + let h = header(poc, &[]); + serial.extend( + push(&mut decoded, kind, &h, &s)? + .into_iter() + .map(|event| (event.picture_index, event.poc, index as u64)), + ); + capture_outputs(&mut trace, || push(&mut headers, kind, &h, &s))?; + } + serial.extend( + complete(&mut decoded, 3, &s)? + .into_iter() + .map(|event| (event.picture_index, event.poc, 4)), + ); + if skipped { + let mut n = nal(1); + n.temporal_id = 1; + let h = header(2, &[]); + let read = |dpb: &mut Dpb| { + dpb.picture(n, &h, &s, &Limits::default(), |_| Ok(())) + }; + assert!(!read(&mut decoded)?.reconstruction_required); + assert!( + !capture_outputs(&mut trace, || read(&mut headers))? + .reconstruction_required + ); + } + let count = 4 + u64::from(skipped); + serial.extend( + finish(&mut decoded)? + .into_iter() + .map(|event| (event.picture_index, event.poc, count)), + ); + capture_outputs(&mut trace, || finish(&mut headers))?; + let observed: Vec<_> = trace + .events + .into_iter() + .flatten() + .map(|output| { + ( + output.event.picture_index, + output.event.poc, + output.before_picture, + ) + }) + .collect(); + assert_eq!(observed, serial, "latency={latency}, skipped={skipped}"); + assert_eq!(observed.len(), 4); + } + } + Ok(()) + }) + .expect("test thread starts") + .join() + .expect("test thread completes") + } + #[test] fn attachments_validate_identity_geometry_uniqueness_and_completion() -> Result<()> { let s = sps(); diff --git a/crates/coreav-hevc/src/lib.rs b/crates/coreav-hevc/src/lib.rs index a27ea03d5..ab70c182f 100644 --- a/crates/coreav-hevc/src/lib.rs +++ b/crates/coreav-hevc/src/lib.rs @@ -10,6 +10,25 @@ #![forbid(unsafe_code)] #![cfg(feature = "native-hevc")] +macro_rules! thread_diagnostic { + (picture $index:expr) => {{ + #[cfg(hevc_thread_diagnostics)] + $crate::thread_diagnostics::picture($index); + }}; + (barrier $reason:ident) => {{ + #[cfg(hevc_thread_diagnostics)] + $crate::thread_diagnostics::barrier($crate::thread_diagnostics::Barrier::$reason); + }}; + (event $reason:ident) => {{ + #[cfg(hevc_thread_diagnostics)] + $crate::thread_diagnostics::event($crate::thread_diagnostics::Event::$reason); + }}; +} +pub(crate) use thread_diagnostic; + +#[cfg(hevc_thread_diagnostics)] +pub mod thread_diagnostics; + mod bits; mod cabac; mod ctu; @@ -17,6 +36,9 @@ mod decoder; mod dpb; mod intra; mod nal; +#[cfg(feature = "native-hevc-rows-private")] +#[doc(hidden)] +pub mod native_rows; mod params; mod parser; mod picture_hash; diff --git a/crates/coreav-hevc/src/native_rows.rs b/crates/coreav-hevc/src/native_rows.rs new file mode 100644 index 000000000..3e54be317 --- /dev/null +++ b/crates/coreav-hevc/src/native_rows.rs @@ -0,0 +1,123 @@ +use crate::{DecodedPicture, OutputMemory, PictureInfo, Result}; + +#[cfg(feature = "test-support")] +thread_local! { + static ROW_OVERLAP: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +#[cfg(feature = "test-support")] +pub fn begin_evidence() { + ROW_OVERLAP.set(Some(0)); +} + +#[cfg(feature = "test-support")] +pub fn take_row_overlap() -> Option { + ROW_OVERLAP.replace(None) +} + +#[cfg(feature = "test-support")] +pub(crate) fn record_row_overlap() { + if let Some(count) = ROW_OVERLAP.get() { + ROW_OVERLAP.set(count.checked_add(1)); + } +} + +#[cfg(feature = "test-support")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Admission { + pub sps_id: u32, + pub sps_hash: [u8; 32], + pub ceiling: usize, + pub workers: usize, + pub lag: usize, + pub frame_bytes: usize, + pub reserved_bytes: usize, + pub limit_bytes: usize, +} + +#[derive(Clone, Copy)] +pub struct OutputHeader { + pub index: u64, + pub poc: i32, + pub timestamp: Option, + pub origin: Option, + pub info: PictureInfo, +} + +#[derive(Clone, Copy)] +pub struct SinkCost { + pub max_packet_bytes: usize, + pub max_nals: usize, + pub max_pending_bytes: u64, + pub queue_capacity: usize, + pub queued_frames: usize, + pub record_bytes: usize, + pub plane_records_bytes: usize, + pub owner_bytes: usize, + pub debt_peak_bytes: usize, +} + +pub struct RowReceive { + pub consume: Option, + pub pop: bool, +} + +pub trait RowSink { + fn prepare( + &mut self, + header: OutputHeader, + memory: &mut OutputMemory<'_>, + bounded: bool, + ) -> Result; + fn reserve_queue( + &mut self, + records: usize, + memory: &mut OutputMemory<'_>, + admit: &mut dyn FnMut(usize) -> Result, + ) -> Result; + fn qualify(&self, info: PictureInfo, probe: &mut CapacityProbe) -> bool; + fn fill(&mut self, front_offset: usize, picture: DecodedPicture<'_>) -> Result<()>; + fn truncate(&mut self, keep: usize); + fn cost(&self, debt: usize, frame_bytes: usize) -> Result; +} + +pub struct CapacityProbe { + limit: usize, + pub peak_bytes: usize, +} + +impl CapacityProbe { + pub fn new(limit: usize) -> Self { + Self { + limit, + peak_bytes: 0, + } + } + + pub fn check(&mut self, count: usize) -> bool { + #[cfg(test)] + if FORCE_CAPACITY_MISMATCH.get() { + return false; + } + if count + .checked_mul(core::mem::size_of::()) + .is_none_or(|bytes| bytes > self.limit) + { + return false; + } + let mut buffer = Vec::::new(); + if buffer.try_reserve_exact(count).is_err() { + return false; + } + let Some(bytes) = buffer.capacity().checked_mul(core::mem::size_of::()) else { + return false; + }; + self.peak_bytes = self.peak_bytes.max(bytes); + buffer.capacity() == count && bytes <= self.limit + } +} + +#[cfg(test)] +std::thread_local! { + pub(crate) static FORCE_CAPACITY_MISMATCH: std::cell::Cell = const { std::cell::Cell::new(false) }; +} diff --git a/crates/coreav-hevc/src/parser.rs b/crates/coreav-hevc/src/parser.rs index f5abcd769..726aaefe6 100644 --- a/crates/coreav-hevc/src/parser.rs +++ b/crates/coreav-hevc/src/parser.rs @@ -164,6 +164,8 @@ impl SliceSyntax { /// stream rather than continuing with partially accepted sequence state. pub struct HeaderParser { core: StreamCore, + #[cfg(hevc_thread_diagnostics)] + output_trace: Option, } impl HeaderParser { @@ -171,6 +173,8 @@ impl HeaderParser { pub fn new(limits: Limits) -> Self { Self { core: StreamCore::new(limits), + #[cfg(hevc_thread_diagnostics)] + output_trace: None, } } @@ -180,13 +184,89 @@ impl HeaderParser { bytes: &[u8], emit: impl FnMut(OutputEvent) -> Result<()>, ) -> Result { + #[cfg(hevc_thread_diagnostics)] + if let Some(outputs) = &mut self.output_trace { + return crate::thread_diagnostics::capture_outputs(outputs, || { + self.core.read_nal(bytes, emit) + }); + } self.core.read_nal(bytes, emit) } /// Complete a stream inspection. Empty or failed streams cannot pass a gate. pub fn finish(&mut self, emit: impl FnMut(OutputEvent) -> Result<()>) -> Result { + #[cfg(hevc_thread_diagnostics)] + if let Some(outputs) = &mut self.output_trace { + return crate::thread_diagnostics::capture_outputs(outputs, || self.core.finish(emit)); + } self.core.finish(emit) } + + #[cfg(hevc_thread_diagnostics)] + #[doc(hidden)] + pub fn diagnostic_trace_outputs(&mut self) -> Result<()> { + if self.core.summary.nals != 0 { + return Err(Error::InvalidData("dependency_output_trace_started")); + } + self.output_trace = Some(Default::default()); + Ok(()) + } + + #[cfg(hevc_thread_diagnostics)] + #[doc(hidden)] + pub fn diagnostic_outputs( + &self, + ) -> Result<[Option; 32]> { + self.output_trace + .as_ref() + .map(|outputs| outputs.events) + .ok_or(Error::InvalidData("dependency_output_trace_missing")) + } + + #[cfg(hevc_thread_diagnostics)] + #[doc(hidden)] + pub fn diagnostic_dependencies(&self) -> Result { + let syntax = self + .core + .picture + .as_ref() + .ok_or(Error::InvalidData("dependency_picture"))? + .independent; + let references = self.core.dpb.current_references(); + let active = crate::reconstruction::diagnostic_active_references( + references, + syntax.kind, + syntax.inter, + )?; + let convert = |entries: [Option; 16]| { + entries.map(|entry| entry.map(|value| (value.index, value.poc, value.long_term))) + }; + Ok(crate::thread_diagnostics::DependencySnapshot { + rps: convert(references.entries), + active: convert(active), + }) + } + + #[cfg(hevc_thread_diagnostics)] + #[doc(hidden)] + pub fn diagnostic_sps_envelope(&self) -> Result<(u32, u32)> { + let picture = self + .core + .picture + .as_ref() + .ok_or(Error::InvalidData("dependency_picture"))?; + let sps = self + .core + .sps + .get(picture.sps_id as usize) + .and_then(Option::as_ref) + .ok_or(Error::Internal("dependency_sps"))?; + let capacity = *sps + .max_dpb + .get(sps.sub_layers as usize) + .ok_or(Error::Internal("dependency_sps_layers"))?; + Ok((capacity, sps.sub_layers)) + } } pub(crate) struct StreamCore { @@ -223,11 +303,17 @@ pub(crate) struct SliceInput<'a> { } pub(crate) trait PictureObserver { - fn reserve_scratch(&mut self, _scratch_bytes: usize, _dpb: &Dpb) -> Result<()> { + fn reserve_scratch(&mut self, scratch_bytes: usize, dpb: &mut Dpb) -> Result<()>; + fn boundary(&mut self, dpb: &mut Dpb, scratch_bytes: usize) -> Result<()>; + fn slice(&mut self, input: SliceInput<'_>, dpb: &mut Dpb) -> Result<()>; + fn reconcile(&mut self) -> Result<()> { Ok(()) } - fn boundary(&mut self, dpb: &mut Dpb, scratch_bytes: usize) -> Result<()>; - fn slice(&mut self, input: SliceInput<'_>, dpb: &Dpb) -> Result<()>; + fn before_picture(&mut self, _dpb: &mut Dpb, _nal: NalHeader, _sps: &Sps) -> Result<()> { + Ok(()) + } + #[cfg(feature = "native-hevc-rows-private")] + fn activated_sps(&mut self, _identity: (u32, [u8; 32])) {} fn sei(&mut self, info: SeiInfo, dpb: &Dpb, scratch_bytes: usize) -> Result<()>; fn output(&mut self, emission: Emission<'_>, scratch_bytes: usize) -> Result<()>; } @@ -302,6 +388,14 @@ impl StreamCore { self.dpb.heap_bytes() } + pub(crate) fn unshare(&mut self) { + self.dpb.unshare(); + } + + pub(crate) fn dpb(&self) -> &Dpb { + &self.dpb + } + pub(crate) fn fail(&mut self) { self.failed = true; self.dpb.fail(); @@ -325,10 +419,12 @@ impl StreamCore { None, memory_limit, |_| Ok(()), - Some(observer), + Some(&mut *observer), ); if result.is_err() { + let earlier = observer.reconcile(); self.fail(); + earlier?; } result } @@ -391,6 +487,10 @@ impl StreamCore { &mut self.summary, 0, )?; + if let Some(observer) = observer.as_mut() { + observer.reconcile()?; + } + self.dpb.unshare(); let result = self .dpb .finish(|event| output(event, &mut observer, &mut emit, &mut self.summary, 0)); @@ -424,15 +524,19 @@ impl StreamCore { return Err(Error::Limit("max_nal_bytes")); } let nal = NalHeader::parse(bytes)?; + #[cfg(hevc_thread_diagnostics)] + if (32..=34).contains(&nal.unit_type) { + crate::thread_diagnostic!(event ParameterSet); + } if bytes.len().saturating_sub(2) > memory_limit { return Err(Error::Limit("reconstruction_memory")); } if let Some(observer) = observer.as_mut() { - observer.reserve_scratch(bytes.len().saturating_sub(2), &self.dpb)?; + observer.reserve_scratch(bytes.len().saturating_sub(2), &mut self.dpb)?; } let rbsp = nal::rbsp(bytes, &self.limits)?; if let Some(observer) = observer.as_mut() { - observer.reserve_scratch(rbsp.capacity(), &self.dpb)?; + observer.reserve_scratch(rbsp.capacity(), &mut self.dpb)?; } let mut bits = Bits::new(&rbsp); let info = match nal.unit_type { @@ -530,7 +634,7 @@ impl StreamCore { rbsp.capacity() .checked_add(bytes) .ok_or(Error::Limit("reconstruction_memory"))?, - &self.dpb, + &mut self.dpb, )?; } Ok(()) @@ -641,6 +745,9 @@ impl StreamCore { ) .ok_or(Error::Limit("reconstruction_memory"))?, )?; + if let Some(observer) = observer.as_mut() { + observer.before_picture(&mut self.dpb, nal, s)?; + } let hash = *self .vps_hash .get(s.vps_id as usize) @@ -668,6 +775,8 @@ impl StreamCore { .ok_or(Error::InvalidData("sps_id"))?, ); let starts_cvs = self.dpb.starts_cvs(nal); + #[cfg(feature = "native-hevc-rows-private")] + let activates_sps = self.active_sps != Some(sps_identity); if !starts_cvs && self .active_sps @@ -687,6 +796,10 @@ impl StreamCore { if starts_cvs { self.active_sps = Some(sps_identity); } + #[cfg(feature = "native-hevc-rows-private")] + if let Some(observer) = observer.as_mut().filter(|_| activates_sps) { + observer.activated_sps(sps_identity); + } Some(order) } else { None @@ -720,7 +833,7 @@ impl StreamCore { .ok_or(Error::Limit("reconstruction_memory"))?, }; if let Some(observer) = observer.as_mut() { - observer.slice(input, &self.dpb)?; + observer.slice(input, &mut self.dpb)?; } else { observe(input)?; } @@ -747,6 +860,7 @@ impl StreamCore { NalInfo::AccessUnitDelimiter } 36 | 37 => { + crate::thread_diagnostic!(event SequenceEnd); if !rbsp.is_empty() { return Err(Error::InvalidData("end_nal_payload")); } @@ -762,6 +876,10 @@ impl StreamCore { self.ended = true; NalInfo::EndOfBitstream } else { + if let Some(observer) = observer.as_mut() { + observer.reconcile()?; + } + self.dpb.unshare(); self.dpb.end_sequence(); NalInfo::EndOfSequence } @@ -776,6 +894,14 @@ impl StreamCore { } 39 | 40 => { let info = sei::parse(&rbsp, nal.unit_type == 40)?; + #[cfg(hevc_thread_diagnostics)] + match nal.unit_type { + 39 => crate::thread_diagnostic!(event PrefixSei), + _ if info.picture_hash.is_some() || info.reserved_hashes != 0 => { + crate::thread_diagnostic!(event SuffixSeiHash); + } + _ => crate::thread_diagnostic!(event SuffixSeiInformational), + } if nal.unit_type == 40 && self.picture.is_none() { return Err(Error::InvalidData("suffix_sei_without_picture")); } diff --git a/crates/coreav-hevc/src/reconstruction.rs b/crates/coreav-hevc/src/reconstruction.rs index aa7f56740..825b65a8c 100644 --- a/crates/coreav-hevc/src/reconstruction.rs +++ b/crates/coreav-hevc/src/reconstruction.rs @@ -6,9 +6,15 @@ use coreav_kernels_hevc::{self as kernels, BitDepth, IntraMode, MotionVector}; mod filter; mod inter; mod parallel; +pub(crate) mod raster; +mod rows; mod syntax; pub(crate) use parallel::Cache; +pub(crate) use rows::{ReadSupport, ReferenceRead, RowPublisher, RowStore}; + +#[cfg(hevc_thread_diagnostics)] +pub(crate) use inter::diagnostic_active_references; pub(crate) fn caught(work: impl FnOnce() -> Result) -> Result { std::panic::catch_unwind(std::panic::AssertUnwindSafe(work)) @@ -173,6 +179,24 @@ struct Cell { protected: bool, } +impl Cell { + const fn empty() -> Self { + Self { + mode: None, + motion: None, + skipped: false, + luma_residual: false, + depth: 0, + reconstructed: 0, + qp: 0, + slice: 0, + edges: 0, + transform_edges: 0, + protected: false, + } + } +} + struct Scratch { coefficients: Vec, residual: Vec, @@ -184,6 +208,102 @@ struct Scratch { inter_output: Vec, } +#[cfg(feature = "native-hevc-rows-private")] +pub(crate) struct ContinuationCapacity { + pub(crate) retained: usize, + pub(crate) base: usize, + pub(crate) row_extra: usize, +} + +#[cfg(feature = "native-hevc-rows-private")] +pub(crate) fn qualify_capacities( + info: PictureInfo, + ctus: usize, + probe: &mut crate::native_rows::CapacityProbe, +) -> bool { + let width = info.width as usize; + let height = info.height as usize; + let Some(area) = width.checked_mul(height) else { + return false; + }; + let Some(ctb) = 1usize.checked_shl(info.ctb_log2) else { + return false; + }; + let Some(band) = ctb.checked_mul(2).and_then(|n| n.checked_add(4)) else { + return false; + }; + let Some(band_area) = width.checked_mul(band.min(height)) else { + return false; + }; + [ + area, + area / 4, + band_area, + band_area / 4, + width, + width / 2, + 1024, + 4096, + ] + .into_iter() + .all(|count| probe.check::(count)) + && [area / 16, band_area / 16] + .into_iter() + .all(|count| probe.check::(count)) + && probe.check::(ctus) + && probe.check::(1024) + && [1024, 4096] + .into_iter() + .all(|count| probe.check::(count)) + && probe.check::(1024) + && probe.check::>(129) + && parallel::qualify_capacities(info, probe) +} + +#[cfg(feature = "native-hevc-rows-private")] +pub(crate) fn continuation_capacity( + info: PictureInfo, + ctus: usize, +) -> Result { + let overflow = || Error::Limit("reconstruction_memory"); + let width = info.width as usize; + let height = info.height as usize; + let area = width.checked_mul(height).ok_or_else(overflow)?; + let retained = area + .checked_mul(3) + .and_then(|n| n.checked_add((area / 16).checked_mul(core::mem::size_of::())?)) + .ok_or_else(overflow)?; + let scratch = 1024 + * (core::mem::size_of::() + + core::mem::size_of::() + + 2 * core::mem::size_of::() + + core::mem::size_of::()) + + 129 * core::mem::size_of::>() + + 2 * 4096 * core::mem::size_of::() + + 4096 * core::mem::size_of::(); + let base = ctus + .checked_mul(core::mem::size_of::()) + .and_then(|n| n.checked_add(scratch)) + .ok_or_else(overflow)?; + let band = 1usize + .checked_shl(info.ctb_log2) + .and_then(|n| n.checked_mul(2)) + .and_then(|n| n.checked_add(4)) + .ok_or_else(overflow)? + .min(height); + let band_area = width.checked_mul(band).ok_or_else(overflow)?; + let row_extra = band_area + .checked_mul(5) + .and_then(|n| n.checked_add((band_area / 16).checked_mul(core::mem::size_of::())?)) + .and_then(|n| n.checked_add(width.checked_mul(8)?)) + .ok_or_else(overflow)?; + Ok(ContinuationCapacity { + retained, + base, + row_extra, + }) +} + impl Scratch { fn heap_bytes(&self) -> Result { let bytes = [ @@ -334,6 +454,14 @@ fn sample_bytes(planes: &[Vec; 3], cells: usize) -> Result { } impl Frame { + pub(crate) fn index(&self) -> u64 { + self.index + } + + pub(crate) fn info(&self) -> PictureInfo { + self.info + } + pub(crate) fn new(input: &SliceInput<'_>, fixed: usize, limit: usize) -> Result { Self::admit(input)?; Self::allocate(input, None, fixed, limit, None) @@ -392,22 +520,7 @@ impl Frame { memory.allocate(chroma, 0)?, memory.allocate(chroma, 0)?, ], - cells: memory.allocate( - cells, - Cell { - mode: None, - motion: None, - skipped: false, - luma_residual: false, - depth: 0, - reconstructed: 0, - qp: 0, - slice: 0, - edges: 0, - transform_edges: 0, - protected: false, - }, - )?, + cells: memory.allocate(cells, Cell::empty())?, filter_ctus: memory.allocate(input.sps.ctu_count as usize, filter::Ctu::default())?, filtered: false, next_ctu: 0, diff --git a/crates/coreav-hevc/src/reconstruction/filter.rs b/crates/coreav-hevc/src/reconstruction/filter.rs index bce7d7106..aebae1f61 100644 --- a/crates/coreav-hevc/src/reconstruction/filter.rs +++ b/crates/coreav-hevc/src/reconstruction/filter.rs @@ -177,14 +177,45 @@ impl Frame { } fn deblock(&mut self, component: usize, horizontal: bool) -> Result<()> { + self.deblock_rows( + component, + horizontal, + 0..self.info.height as usize, + super::rows::SampleRows::full(), + ) + } + + pub(super) fn deblock_rows( + &mut self, + component: usize, + horizontal: bool, + rows: core::ops::Range, + storage: super::rows::SampleRows<'_>, + ) -> Result<()> { + if rows.start > rows.end + || rows.end > self.info.height as usize + || rows.start % 8 != 0 + || rows.end % 8 != 0 + { + return Err(Error::InvalidData("filter_row_range")); + } let scale = if component == 0 { 1 } else { 2 }; let width = self.info.width as usize / scale; let height = self.info.height as usize / scale; + let origin = storage.offset(component, width)?; let edge_extent = if horizontal { height } else { width }; let along_extent = if horizontal { width } else { height }; for edge in (8..edge_extent).step_by(8) { + if horizontal && (edge * scale <= rows.start || edge * scale > rows.end) { + continue; + } self.check_deadline()?; - for along in (0..along_extent).step_by(4) { + let along_range = if horizontal { + 0..along_extent + } else { + rows.start / scale..rows.end / scale + }; + for along in along_range.step_by(4) { let [x, y] = if horizontal { [along, edge] } else { @@ -196,8 +227,8 @@ impl Frame { } else { [lx - 1, ly] }; - let qcell = *self.cell(lx, ly)?; - let pcell = *self.cell(px, py)?; + let qcell = *storage.cell(self, lx, ly)?; + let pcell = *storage.cell(self, px, py)?; let ctu = self.filter_ctu(lx, ly)?; if qcell.edges & (1 << usize::from(horizontal)) == 0 || ctu.deblock_disabled @@ -235,6 +266,9 @@ impl Frame { } else { (along + row) * width + edge + index - 4 }; + let address = address + .checked_sub(origin) + .ok_or(Error::InvalidData("deblock_sample"))?; *value = *self .planes .get(component) @@ -250,7 +284,14 @@ impl Frame { strength, protected, )?; - self.write_edge(component, horizontal, edge, along, strength, result)?; + self.write_edge( + component, + horizontal, + [edge, along], + strength, + result, + storage, + )?; } else { let mut samples = [[0; 4]; 4]; for (row, line) in samples.iter_mut().enumerate() { @@ -260,6 +301,9 @@ impl Frame { } else { (along + row) * width + edge + index - 2 }; + let address = address + .checked_sub(origin) + .ok_or(Error::InvalidData("deblock_sample"))?; *value = *self .planes .get(component) @@ -278,7 +322,14 @@ impl Frame { ctu.offsets[1], protected, )?; - self.write_edge(component, horizontal, edge, along, strength, result)?; + self.write_edge( + component, + horizontal, + [edge, along], + strength, + result, + storage, + )?; } *self .stats @@ -295,12 +346,13 @@ impl Frame { &mut self, component: usize, horizontal: bool, - edge: usize, - along: usize, + [edge, along]: [usize; 2], strength: BoundaryStrength, result: [[u16; N]; 4], + storage: super::rows::SampleRows<'_>, ) -> Result<()> { let width = self.info.width as usize / if component == 0 { 1 } else { 2 }; + let origin = storage.offset(component, width)?; let plane = self .planes .get_mut(component) @@ -313,6 +365,9 @@ impl Frame { } else { (along + row) * width + edge + index - N / 2 }; + let address = address + .checked_sub(origin) + .ok_or(Error::InvalidData("deblock_sample"))?; let target = plane .get_mut(address) .ok_or(Error::InvalidData("deblock_sample"))?; @@ -354,11 +409,37 @@ impl Frame { source: Option<&[u16]>, output: &mut [u16], ) -> Result<()> { + self.sao_rows( + component, + source, + output, + 0..self.info.height as usize, + super::rows::SampleRows::full(), + ) + } + + pub(super) fn sao_rows( + &mut self, + component: usize, + source: Option<&[u16]>, + output: &mut [u16], + rows: core::ops::Range, + storage: super::rows::SampleRows<'_>, + ) -> Result<()> { + let ctb = 1usize << self.info.ctb_log2; + if rows.start > rows.end + || rows.end > self.info.height as usize + || rows.start % ctb != 0 + || rows.end != self.info.height as usize && rows.end % ctb != 0 + { + return Err(Error::InvalidData("filter_row_range")); + } let scale = if component == 0 { 1 } else { 2 }; let width = self.info.width as usize / scale; let height = self.info.height as usize / scale; let side = (1usize << self.info.ctb_log2) / scale; - for cy in (0..height).step_by(side) { + let origin = storage.offset(component, width)?; + for cy in (rows.start / scale..rows.end / scale).step_by(side) { self.check_deadline()?; for cx in (0..width).step_by(side) { let sao = *self @@ -440,25 +521,29 @@ impl Frame { let mut x = x0; while x < x1 { let start = x; - let protected = self.cell(x * scale, y * scale)?.protected; + let protected = storage.cell(self, x * scale, y * scale)?.protected; x = ((x / (4 / scale) + 1) * (4 / scale)).min(x1); while x < x1 - && self.cell(x * scale, y * scale)?.protected == protected + && storage.cell(self, x * scale, y * scale)?.protected + == protected { x = (x + 4 / scale).min(x1); } if protected { continue; } + let begin = (y * width + start) + .checked_sub(origin) + .ok_or(Error::InvalidData("sao_sample"))?; let destination = output - .get_mut(y * width + start..y * width + x) + .get_mut(begin..begin + x - start) .ok_or(Error::InvalidData("sao_sample"))?; let changed = if let Some(band) = band { band.apply(destination)? } else { let source = source.ok_or(Error::InvalidData("sao_source"))?; let samples = source - .get(y * width + start..y * width + x) + .get(begin..begin + x - start) .ok_or(Error::InvalidData("sao_sample"))?; let neighbour = |sign| -> Result<&[u16]> { let nx = start @@ -467,8 +552,11 @@ impl Frame { let ny = y .checked_add_signed(sign * dy) .ok_or(Error::InvalidData("sao_neighbour"))?; + let begin = (ny * width + nx) + .checked_sub(origin) + .ok_or(Error::InvalidData("sao_neighbour"))?; source - .get(ny * width + nx..ny * width + nx + x - start) + .get(begin..begin + x - start) .ok_or(Error::InvalidData("sao_neighbour")) }; edge.ok_or(Error::InvalidData("sao_class"))?.apply( @@ -598,6 +686,88 @@ mod tests { }) } + #[test] + fn private_filter_bands_match_whole_picture_samples_and_counts() -> Result<()> { + for ctb in [16usize, 32, 64] { + for height in [8, ctb, ctb + 8, 3 * ctb + 8] { + for bits in [8, 10] { + for kind in [1, 2] { + for class in 0..4 { + let prepare = || -> Result { + let mut frame = sao_frame(bits)?; + let width = 2 * ctb + 8; + frame.info.width = width as u32; + frame.info.height = height as u32; + frame.info.visible_width = width as u32; + frame.info.visible_height = height as u32; + frame.info.ctb_log2 = ctb.trailing_zeros(); + frame.info.tiles = false; + let retained = + super::super::raster::tests::TestSink::new(frame.info).picture; + frame.planes = retained.planes; + frame.cells = retained.cells; + let count = width.div_ceil(ctb) * height.div_ceil(ctb); + frame.next_ctu = count as u32; + frame.filter_ctus.resize(count, Ctu::default()); + let scale = 1 << (bits - 8); + for (component, plane) in frame.planes.iter_mut().enumerate() { + let width = width / if component == 0 { 1 } else { 2 }; + for (index, sample) in plane.iter_mut().enumerate() { + let (x, y) = (index % width, index / width); + *sample = (80 + + ((x / 8 + y / 8) % 2) as u16 * 6 + + if class == 0 { 0 } else { ((x + y) % 5) as u16 }) + * scale; + } + } + for (index, cell) in frame.cells.iter_mut().enumerate() { + cell.edges = 3; + cell.qp = 30; + cell.protected = index % 19 == 0; + } + for (index, ctu) in frame.filter_ctus.iter_mut().enumerate() { + ctu.seen = true; + ctu.order = index as u32; + ctu.deblock_disabled = index % 7 == 0; + ctu.slice = (index / 3) as u32; + ctu.across_slices = index % 3 != 0; + ctu.sao = [Sao { + kind, + class, + band: 10, + offsets: [2, 1, -1, -2], + }; 3]; + } + Ok(frame) + }; + let mut expected = prepare()?; + if ctb == 16 && height == 8 && bits == 8 && kind == 1 && class == 0 { + let mut strong = prepare()?; + let before = strong.planes[0][4 * 40 + 21]; + strong.deblock_rows( + 0, + false, + 0..8, + super::super::rows::SampleRows::full(), + )?; + assert_ne!(strong.planes[0][4 * 40 + 21], before); + } + expected.finalize(0, usize::MAX)?; + let actual = super::super::raster::tests::filter_fixture(prepare()?)?; + assert_eq!( + actual.planes, expected.planes, + "ctb={ctb} height={height} bits={bits} kind={kind} class={class}" + ); + assert_eq!(actual.stats, expected.stats); + assert_eq!(actual.inter_stats, expected.inter_stats); + } + } + } + } + } + Ok(()) + } + #[test] fn sao_rectangles_match_sample_decisions_at_every_block_corner() -> Result<()> { for bits in [8, 10] { diff --git a/crates/coreav-hevc/src/reconstruction/inter.rs b/crates/coreav-hevc/src/reconstruction/inter.rs index f271d4e43..55b802370 100644 --- a/crates/coreav-hevc/src/reconstruction/inter.rs +++ b/crates/coreav-hevc/src/reconstruction/inter.rs @@ -1,4 +1,6 @@ -use super::{Motion, MotionReference, ReferenceState, RetainedPicture, syntax}; +#[cfg(test)] +use super::RetainedPicture; +use super::{Motion, MotionReference, ReferenceState, syntax}; use crate::{ Error, Result, dpb::{ReferenceIdentity, References}, @@ -157,6 +159,30 @@ fn identity(references: References, entry: u8) -> Result { .ok_or(Error::InvalidData("inter_reference_catalog")) } +#[cfg(hevc_thread_diagnostics)] +pub(crate) fn diagnostic_active_references( + references: References, + kind: u32, + inter: Option, +) -> Result<[Option; 16]> { + if kind == 2 { + return Ok([None; 16]); + } + let lists = Lists::new( + references, + inter.ok_or(Error::InvalidData("dependency_inter_slice"))?, + kind, + )?; + let mut selected = [None; 16]; + for entry in lists.entries.iter().flatten().flatten() { + *selected + .get_mut(usize::from(*entry)) + .ok_or(Error::InvalidData("inter_reference_catalog"))? = + Some(identity(references, *entry)?); + } + Ok(selected) +} + impl Lists { fn new(references: References, inter: InterSlice, kind: u32) -> Result { let before = usize::from(references.before); @@ -356,7 +382,7 @@ fn scale(vector: MotionVector, from: i64, to: i64) -> Result { } fn collocated_vector( - picture: &RetainedPicture, + picture: &super::rows::ReferenceRead<'_>, position: (usize, usize), list: usize, no_backward: bool, @@ -365,14 +391,10 @@ fn collocated_vector( current_poc: i32, ) -> Result> { let (x, y) = position; - if x >= picture.info.width as usize || y >= picture.info.height as usize { - return Err(Error::InvalidData("inter_collocated_position")); - } - let cell = picture - .cells - .get((y / 4) * (picture.info.width as usize / 4) + x / 4) - .ok_or(Error::InvalidData("inter_collocated_cell"))?; - let Some(motion) = cell.motion else { + let probe = picture.motion(x, y)?; + #[cfg(hevc_thread_diagnostics)] + crate::thread_diagnostics::motion_read(y); + let Some(motion) = probe.motion else { return Ok(None); }; let selected = match motion { @@ -385,7 +407,7 @@ fn collocated_vector( }) .ok_or(Error::InvalidData("inter_collocated_list"))?, }; - let catalog = picture + let catalog = probe .reference .ok_or(Error::InvalidData("inter_collocated_catalog"))?; let source = identity(catalog.references, selected.reference)?; @@ -416,7 +438,17 @@ fn temporal( collocated.index, ZERO, )?; - let picture = dpb.reference(identity(current.references, selected.reference)?)?; + let reference_id = identity(current.references, selected.reference)?; + let _picture = dpb.reference_info(reference_id)?; + #[cfg(hevc_thread_diagnostics)] + let _row_read = + crate::thread_diagnostics::ReadGuard::new(crate::thread_diagnostics::ReadContext { + picture: walker.frame.index, + row: pu.y >> walker.frame.info.ctb_log2, + reference: identity(current.references, selected.reference)?, + reference_ctb_log2: _picture.ctb_log2, + scale: 1, + }); let target = identity(current.references, lists.get(list, index, ZERO)?.reference)?; let mut no_backward = true; for entries in lists.entries { @@ -428,31 +460,28 @@ fn temporal( } let x = pu.x + pu.width; let y = pu.y + pu.height; - if pu.y >> walker.input.sps.ctb_log2 == y >> walker.input.sps.ctb_log2 - && x < walker.frame.info.width as usize - && y < walker.frame.info.height as usize - { - if let Some(vector) = collocated_vector( - picture, - (x & !15, y & !15), + let probe = |position: (usize, usize)| { + let picture = + dpb.read_reference(reference_id, super::ReadSupport::Motion { row: position.1 })?; + collocated_vector( + &picture, + position, list, no_backward, collocated.from_l0, target, current.poc, - )? { + ) + }; + if pu.y >> walker.input.sps.ctb_log2 == y >> walker.input.sps.ctb_log2 + && x < walker.frame.info.width as usize + && y < walker.frame.info.height as usize + { + if let Some(vector) = probe((x & !15, y & !15))? { return Ok(Some(vector)); } } - collocated_vector( - picture, - ((pu.x + pu.width / 2) & !15, (pu.y + pu.height / 2) & !15), - list, - no_backward, - collocated.from_l0, - target, - current.poc, - ) + probe(((pu.x + pu.width / 2) & !15, (pu.y + pu.height / 2) & !15)) } fn motion(first: Option, second: Option) -> Result { @@ -895,7 +924,7 @@ pub(super) fn predict( value .map(|value| { identity(current.references, value.reference) - .and_then(|reference| dpb.reference(reference)) + .and_then(|reference| dpb.reference_info(reference)) }) .transpose() }); @@ -916,16 +945,16 @@ pub(super) fn predict( } } for picture in pictures.into_iter().flatten() { - if picture.info.width != walker.frame.info.width - || picture.info.height != walker.frame.info.height - || picture.info.bit_depth != walker.frame.info.bit_depth + if picture.width != walker.frame.info.width + || picture.height != walker.frame.info.height + || picture.bit_depth != walker.frame.info.bit_depth { return Err(Error::InvalidData("inter_reference_format")); } } for y in (pu.y..pu.y + pu.height).step_by(4) { for x in (pu.x..pu.x + pu.width).step_by(4) { - let cell = walker.frame.cell(x, y)?; + let cell = walker.cell(x, y)?; if cell.slice != walker.slice || cell.motion.is_some() || cell.mode.is_some() @@ -950,28 +979,45 @@ pub(super) fn predict( let Some(value) = value else { continue; }; - let picture = pictures + let _picture = pictures .get(list) .copied() .flatten() .ok_or(Error::InvalidData("inter_reference_samples"))?; - let plane = picture - .planes - .get(component) - .ok_or(Error::InvalidData("inter_reference_plane"))?; - kernels::interpolate_inter_with_rows( - kernels::InterPrediction { - depth: walker.frame.depth, - component: if component == 0 { - kernels::Component::Luma - } else { - kernels::Component::Chroma - }, - x, - y, - motion: value.vector, + let config = kernels::InterPrediction { + depth: walker.frame.depth, + component: if component == 0 { + kernels::Component::Luma + } else { + kernels::Component::Chroma + }, + x, + y, + motion: value.vector, + }; + let support = config.support(plane_width, plane_height, width, height)?; + let guard = dpb.read_reference( + identity(current.references, value.reference)?, + super::ReadSupport::Samples { + component, + last_row: *support.rows().end(), }, - kernels::RefPlane::new(plane, plane_width, plane_width, plane_height)?, + )?; + let reference = guard.plane()?; + #[cfg(hevc_thread_diagnostics)] + let _row_read = + crate::thread_diagnostics::ReadGuard::new(crate::thread_diagnostics::ReadContext { + picture: walker.frame.index, + row: pu.y >> walker.frame.info.ctb_log2, + reference: identity(current.references, value.reference)?, + reference_ctb_log2: _picture.ctb_log2, + scale: divisor, + }); + #[cfg(hevc_thread_diagnostics)] + let reference = reference.diagnostic_reads(crate::thread_diagnostics::sample_read); + kernels::interpolate_inter_with_rows( + config, + reference, kernels::RectMut::new( walker .frame @@ -1032,6 +1078,9 @@ pub(super) fn predict( } } let offset = y * plane_width + x; + let offset = offset + .checked_sub(walker.rows.offset(component, plane_width)?) + .ok_or(Error::InvalidData("inter_destination"))?; let plane = frame .planes .get_mut(component) @@ -1089,7 +1138,7 @@ pub(super) fn predict( } for y in (pu.y..pu.y + pu.height).step_by(4) { for x in (pu.x..pu.x + pu.width).step_by(4) { - let index = walker.frame.cell_index(x, y)?; + let index = walker.cell_index(x, y)?; let cell = walker .frame .cells @@ -1364,6 +1413,43 @@ mod tests { MotionVector { x, y } } + #[cfg(hevc_thread_diagnostics)] + #[test] + fn header_dependencies_use_active_lists_and_long_term_modifications() -> Result<()> { + let references = catalog(); + let p = diagnostic_active_references(references, 1, Some(controls([1, 0])))?; + assert_eq!( + p.iter() + .flatten() + .map(|value| value.index) + .collect::>(), + [0] + ); + let b = diagnostic_active_references(references, 0, Some(controls([1, 1])))?; + assert_eq!( + b.iter() + .flatten() + .map(|value| value.index) + .collect::>(), + [0, 2] + ); + let mut modified = controls([1, 0]); + modified.modifications[0] = Some([3; 15]); + let long = diagnostic_active_references(references, 1, Some(modified))?; + assert_eq!( + long.iter() + .flatten() + .map(|value| (value.index, value.long_term)) + .collect::>(), + [(3, true)] + ); + assert_eq!( + diagnostic_active_references(references, 2, None)?, + [None; 16] + ); + Ok(()) + } + #[test] fn partitions_follow_literal_normative_geometry_and_order() -> Result<()> { let area = syntax::Area { @@ -1731,28 +1817,86 @@ mod tests { long_term: false, }; assert_eq!( - collocated_vector(&picture, (16, 16), 0, true, true, target, 20)?, + collocated_vector( + &super::super::rows::ReferenceRead::new( + &picture, + super::super::ReadSupport::Motion { row: usize::MAX } + ), + (16, 16), + 0, + true, + true, + target, + 20 + )?, Some(vector(160, 80)) ); assert_eq!( - collocated_vector(&picture, (16, 16), 1, true, false, target, 20)?, + collocated_vector( + &super::super::rows::ReferenceRead::new( + &picture, + super::super::ReadSupport::Motion { row: usize::MAX } + ), + (16, 16), + 1, + true, + false, + target, + 20 + )?, Some(vector(20, 12)) ); assert_eq!( - collocated_vector(&picture, (16, 16), 0, false, true, target, 20)?, + collocated_vector( + &super::super::rows::ReferenceRead::new( + &picture, + super::super::ReadSupport::Motion { row: usize::MAX } + ), + (16, 16), + 0, + false, + true, + target, + 20 + )?, Some(vector(20, 12)) ); assert_eq!( - collocated_vector(&picture, (16, 16), 1, false, false, target, 20)?, + collocated_vector( + &super::super::rows::ReferenceRead::new( + &picture, + super::super::ReadSupport::Motion { row: usize::MAX } + ), + (16, 16), + 1, + false, + false, + target, + 20 + )?, Some(vector(160, 80)) ); assert_eq!( - collocated_vector(&picture, (0, 0), 0, true, true, target, 20)?, + collocated_vector( + &super::super::rows::ReferenceRead::new( + &picture, + super::super::ReadSupport::Motion { row: usize::MAX } + ), + (0, 0), + 0, + true, + true, + target, + 20 + )?, None ); assert_eq!( collocated_vector( - &picture, + &super::super::rows::ReferenceRead::new( + &picture, + super::super::ReadSupport::Motion { row: usize::MAX } + ), (16, 16), 0, true, @@ -1768,7 +1912,10 @@ mod tests { picture.cells[36].motion = Some(Motion::L0(lists.get(0, 3, vector(-17, 23))?)); assert_eq!( collocated_vector( - &picture, + &super::super::rows::ReferenceRead::new( + &picture, + super::super::ReadSupport::Motion { row: usize::MAX } + ), (16, 16), 0, true, @@ -1782,8 +1929,169 @@ mod tests { Some(vector(-17, 23)) ); picture.reference = None; - assert!(collocated_vector(&picture, (16, 16), 0, true, true, target, 20).is_err()); - assert!(collocated_vector(&picture, (32, 16), 0, true, true, target, 20).is_err()); + assert!( + collocated_vector( + &super::super::rows::ReferenceRead::new( + &picture, + super::super::ReadSupport::Motion { row: usize::MAX } + ), + (16, 16), + 0, + true, + true, + target, + 20 + ) + .is_err() + ); + assert!( + collocated_vector( + &super::super::rows::ReferenceRead::new( + &picture, + super::super::ReadSupport::Motion { row: usize::MAX } + ), + (32, 16), + 0, + true, + true, + target, + 20 + ) + .is_err() + ); + Ok(()) + } + + #[cfg(hevc_thread_diagnostics)] + #[test] + fn row_dependency_trace_observes_temporal_probes_and_chroma_coordinates() -> Result<()> { + use crate::thread_diagnostics::{ + ReadContext, ReadGuard, begin_rows, row_picture, take_rows, + }; + let fixture = Fixture::new(8, 1)?; + begin_rows()?; + row_picture(1, 9, fixture.info)?; + let reference = ReferenceIdentity { + index: 0, + poc: 7, + long_term: false, + }; + { + let _guard = ReadGuard::new(ReadContext { + picture: 1, + row: 0, + reference, + reference_ctb_log2: 4, + scale: 1, + }); + let picture = RetainedPicture::for_test(0, fixture.info, None); + assert_eq!( + collocated_vector( + &super::super::rows::ReferenceRead::new( + &picture, + super::super::ReadSupport::Motion { row: usize::MAX } + ), + (32, 32), + 0, + true, + true, + reference, + 9 + )?, + None + ); + } + { + let _guard = ReadGuard::new(ReadContext { + picture: 1, + row: 0, + reference, + reference_ctb_log2: 4, + scale: 2, + }); + let source = [42u16; 32 * 32]; + let mut output = [0i32; 4 * 4]; + kernels::interpolate_inter( + kernels::InterPrediction { + depth: kernels::BitDepth::new(8)?, + component: kernels::Component::Chroma, + x: 0, + y: 4, + motion: MotionVector { x: 0, y: 1 }, + }, + kernels::RefPlane::new(&source, 32, 32, 32)? + .diagnostic_reads(crate::thread_diagnostics::sample_read), + kernels::RectMut::new(&mut output, 4, 4, 4)?, + )?; + } + let pictures = take_rows(1)?; + let row = pictures + .first() + .and_then(|picture| picture.rows.first()) + .ok_or(Error::Internal("row_test"))?; + assert_eq!((row.sample_calls, row.motion_calls), (1, 1)); + let read = row + .references + .first() + .and_then(Option::as_ref) + .ok_or(Error::Internal("row_test_reference"))?; + assert_eq!( + (read.index, read.poc, read.sample_row, read.motion_row), + (0, 7, Some(1), Some(2)) + ); + Ok(()) + } + + #[cfg(hevc_thread_diagnostics)] + #[test] + fn row_dependency_trace_includes_chroma_reach_beyond_luma() -> Result<()> { + let fixture = Fixture::new(8, 1)?; + let dpb = fixture.dpb()?; + let inter = controls([1, 0]); + let input = fixture.input(1, inter); + let mut frame = + super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000, None)?; + for cell in &mut frame.cells { + cell.slice = 1; + } + crate::thread_diagnostics::begin_rows()?; + crate::thread_diagnostics::row_picture(1, 2, fixture.info)?; + let value = Lists::new(dpb.current_references(), inter, 1)?.get(0, 0, vector(0, 12))?; + let mut walker = syntax::Walker { + rows: crate::reconstruction::rows::SampleRows::full(), + frame: &mut frame, + input: &input, + location: fixture.layout.at_raster(0)?, + slice: 1, + dpb: Some(&dpb), + }; + predict( + &mut walker, + Pu { + x: 0, + y: 8, + width: 4, + height: 4, + }, + Motion::L0(value), + false, + false, + )?; + let pictures = crate::thread_diagnostics::take_rows(1)?; + let row = pictures + .first() + .and_then(|picture| picture.rows.first()) + .ok_or(Error::Internal("row_test"))?; + assert_eq!(row.sample_calls, 3); + let read = row + .references + .first() + .and_then(Option::as_ref) + .ok_or(Error::Internal("row_test_reference"))?; + assert_eq!( + (read.index, read.sample_row, read.motion_row), + (0, Some(1), None) + ); Ok(()) } @@ -1864,6 +2172,7 @@ mod tests { location: fixture.layout.at_raster(0)?, slice: 1, dpb: Some(&dpb), + rows: crate::reconstruction::rows::SampleRows::full(), }; let lists = state(&walker)?.2; let motion = match direction { @@ -1897,6 +2206,117 @@ mod tests { Ok(()) } + #[test] + fn private_row_inter_prediction_uses_local_planes_and_motion_cells() -> Result<()> { + for bits in [8, 10] { + for weighted in [false, true] { + let fixture = Fixture::new(bits, 1)?; + let dpb = fixture.dpb()?; + let mut inter = controls([1, 0]); + if weighted { + inter.weights = Some(crate::slice::Weights { + denominators: [0; 2], + lists: [[[crate::slice::Weight { + weight: 2, + offset: -1, + }; 3]; 15]; 2], + }); + } + let input = fixture.input(1, inter); + let mut serial = super::super::Frame::new_full( + &input, + 2, + dpb.current_references(), + 0, + usize::MAX, + None, + )?; + let mut private = super::super::Frame::new_full( + &input, + 2, + dpb.current_references(), + 0, + usize::MAX, + None, + )?; + for frame in [&mut serial, &mut private] { + for cell in &mut frame.cells { + cell.slice = 1; + } + } + for (component, plane) in private.planes.iter_mut().enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + *plane = plane[16 / scale * (64 / scale)..48 / scale * (64 / scale)].to_vec(); + } + private.cells = private.cells[4 * 16..12 * 16].to_vec(); + let pu = Pu { + x: 16, + y: 32, + width: 8, + height: 8, + }; + let motion = Motion::L0(MotionReference { + reference: 0, + index: 0, + vector: vector(-13, 7), + }); + predict( + &mut syntax::Walker { + frame: &mut serial, + input: &input, + location: fixture.layout.at_raster(9)?, + slice: 1, + dpb: Some(&dpb), + rows: crate::reconstruction::rows::SampleRows::full(), + }, + pu, + motion, + false, + false, + )?; + predict( + &mut syntax::Walker { + frame: &mut private, + input: &input, + location: fixture.layout.at_raster(9)?, + slice: 1, + dpb: Some(&dpb), + rows: crate::reconstruction::rows::SampleRows { + origin: 16, + above: None, + above_y: 0, + }, + }, + pu, + motion, + false, + false, + )?; + assert_eq!(private.inter_stats, serial.inter_stats); + for (component, plane) in private.planes.iter().enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + assert_eq!( + plane, + &serial.planes[component] + [16 / scale * (64 / scale)..48 / scale * (64 / scale)] + ); + } + assert_eq!( + private + .cells + .iter() + .map(|c| (c.motion, c.edges)) + .collect::>(), + serial.cells[4 * 16..12 * 16] + .iter() + .map(|c| (c.motion, c.edges)) + .collect::>() + ); + } + } + Ok(()) + } + #[test] fn prediction_uses_duplicate_active_index_weights_and_marks_motion_without_reconstruction() -> Result<()> { @@ -1933,6 +2353,7 @@ mod tests { cell.slice = 1; } let mut walker = syntax::Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -2045,6 +2466,7 @@ mod tests { cell.slice = 1; } let mut walker = syntax::Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -2108,6 +2530,7 @@ mod tests { let index = frame.cell_index(neighbour.0, neighbour.1)?; frame.cells[index].motion = Some(Motion::L0(reference)); let walker = syntax::Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -2140,6 +2563,7 @@ mod tests { frame.cells[16].motion = Some(Motion::L0(reference)); frame.cells[32].motion = Some(Motion::L0(reference)); let walker = syntax::Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -2172,6 +2596,7 @@ mod tests { } frame.cells[49].motion = Some(Motion::L0(reference)); let walker = syntax::Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -2204,6 +2629,7 @@ mod tests { } frame.cells[55].motion = Some(Motion::L0(reference)); let walker = syntax::Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(2)?, @@ -2258,6 +2684,7 @@ mod tests { cell.slice = 1; } let walker = syntax::Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -2305,6 +2732,7 @@ mod tests { let mut frame = super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000, None)?; let walker = syntax::Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -2340,6 +2768,7 @@ mod tests { cell.slice = 1; } let mut walker = syntax::Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -2386,6 +2815,7 @@ mod tests { let mut frame = super::super::Frame::new_full(&input, 2, dpb.current_references(), 0, 1_000_000, None)?; let walker = syntax::Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, diff --git a/crates/coreav-hevc/src/reconstruction/parallel.rs b/crates/coreav-hevc/src/reconstruction/parallel.rs index 1a58d4c22..fe911f0bd 100644 --- a/crates/coreav-hevc/src/reconstruction/parallel.rs +++ b/crates/coreav-hevc/src/reconstruction/parallel.rs @@ -21,6 +21,11 @@ pub(crate) struct Cache { } impl Cache { + #[cfg(test)] + pub(crate) fn frames_for_test(&mut self) -> &mut Vec { + &mut self.frames + } + #[cfg(test)] pub(crate) fn hits(&self) -> usize { self.hits @@ -254,6 +259,31 @@ fn reserve(values: &mut Vec, count: usize, memory: &mut Memory) -> Result< ) } +#[cfg(feature = "native-hevc-rows-private")] +pub(super) fn qualify_capacities( + info: crate::PictureInfo, + probe: &mut crate::native_rows::CapacityProbe, +) -> bool { + let Some(ctb) = 1usize.checked_shl(info.ctb_log2) else { + return false; + }; + let width = info.width as usize; + let height = info.height as usize; + let columns = width.div_ceil(ctb); + let rows = height.div_ceil(ctb); + [ctb.min(height), height % ctb].into_iter().all(|lines| { + let Some(area) = width.checked_mul(lines) else { + return false; + }; + [area, area / 4] + .into_iter() + .all(|count| probe.check::(count)) + && probe.check::(area / 16) + }) && probe.check::(columns) + && (1..=rows).all(|count| probe.check::(count)) + && (1..16).all(|count| probe.check::(count)) +} + fn copy_rows( source: &[T], destination: &mut [T], @@ -416,9 +446,9 @@ impl Frame { index, info, depth, - planes, - cells, - filter_ctus, + planes: _, + cells: _, + filter_ctus: _, filtered, next_ctu, segment_closed, @@ -440,11 +470,6 @@ impl Frame { *index = source.index; *info = source.info; *depth = source.depth; - for (target, source) in planes.iter_mut().zip(&source.planes) { - target.copy_from_slice(source); - } - cells.copy_from_slice(&source.cells); - filter_ctus.copy_from_slice(&source.filter_ctus); *filtered = source.filtered; *next_ctu = source.next_ctu; *segment_closed = source.segment_closed; @@ -477,6 +502,38 @@ impl Frame { inter_output.copy_from_slice(&source.scratch.inter_output); } + fn reset_row(&mut self, index: usize) -> Result<()> { + let ctb = 1usize << self.info.ctb_log2; + let width = self.info.width as usize; + let height = self.info.height as usize; + let start = index.saturating_sub(1) * ctb; + let end = ((index + 2) * ctb).min(height); + self.cells + .get_mut(start / 4 * (width / 4)..end / 4 * (width / 4)) + .ok_or(Error::InvalidData("wpp_row_storage"))? + .fill(Cell { + mode: None, + motion: None, + skipped: false, + luma_residual: false, + depth: 0, + reconstructed: 0, + qp: 0, + slice: 0, + edges: 0, + transform_edges: 0, + protected: false, + }); + let columns = width.div_ceil(ctb); + self.filter_ctus + .get_mut( + index.saturating_sub(1) * columns..(index + 2).min(height.div_ceil(ctb)) * columns, + ) + .ok_or(Error::InvalidData("wpp_row_storage"))? + .fill(Ctu::default()); + Ok(()) + } + #[expect( clippy::too_many_arguments, reason = "same slice and resource boundary as serial reconstruction" @@ -651,16 +708,19 @@ fn work( frame.inter_stats = InterStats::default(); frame.wpp = None; let mut budget = allowance; - let result = caught(|| decode_row(frame, input, dpb, ranges, rows, index, &mut budget)) - .map(|()| Continuation { - next_ctu: frame.next_ctu, - closed: frame.segment_closed, - contexts: frame.contexts, - checkpoint: frame.wpp, - qp: frame.qp, - stats: frame.stats, - inter: frame.inter_stats, - }); + let result = caught(|| { + frame.reset_row(index)?; + decode_row(frame, input, dpb, ranges, rows, index, &mut budget) + }) + .map(|()| Continuation { + next_ctu: frame.next_ctu, + closed: frame.segment_closed, + contexts: frame.contexts, + checkpoint: frame.wpp, + qp: frame.qp, + stats: frame.stats, + inter: frame.inter_stats, + }); row.lock().outcome = Some(Outcome { result, consumed: allowance - budget, @@ -768,6 +828,78 @@ fn decode_row( mod cache_tests { use super::*; + #[test] + fn reused_metadata_is_invalidated_only_in_the_assigned_row_and_neighbour_halos() -> Result<()> { + let mut core = crate::parser::StreamCore::new(crate::Limits::default()); + for set in crate::tests::synthetic_parameter_nals_with_wpp(true) { + core.read_nal(&set, |_| Ok(()))?; + } + let nal = crate::tests::intra_first_with_wpp(21, 1, Some(0)); + core.read_selected( + &nal, + |_| Ok(()), + Some(0), + usize::MAX, + |input| { + for height in [504, 512] { + let mut sps = *input.sps; + sps.height = height; + sps.visible_height = height; + sps.ctu_count = 8; + let input = SliceInput { + sps: &sps, + picture: crate::PictureInfo { + height, + visible_height: height, + ..input.picture + }, + ..input + }; + let source = Frame::new(&input, 0, usize::MAX)?; + let mut frame = Frame::new(&input, 0, usize::MAX)?; + for index in [0usize, 4, 7] { + for plane in &mut frame.planes { + plane.fill(255); + } + for cell in &mut frame.cells { + cell.slice = 1; + cell.reconstructed = 7; + cell.mode = Some(coreav_kernels_hevc::IntraMode::new(34)?); + } + for ctu in &mut frame.filter_ctus { + ctu.seen = true; + } + frame.filtered = true; + frame.hash_seen = true; + frame.next_ctu = 99; + frame.segment_closed = true; + frame.reset_private(&source); + assert!(!frame.filtered && !frame.hash_seen && !frame.segment_closed); + assert_eq!(frame.next_ctu, 0); + assert!(frame.cells.iter().all(|cell| cell.slice == 1)); + assert!(frame.filter_ctus.iter().all(|ctu| ctu.seen)); + frame.reset_row(index)?; + let start = index.saturating_sub(1); + let end = (index + 2).min(8); + for (at, cell) in frame.cells.iter().enumerate() { + let row = at / (64 * 64 / 16); + let reset = (start..end).contains(&row); + assert_eq!(cell.slice == 0, reset); + assert_eq!(cell.reconstructed == 0, reset); + assert_eq!(cell.mode.is_none(), reset); + } + for (row, ctu) in frame.filter_ctus.iter().enumerate() { + assert_eq!(!ctu.seen, (start..end).contains(&row)); + } + assert!(frame.planes.iter().flatten().all(|sample| *sample == 255)); + } + } + Ok(()) + }, + )?; + Ok(()) + } + #[test] fn empty_allocated_worker_slots_are_charged_and_revocable() -> Result<()> { let mut cache = Cache::default(); diff --git a/crates/coreav-hevc/src/reconstruction/raster.rs b/crates/coreav-hevc/src/reconstruction/raster.rs new file mode 100644 index 000000000..99f8349a6 --- /dev/null +++ b/crates/coreav-hevc/src/reconstruction/raster.rs @@ -0,0 +1,772 @@ +use super::{ + Cell, Frame, InterStats, Memory, ReferenceState, RetainedPicture, allocation_bytes, + rows::SampleRows, +}; +use crate::{Error, PictureInfo, Result, intra::IntraStats, parser::SliceInput}; +use core::ops::Range; + +pub(crate) trait Sink { + fn motion(&mut self, band: MotionBand<'_>) -> Result<()>; + fn samples(&mut self, band: SampleBand<'_>) -> Result<()>; +} + +pub(crate) struct MotionBand<'a> { + pub(super) rows: Range, + pub(super) cells: &'a [Cell], +} + +pub(crate) struct SampleBand<'a> { + pub(super) origin: usize, + pub(super) dirty: Range, + pub(super) filtered_through: usize, + pub(super) planes: &'a [Vec; 3], +} + +pub(crate) struct Producer { + frame: Frame, + window: Window, + failed: bool, +} + +struct Window { + origin: usize, + reconstructed: usize, + filtered: usize, + intra: [Vec; 3], + pre_sao: [Vec; 3], + source: Vec, +} + +pub(crate) struct Completion { + pub(super) index: u64, + pub(super) info: PictureInfo, + pub(crate) stats: IntraStats, + pub(crate) inter_stats: InterStats, + pub(super) reference: Option, +} + +impl Completion { + pub(super) fn retain( + self, + planes: [Vec; 3], + cells: Vec, + timestamp: Option, + origin: Option, + ) -> Result { + let width = self.info.width as usize; + let height = self.info.height as usize; + if planes[0].len() != width * height + || planes[1].len() != width / 2 * (height / 2) + || planes[2].len() != planes[1].len() + || cells.len() != width / 4 * (height / 4) + { + return Err(Error::InvalidData("selected_picture_incomplete")); + } + Ok(RetainedPicture { + index: self.index, + info: self.info, + planes, + cells, + timestamp, + origin, + stats: self.stats, + inter_stats: self.inter_stats, + reference: self.reference, + }) + } +} + +impl Producer { + #[expect( + clippy::result_large_err, + reason = "return the untouched fallback owner without allocating" + )] + pub(crate) fn try_from_full( + mut frame: Frame, + fixed: usize, + limit: usize, + ) -> core::result::Result<(Self, RetainedPicture), (Frame, Error)> { + let prepared = (|| { + if frame.info.tiles || frame.info.wpp { + return Err(Error::Unsupported("raster_picture_layout")); + } + let width = frame.info.width as usize; + let height = frame.info.height as usize; + if frame.next_ctu != 0 + || frame.segment_closed + || frame.filtered + || frame.hash_seen + || frame.wpp.is_some() + || frame.reference.is_none() + || frame.stats != IntraStats::default() + || frame.inter_stats != InterStats::default() + || frame.filter_ctus.iter().any(|ctu| ctu.seen) + || frame.cells.iter().any(|cell| cell.reconstructed != 0) + || frame.planes[0].len() != width * height + || frame.planes[1].len() != width / 2 * (height / 2) + || frame.planes[2].len() != frame.planes[1].len() + || frame.cells.len() != width / 4 * (height / 4) + { + return Err(Error::InvalidData("raster_picture_state")); + } + let mut memory = Memory::new(fixed, limit)?; + memory.charge(frame.heap_bytes()?)?; + let ctb = 1usize << frame.info.ctb_log2; + let capacity = (2 * ctb + 4).min(height); + let mut planes = [ + memory.allocate(capacity * width, 0)?, + memory.allocate(capacity / 2 * (width / 2), 0)?, + memory.allocate(capacity / 2 * (width / 2), 0)?, + ]; + let mut cells = memory.allocate(capacity / 4 * (width / 4), Cell::empty())?; + let window = Window::allocate(frame.info, &mut memory)?; + let end = (2 * ctb).min(height); + for (component, plane) in planes.iter_mut().enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + plane.truncate(end / scale * (width / scale)); + } + cells.truncate(end / 4 * (width / 4)); + Ok((planes, cells, window)) + })(); + let (planes, cells, window) = match prepared { + Ok(prepared) => prepared, + Err(error) => return Err((frame, error)), + }; + let backing = RetainedPicture { + index: frame.index, + info: frame.info, + planes: core::mem::replace(&mut frame.planes, planes), + cells: core::mem::replace(&mut frame.cells, cells), + timestamp: None, + origin: None, + stats: frame.stats, + inter_stats: frame.inter_stats, + reference: frame.reference, + }; + Ok(( + Self { + frame, + window, + failed: false, + }, + backing, + )) + } + + pub(crate) fn heap_bytes(&self) -> Result { + self.window + .extra_bytes()? + .checked_add(self.frame.heap_bytes()?) + .ok_or(Error::Limit("reconstruction_memory")) + } + + pub(crate) fn decode( + &mut self, + input: SliceInput<'_>, + references: &dyn crate::dpb::ReferenceProvider, + remaining: &mut u64, + fixed: usize, + limit: usize, + sink: &mut impl Sink, + ) -> Result { + if self.failed { + return Err(Error::InvalidData("selected_segment_state")); + } + let result = (|| { + if input.syntax.kind > 2 || input.syntax.kind < 2 && input.syntax.inter.is_none() { + return Err(Error::InvalidData("inter_segment_state")); + } + let fixed = fixed + .checked_add(self.window.extra_bytes()?) + .ok_or(Error::Limit("reconstruction_memory"))?; + let (ranges, _, peak) = self.frame.prepare_slice(&input, fixed, limit, None)?; + if input.pps.tiles || input.pps.wpp { + return Err(Error::Unsupported("raster_picture_layout")); + } + let window = &mut self.window; + self.frame + .traverse_slice(&input, &ranges, remaining, |frame, slice, cabac| { + let location = frame.decode_ctu_rows( + &input, + Some(references), + slice, + cabac, + SampleRows { + origin: window.origin, + above: (window.reconstructed != 0).then_some(&window.intra), + above_y: window.reconstructed, + }, + )?; + if location.x + 1 == input.layout.width { + window.row_complete(frame, sink)?; + } + Ok(location) + })?; + if self.frame.complete() { + self.window.filter_band(&mut self.frame, sink)?; + self.frame.filtered = true; + } + Ok(peak) + })(); + self.failed = result.is_err(); + result + } + + pub(crate) fn finish(self) -> Result { + if self.failed + || !self.frame.complete() + || !self.frame.filtered + || self.window.filtered != self.frame.info.height as usize + || self.frame.filter_ctus.iter().any(|ctu| !ctu.seen) + { + return Err(Error::InvalidData("selected_picture_incomplete")); + } + Ok(Completion { + index: self.frame.index, + info: self.frame.info, + stats: self.frame.stats, + inter_stats: self.frame.inter_stats, + reference: self.frame.reference, + }) + } + + pub(crate) fn hash(&mut self, sei: crate::SeiInfo, planes: [&[u16]; 3]) -> Result<()> { + if sei.picture_hash.is_none() && sei.reserved_hashes == 0 { + return Ok(()); + } + if !self.frame.complete() || self.frame.filter_ctus.iter().any(|ctu| !ctu.seen) { + return Err(Error::InvalidData("selected_picture_incomplete")); + } + if !self.frame.filtered || self.frame.hash_seen { + return Err(Error::InvalidData("selected_picture_hash_placement")); + } + self.frame.hash_seen = true; + if let Some(expected) = sei.picture_hash { + crate::picture_hash::verify( + &expected, + planes, + self.frame.info.width as usize, + self.frame.info.height as usize, + self.frame.info.bit_depth, + )?; + self.frame.stats.picture_hashes += 1; + } + self.frame.stats.reserved_hashes += u64::from(sei.reserved_hashes); + Ok(()) + } +} + +impl Window { + fn allocate(info: PictureInfo, memory: &mut Memory<'_>) -> Result { + let width = info.width as usize; + Ok(Self { + origin: 0, + reconstructed: 0, + filtered: 0, + intra: [ + memory.allocate(width, 0)?, + memory.allocate(width / 2, 0)?, + memory.allocate(width / 2, 0)?, + ], + pre_sao: [ + memory.allocate(width, 0)?, + memory.allocate(width / 2, 0)?, + memory.allocate(width / 2, 0)?, + ], + source: memory.allocate( + (2 * (1usize << info.ctb_log2) + 4).min(info.height as usize) * width, + 0, + )?, + }) + } + + fn extra_bytes(&self) -> Result { + self.intra.iter().chain(&self.pre_sao).try_fold( + allocation_bytes(&self.source)?, + |total, plane| { + total + .checked_add(allocation_bytes(plane)?) + .ok_or(Error::Limit("reconstruction_memory")) + }, + ) + } + + fn row_complete(&mut self, frame: &mut Frame, sink: &mut impl Sink) -> Result<()> { + let width = frame.info.width as usize; + let ctb = 1usize << frame.info.ctb_log2; + let start = self.reconstructed; + let end = (start + ctb).min(frame.info.height as usize); + for (component, (border, plane)) in self.intra.iter_mut().zip(&frame.planes).enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + let width = width / scale; + let last = (end / scale - self.origin / scale - 1) * width; + border.copy_from_slice( + plane + .get(last..last + width) + .ok_or(Error::Internal("raster_intra_border"))?, + ); + } + let begin = (start - self.origin) / 4 * (width / 4); + let last = (end - self.origin) / 4 * (width / 4); + sink.motion(MotionBand { + rows: start..end, + cells: frame + .cells + .get(begin..last) + .ok_or(Error::Internal("raster_motion_band"))?, + })?; + let storage = SampleRows { + origin: self.origin, + above: None, + above_y: 0, + }; + for component in 0..2 { + frame.deblock_rows(component, false, start..end, storage)?; + } + self.reconstructed = end; + if start != 0 { + self.filter_band(frame, sink)?; + if end != frame.info.height as usize { + self.advance(frame); + } + } + Ok(()) + } + + fn filter_band(&mut self, frame: &mut Frame, sink: &mut impl Sink) -> Result<()> { + let start = self.filtered; + let end = (start + (1usize << frame.info.ctb_log2)).min(frame.info.height as usize); + let storage = SampleRows { + origin: self.origin, + above: None, + above_y: 0, + }; + for component in 0..2 { + frame.deblock_rows(component, true, start..end, storage)?; + } + for (component, previous) in self.pre_sao.iter_mut().enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + let width = frame.info.width as usize / scale; + let mut output = core::mem::take( + frame + .planes + .get_mut(component) + .ok_or(Error::Internal("raster_component"))?, + ); + let result = (|| { + let source = self + .source + .get_mut(..output.len()) + .ok_or(Error::Internal("raster_source"))?; + source.copy_from_slice(&output); + if start != 0 { + let above = (start / scale - self.origin / scale - 1) * width; + source + .get_mut(above..above + width) + .ok_or(Error::Internal("raster_sao_border"))? + .copy_from_slice(previous); + } + let last = (end / scale - self.origin / scale - 1) * width; + previous.copy_from_slice( + source + .get(last..last + width) + .ok_or(Error::Internal("raster_sao_border"))?, + ); + frame.sao_rows(component, Some(source), &mut output, start..end, storage) + })(); + *frame + .planes + .get_mut(component) + .ok_or(Error::Internal("raster_component"))? = output; + result?; + } + sink.samples(SampleBand { + origin: self.origin, + dirty: start..self.reconstructed, + filtered_through: end, + planes: &frame.planes, + })?; + self.filtered = end; + Ok(()) + } + + fn advance(&mut self, frame: &mut Frame) { + let origin = self.filtered.saturating_sub(4); + let end = + (self.filtered + 2 * (1usize << frame.info.ctb_log2)).min(frame.info.height as usize); + for (component, plane) in frame.planes.iter_mut().enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + let width = frame.info.width as usize / scale; + let begin = (origin - self.origin) / scale * width; + let retained = plane.len() - begin; + plane.copy_within(begin.., 0); + plane.truncate(retained); + plane.resize((end - origin) / scale * width, 0); + } + let begin = (origin - self.origin) / 4 * (frame.info.width as usize / 4); + let retained = frame.cells.len() - begin; + frame.cells.copy_within(begin.., 0); + frame.cells.truncate(retained); + frame.cells.resize( + (end - origin) / 4 * (frame.info.width as usize / 4), + Cell::empty(), + ); + self.origin = origin; + } +} + +#[cfg(test)] +pub(in crate::reconstruction) mod tests { + use super::*; + use crate::dpb::References; + + pub(in crate::reconstruction) fn set_deadline( + producer: &mut Producer, + deadline: Option, + ) { + producer.frame.deadline = deadline; + } + + pub(in crate::reconstruction) fn conversion(input: SliceInput<'_>) -> Result<()> { + let frame = Frame::new_full(&input, 0, References::default(), 0, usize::MAX, None)?; + let original_bytes = frame.heap_bytes()?; + let (producer, backing) = + Producer::try_from_full(frame, 0, usize::MAX).map_err(|(_, error)| error)?; + let bytes = producer.heap_bytes()? + backing.heap_bytes()?; + #[cfg(feature = "native-hevc-rows-private")] + { + let capacity = + super::super::continuation_capacity(input.picture, input.sps.ctu_count as usize)?; + assert_eq!(backing.heap_bytes()?, capacity.retained); + assert_eq!(original_bytes, capacity.retained + capacity.base); + assert_eq!(producer.heap_bytes()?, capacity.base + capacity.row_extra); + } + let fixed = 17 + input.scratch_bytes; + let dpb = crate::dpb::Dpb::new_reconstruction(); + for limit in [ + fixed + original_bytes - 1, + fixed + original_bytes, + fixed + bytes - 1, + fixed + bytes, + ] { + let frame = Frame::new_full(&input, 0, References::default(), 0, usize::MAX, None)?; + let planes = frame.planes.each_ref().map(|plane| plane.as_ptr()); + let cells = frame.cells.as_ptr(); + let metadata = frame.filter_ctus.as_ptr(); + let scratch = frame.scratch.coefficients.as_ptr(); + let contexts = frame.contexts; + let qp = frame.qp; + match Producer::try_from_full(frame, fixed, limit) { + Ok((producer, backing)) => { + assert_eq!(limit, fixed + bytes); + assert_eq!( + backing.planes.each_ref().map(|plane| plane.as_ptr()), + planes + ); + assert_eq!(backing.cells.as_ptr(), cells); + assert_eq!(producer.frame.filter_ctus.as_ptr(), metadata); + assert_eq!(producer.frame.scratch.coefficients.as_ptr(), scratch); + assert_eq!(producer.frame.contexts, contexts); + assert_eq!(producer.frame.qp, qp); + assert_eq!(producer.heap_bytes()? + backing.heap_bytes()?, bytes); + } + Err((mut frame, error)) => { + assert!(limit < fixed + bytes); + assert_eq!(error, Error::Limit("reconstruction_memory")); + assert_eq!(frame.planes.each_ref().map(|plane| plane.as_ptr()), planes); + assert_eq!(frame.cells.as_ptr(), cells); + assert_eq!(frame.filter_ctus.as_ptr(), metadata); + assert_eq!(frame.scratch.coefficients.as_ptr(), scratch); + assert_eq!(frame.contexts, contexts); + assert_eq!(frame.qp, qp); + assert_eq!(frame.heap_bytes()?, original_bytes); + assert_eq!(frame.next_ctu, 0); + assert!(!frame.segment_closed); + assert_eq!(frame.stats, IntraStats::default()); + frame.decode_full(input, &dpb, &mut 1000, 0, usize::MAX, 1, None)?; + assert!(frame.complete()); + } + } + } + let mut frame = Frame::new_full(&input, 0, References::default(), 0, usize::MAX, None)?; + frame.next_ctu = 1; + match Producer::try_from_full(frame, 0, usize::MAX) { + Ok(_) => panic!("partially reconstructed frame accepted"), + Err((frame, error)) => { + assert_eq!(frame.next_ctu, 1); + assert_eq!(error, Error::InvalidData("raster_picture_state")); + } + } + Ok(()) + } + + pub(in crate::reconstruction) fn producer( + input: &SliceInput<'_>, + poc: i32, + references: References, + ) -> Result<(Producer, TestSink)> { + let frame = Frame::new_full(input, poc, references, 0, usize::MAX, None)?; + let (producer, backing) = Producer::try_from_full(frame, input.scratch_bytes, usize::MAX) + .map_err(|(_, error)| error)?; + Ok((producer, TestSink::from_backing(backing))) + } + + pub(in crate::reconstruction) fn assert_entropy(producer: &Producer, serial: &Frame) { + assert_eq!(producer.frame.contexts, serial.contexts); + assert_eq!(producer.frame.qp, serial.qp); + assert_eq!(producer.frame.next_ctu, serial.next_ctu); + assert_eq!(producer.frame.segment_closed, serial.segment_closed); + assert_eq!(producer.frame.stats, serial.stats); + } + + pub(in crate::reconstruction) fn intra_border(input: SliceInput<'_>) -> Result<()> { + let dpb = crate::dpb::Dpb::new_reconstruction(); + let initial = SliceInput { + header: crate::SliceHeader { + first: true, + address: 0, + ..input.header + }, + ..input + }; + let mut serial = Frame::new_full(&initial, 0, References::default(), 0, usize::MAX, None)?; + let (mut producer, mut sink) = producer(&initial, 0, References::default())?; + let width = input.picture.width as usize; + let ctb = 1usize << input.picture.ctb_log2; + for frame in [&mut serial, &mut producer.frame] { + for (component, plane) in frame.planes.iter_mut().enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + for (index, sample) in plane + .iter_mut() + .take(width / scale * (ctb / scale)) + .enumerate() + { + *sample = ((80 + (index % (width / scale) / (8 / scale)) % 2 * 6) + << (input.picture.bit_depth - 8)) as u16; + } + } + for (index, cell) in frame + .cells + .iter_mut() + .take(width / 4 * (ctb / 4)) + .enumerate() + { + let (x, y) = (index % (width / 4) * 4, index / (width / 4) * 4); + cell.mode = Some(super::super::IntraMode::new(26)?); + cell.reconstructed = 7; + cell.qp = 30; + cell.slice = 1; + cell.edges = u8::from(x % 8 == 0) | (u8::from(y % 8 == 0) << 1); + cell.transform_edges = cell.edges; + } + frame.filter_ctus[0] = super::super::filter::Ctu { + seen: true, + order: 0, + slice: 1, + across_slices: true, + ..Default::default() + }; + frame.next_ctu = 1; + frame.segment_closed = true; + } + let unfiltered = serial.planes[0][(ctb - 1) * width..ctb * width].to_vec(); + producer + .window + .row_complete(&mut producer.frame, &mut sink)?; + assert_eq!(producer.window.intra[0], unfiltered); + assert_ne!( + &producer.frame.planes[0][(ctb - 1) * width..ctb * width], + &unfiltered + ); + assert!(producer.frame.stats.deblock_changed_samples[0] > 0); + assert!(producer.frame.inter_stats.boundary_strengths[2] > 0); + let mut serial_bins = 1000; + let mut raster_bins = 1000; + serial.decode_full(input, &dpb, &mut serial_bins, 0, usize::MAX, 1, None)?; + producer.decode(input, &dpb, &mut raster_bins, 0, usize::MAX, &mut sink)?; + serial.finalize(0, usize::MAX)?; + let actual = sink.retain(producer.finish()?)?; + assert_eq!(actual.stats.pcm_units, 0); + assert!(actual.cells[ctb / 4 * (width / 4)].mode.is_some()); + assert_eq!(raster_bins, serial_bins); + assert_eq!(actual.planes, serial.planes); + assert_eq!(actual.stats, serial.stats); + assert_eq!(actual.inter_stats, serial.inter_stats); + Ok(()) + } + + pub(in crate::reconstruction) fn qp_continuation( + first: SliceInput<'_>, + second: SliceInput<'_>, + ) -> Result<()> { + let dpb = crate::dpb::Dpb::new_reconstruction(); + let mut serial = Frame::new_full(&first, 0, References::default(), 0, usize::MAX, None)?; + let (mut producer, mut sink) = producer(&first, 0, References::default())?; + let mut serial_bins = 1000; + let mut producer_bins = 1000; + serial.decode_full(first, &dpb, &mut serial_bins, 0, usize::MAX, 1, None)?; + producer.decode(first, &dpb, &mut producer_bins, 0, usize::MAX, &mut sink)?; + assert_eq!(sink.motion, 0); + assert_eq!(sink.filtered, 0); + serial.qp = super::super::syntax::Qp::new(29); + producer.frame.qp = serial.qp; + serial.decode_full(second, &dpb, &mut serial_bins, 0, usize::MAX, 1, None)?; + producer.decode(second, &dpb, &mut producer_bins, 0, usize::MAX, &mut sink)?; + assert_entropy(&producer, &serial); + assert_eq!(serial_bins, producer_bins); + let actual = sink.retain(producer.finish()?)?; + assert_eq!( + actual.cells.iter().map(|cell| cell.qp).collect::>(), + serial.cells.iter().map(|cell| cell.qp).collect::>() + ); + assert!(actual.cells.iter().any(|cell| cell.qp == 29)); + Ok(()) + } + + pub(in crate::reconstruction) struct TestSink { + pub(in crate::reconstruction) picture: RetainedPicture, + pub(in crate::reconstruction) motion: usize, + pub(in crate::reconstruction) filtered: usize, + pub(in crate::reconstruction) commits: usize, + prefix: [Vec; 3], + } + + impl TestSink { + pub(in crate::reconstruction) fn new(info: PictureInfo) -> Self { + let mut picture = RetainedPicture::for_test( + 0, + PictureInfo { + width: 8, + height: 8, + ..info + }, + None, + ); + picture.info = info; + let width = info.width as usize; + let height = info.height as usize; + picture.planes = [ + vec![0; width * height], + vec![0; width / 2 * (height / 2)], + vec![0; width / 2 * (height / 2)], + ]; + picture.cells = vec![picture.cells[0]; width / 4 * (height / 4)]; + Self::from_backing(picture) + } + + fn from_backing(picture: RetainedPicture) -> Self { + Self { + picture, + motion: 0, + filtered: 0, + commits: 0, + prefix: core::array::from_fn(|_| Vec::new()), + } + } + + pub(in crate::reconstruction) fn retain( + self, + completion: Completion, + ) -> Result { + completion.retain(self.picture.planes, self.picture.cells, None, None) + } + } + + impl Sink for TestSink { + fn motion(&mut self, band: MotionBand<'_>) -> Result<()> { + assert_eq!(band.rows.start, self.motion); + let stride = self.picture.info.width as usize / 4; + self.picture.cells[band.rows.start / 4 * stride..band.rows.end / 4 * stride] + .copy_from_slice(band.cells); + self.motion = band.rows.end; + Ok(()) + } + + fn samples(&mut self, band: SampleBand<'_>) -> Result<()> { + assert_eq!(band.dirty.start, self.filtered); + assert!(band.filtered_through <= band.dirty.end); + assert!(band.dirty.end <= self.motion); + for component in 0..3 { + let scale = if component == 0 { 1 } else { 2 }; + let width = self.picture.info.width as usize / scale; + let plane = &mut self.picture.planes[component]; + let prefix = self.filtered / scale * width; + assert_eq!(&plane[..prefix], &self.prefix[component]); + let begin = band.dirty.start / scale * width; + let end = band.dirty.end / scale * width; + let origin = band.origin / scale * width; + plane[begin..end] + .copy_from_slice(&band.planes[component][begin - origin..end - origin]); + assert_eq!(&plane[..prefix], &self.prefix[component]); + self.prefix[component] = plane[..band.filtered_through / scale * width].to_vec(); + } + self.filtered = band.filtered_through; + self.commits += 1; + Ok(()) + } + } + + pub(in crate::reconstruction) fn filter_fixture(mut frame: Frame) -> Result { + let width = frame.info.width as usize; + let height = frame.info.height as usize; + let ctb = 1usize << frame.info.ctb_log2; + let samples = core::mem::take(&mut frame.planes); + let cells = core::mem::take(&mut frame.cells); + let capacity = (2 * ctb + 4).min(height); + let mut memory = Memory::new(0, usize::MAX)?; + frame.planes = [ + memory.allocate(capacity * width, 0)?, + memory.allocate(capacity / 2 * (width / 2), 0)?, + memory.allocate(capacity / 2 * (width / 2), 0)?, + ]; + frame.cells = memory.allocate(capacity / 4 * (width / 4), cells[0])?; + let mut sink = TestSink::new(frame.info); + let window = Window::allocate(frame.info, &mut memory)?; + let end = (2 * ctb).min(height); + for (component, plane) in frame.planes.iter_mut().enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + plane.truncate(end / scale * (width / scale)); + } + frame.cells.truncate(end / 4 * (width / 4)); + let mut producer = Producer { + frame, + window, + failed: false, + }; + let bytes = producer.heap_bytes()?; + for start in (0..height).step_by(ctb) { + let end = (start + ctb).min(height); + let window = &mut producer.window; + let frame = &mut producer.frame; + for (component, source) in samples.iter().enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + let width = width / scale; + let begin = start / scale * width; + let last = end / scale * width; + let origin = window.origin / scale * width; + frame.planes[component][begin - origin..last - origin] + .copy_from_slice(&source[begin..last]); + } + let begin = start / 4 * (width / 4); + let last = end / 4 * (width / 4); + let origin = window.origin / 4 * (width / 4); + frame.cells[begin - origin..last - origin].copy_from_slice(&cells[begin..last]); + window.row_complete(frame, &mut sink)?; + assert_eq!(producer.heap_bytes()?, bytes); + assert!(producer.frame.planes[0].capacity() <= capacity * width); + for (component, source) in samples.iter().enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + let width = width / scale; + let last = (end / scale - 1) * width; + assert_eq!(producer.window.intra[component], source[last..last + width]); + } + } + producer + .window + .filter_band(&mut producer.frame, &mut sink)?; + producer.frame.filtered = true; + assert_eq!(sink.filtered, height); + assert_eq!(sink.commits, height.div_ceil(ctb)); + sink.retain(producer.finish()?) + } +} diff --git a/crates/coreav-hevc/src/reconstruction/rows.rs b/crates/coreav-hevc/src/reconstruction/rows.rs new file mode 100644 index 000000000..8c0c81002 --- /dev/null +++ b/crates/coreav-hevc/src/reconstruction/rows.rs @@ -0,0 +1,260 @@ +use super::{Cell, Frame, Motion, ReferenceState, RetainedPicture}; +use crate::{Error, Result}; +use std::sync::RwLockReadGuard; + +mod store; +pub(crate) use store::{RowPublisher, RowStore}; + +#[derive(Clone, Copy)] +pub(crate) enum ReadSupport { + Samples { component: usize, last_row: usize }, + Motion { row: usize }, +} + +pub(crate) struct ReferenceRead<'a> { + picture: ReadPicture<'a>, + support: ReadSupport, +} + +enum ReadPicture<'a> { + Complete(&'a RetainedPicture), + Building(RwLockReadGuard<'a, Option>), +} + +pub(super) struct MotionProbe { + pub(super) motion: Option, + pub(super) reference: Option, +} + +impl<'a> ReferenceRead<'a> { + pub(crate) fn new(picture: &'a RetainedPicture, support: ReadSupport) -> Self { + Self { + picture: ReadPicture::Complete(picture), + support, + } + } + + fn guarded( + picture: RwLockReadGuard<'a, Option>, + support: ReadSupport, + ) -> Self { + Self { + picture: ReadPicture::Building(picture), + support, + } + } + + fn picture(&self) -> Result<&RetainedPicture> { + match &self.picture { + ReadPicture::Complete(picture) => Ok(picture), + ReadPicture::Building(picture) => picture + .as_ref() + .ok_or(Error::Internal("reference_row_storage")), + } + } + + pub(crate) fn plane(&self) -> Result> { + let ReadSupport::Samples { + component, + last_row, + } = self.support + else { + return Err(Error::Internal("reference_read_kind")); + }; + let scale = match component { + 0 => 1, + 1 | 2 => 2, + _ => return Err(Error::InvalidData("inter_reference_plane")), + }; + let picture = self.picture()?; + let width = picture.info.width as usize / scale; + let height = picture.info.height as usize / scale; + let rows = last_row + .checked_add(1) + .filter(|rows| *rows <= height) + .ok_or(Error::InvalidData("inter_reference_rows"))?; + let length = rows.checked_mul(width).ok_or(Error::Limit("row_address"))?; + let data = picture + .planes + .get(component) + .and_then(|plane| plane.get(..length)) + .ok_or(Error::InvalidData("inter_reference_plane"))?; + Ok(coreav_kernels_hevc::RefPlane::prefix( + data, width, width, height, rows, + )?) + } + + pub(super) fn motion(&self, x: usize, y: usize) -> Result { + let ReadSupport::Motion { row } = self.support else { + return Err(Error::Internal("reference_read_kind")); + }; + let picture = self.picture()?; + if x >= picture.info.width as usize || y >= picture.info.height as usize || y > row { + return Err(Error::InvalidData("inter_collocated_position")); + } + let Cell { motion, .. } = *picture + .cells + .get((y / 4) * (picture.info.width as usize / 4) + x / 4) + .ok_or(Error::InvalidData("inter_collocated_cell"))?; + Ok(MotionProbe { + motion, + reference: picture.reference, + }) + } +} + +#[derive(Clone, Copy)] +pub(super) struct SampleRows<'a> { + pub(super) origin: usize, + pub(super) above: Option<&'a [Vec; 3]>, + pub(super) above_y: usize, +} + +impl SampleRows<'_> { + pub(super) fn cell_index(self, frame: &Frame, x: usize, y: usize) -> Result { + if self.origin % 4 != 0 { + return Err(Error::InvalidData("row_origin")); + } + let offset = (self.origin / 4) + .checked_mul(frame.info.width as usize / 4) + .ok_or(Error::Limit("row_address"))?; + frame + .cell_index(x, y)? + .checked_sub(offset) + .ok_or(Error::InvalidData("intra_cell_address")) + } + + pub(super) fn cell(self, frame: &Frame, x: usize, y: usize) -> Result<&Cell> { + frame + .cells + .get(self.cell_index(frame, x, y)?) + .ok_or(Error::InvalidData("intra_cell_address")) + } + + pub(super) const fn full() -> Self { + Self { + origin: 0, + above: None, + above_y: 0, + } + } + + pub(super) fn offset(self, component: usize, width: usize) -> Result { + let scale = match component { + 0 => 1, + 1 | 2 => 2, + _ => return Err(Error::InvalidData("row_component")), + }; + if self.origin % 4 != 0 { + return Err(Error::InvalidData("row_origin")); + } + (self.origin / scale) + .checked_mul(width) + .ok_or(Error::Limit("row_address")) + } + + pub(super) fn sample( + self, + planes: &[Vec; 3], + component: usize, + width: usize, + x: usize, + y: usize, + ) -> Result { + let offset = self.offset(component, width)?; + let scale = if component == 0 { 1 } else { 2 }; + if let Some(above) = self.above { + if y.checked_add(1) == Some(self.above_y / scale) { + return above + .get(component) + .and_then(|plane| plane.get(x)) + .copied() + .ok_or(Error::InvalidData("row_intra_border")); + } + } + y.checked_mul(width) + .and_then(|n| n.checked_add(x)) + .and_then(|n| n.checked_sub(offset)) + .and_then(|n| planes.get(component).and_then(|plane| plane.get(n))) + .copied() + .ok_or(Error::InvalidData("intra_reference_sample")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reference_capabilities_separate_motion_from_filtered_component_prefixes() -> Result<()> { + let info = crate::PictureInfo { + width: 32, + height: 32, + visible_width: 32, + visible_height: 32, + bit_depth: 8, + profile_idc: 1, + ctb_log2: 4, + tiles: false, + wpp: false, + conformance_window: [0; 4], + vui: Default::default(), + }; + let picture = RetainedPicture::for_test(0, info, None); + let motion = ReferenceRead::new(&picture, ReadSupport::Motion { row: 15 }); + assert!(motion.motion(0, 15)?.motion.is_none()); + assert!(matches!( + motion.motion(0, 16), + Err(Error::InvalidData("inter_collocated_position")) + )); + assert!(matches!( + motion.plane(), + Err(Error::Internal("reference_read_kind")) + )); + let samples = ReferenceRead::new( + &picture, + ReadSupport::Samples { + component: 0, + last_row: 15, + }, + ); + assert!(matches!( + samples.motion(0, 0), + Err(Error::Internal("reference_read_kind")) + )); + let plane = samples.plane()?; + let mut output = [0; 16]; + let config = coreav_kernels_hevc::InterPrediction { + depth: coreav_kernels_hevc::BitDepth::new(8)?, + component: coreav_kernels_hevc::Component::Luma, + x: 0, + y: 12, + motion: coreav_kernels_hevc::MotionVector { x: 0, y: 0 }, + }; + coreav_kernels_hevc::interpolate_inter( + config, + plane, + coreav_kernels_hevc::RectMut::new(&mut output, 4, 4, 4)?, + )?; + assert!( + coreav_kernels_hevc::interpolate_inter( + coreav_kernels_hevc::InterPrediction { y: 13, ..config }, + plane, + coreav_kernels_hevc::RectMut::new(&mut output, 4, 4, 4)? + ) + .is_err() + ); + assert!( + ReferenceRead::new( + &picture, + ReadSupport::Samples { + component: 0, + last_row: usize::MAX + } + ) + .plane() + .is_err() + ); + Ok(()) + } +} diff --git a/crates/coreav-hevc/src/reconstruction/rows/store.rs b/crates/coreav-hevc/src/reconstruction/rows/store.rs new file mode 100644 index 000000000..2581b4e86 --- /dev/null +++ b/crates/coreav-hevc/src/reconstruction/rows/store.rs @@ -0,0 +1,809 @@ +use super::{ReadSupport, ReferenceRead}; +use crate::reconstruction::{Cell, RetainedPicture}; +use crate::{Error, PictureInfo, Result, dpb::ReferenceIdentity}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock, RwLock}; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Outcome { + Building, + Finishing, + Complete, + Failed, +} + +struct Progress { + motion: usize, + filtered: usize, + outcome: Outcome, + #[cfg(test)] + waits: usize, +} + +pub(crate) struct RowStore { + identity: ReferenceIdentity, + generation: u64, + info: PictureInfo, + deadline: Option, + backing_bytes: usize, + completed: OnceLock, + storage: RwLock>, + progress: Mutex, + changed: Condvar, + #[cfg(test)] + observing: Condvar, +} + +pub(crate) struct RowPublisher { + store: Arc, +} + +impl Drop for RowPublisher { + fn drop(&mut self) { + self.store.fail(); + } +} + +impl RowPublisher { + #[cfg(test)] + pub(crate) fn wait_for_reader(&self) -> Result<()> { + let progress = self.store.progress()?; + let (progress, _) = self + .store + .observing + .wait_timeout_while(progress, std::time::Duration::from_secs(3), |progress| { + progress.waits == 0 + }) + .map_err(|_| Error::Internal("reference_row_poison"))?; + if progress.waits == 0 { + return Err(Error::Internal("dependent_row_job_did_not_start")); + } + Ok(()) + } + + pub(crate) fn bytes(&self) -> Result { + self.store.bytes() + } + + pub(crate) fn fail(&self) { + self.store.fail(); + } + + pub(crate) fn hash( + &mut self, + producer: &mut crate::reconstruction::raster::Producer, + hash: crate::SeiInfo, + ) -> Result<()> { + let result = (|| { + let storage = self + .store + .storage + .read() + .map_err(|_| Error::Internal("reference_row_poison"))?; + let picture = storage + .as_ref() + .ok_or(Error::Internal("reference_row_storage"))?; + producer.hash( + hash, + picture.planes.each_ref().map(|plane| plane.as_slice()), + ) + })(); + if result.is_err() { + self.store.fail(); + } + result + } + + pub(crate) fn complete( + self, + completion: crate::reconstruction::raster::Completion, + timestamp: Option, + origin: Option, + ) -> Result<()> { + self.finish(|picture| completion.retain(picture.planes, picture.cells, timestamp, origin)) + } + + pub(super) fn motion(&mut self, rows: core::ops::Range, cells: &[Cell]) -> Result<()> { + let result = self.store.motion(rows, cells); + if result.is_err() { + self.store.fail(); + } + result + } + + pub(super) fn samples( + &mut self, + origin: usize, + dirty: core::ops::Range, + filtered: usize, + planes: &[Vec; 3], + ) -> Result<()> { + let result = self.store.samples(origin, dirty, filtered, planes); + if result.is_err() { + self.store.fail(); + } + result + } + + pub(super) fn finish( + self, + finish: impl FnOnce(RetainedPicture) -> Result, + ) -> Result<()> { + let mut storage = match self.store.storage.write() { + Ok(storage) => storage, + Err(error) => { + drop(error.into_inner()); + return Err(Error::Internal("reference_row_poison")); + } + }; + let mut progress = self.store.progress()?; + if progress.outcome != Outcome::Building + || progress.motion != self.store.info.height as usize + || progress.filtered != progress.motion + { + return Err(Error::InvalidData("selected_picture_incomplete")); + } + progress.outcome = Outcome::Finishing; + let backing = storage + .take() + .ok_or(Error::Internal("reference_row_storage"))?; + drop(progress); + drop(storage); + let picture = finish(backing)?; + if picture.index != self.store.identity.index + || picture.info != self.store.info + || picture.heap_bytes()? != self.store.backing_bytes + { + return Err(Error::Internal("reference_row_identity")); + } + self.store + .completed + .set(picture) + .map_err(|_| Error::Internal("reference_row_storage"))?; + let mut progress = self.store.progress()?; + if progress.outcome != Outcome::Finishing { + return Err(Error::Internal("reference_row_failed")); + } + progress.outcome = Outcome::Complete; + self.store.changed.notify_all(); + Ok(()) + } +} + +impl RowStore { + pub(crate) const fn record_bytes() -> usize { + core::mem::size_of::() + 2 * core::mem::size_of::() + } + pub(crate) fn generation(&self) -> u64 { + self.generation + } + + pub(crate) fn take_picture(&mut self) -> Option { + (self.progress.get_mut().ok()?.outcome == Outcome::Complete) + .then(|| self.completed.take()) + .flatten() + } + pub(crate) fn new( + identity: ReferenceIdentity, + generation: u64, + backing: RetainedPicture, + deadline: Option, + memory: &mut crate::reconstruction::Memory<'_>, + ) -> Result<(Arc, RowPublisher)> { + if identity.index != backing.index + || backing + .reference + .is_some_and(|state| state.poc != identity.poc) + { + return Err(Error::Internal("reference_row_identity")); + } + let width = backing.info.width as usize; + let height = backing.info.height as usize; + let samples = width + .checked_mul(height) + .ok_or(Error::Limit("picture_samples"))?; + if backing.planes[0].len() != samples + || backing.planes[1].len() != samples / 4 + || backing.planes[2].len() != samples / 4 + || backing.cells.len() != samples / 16 + { + return Err(Error::InvalidData("reference_row_storage")); + } + memory.charge(Self::record_bytes())?; + let store = Arc::new(Self { + identity, + generation, + info: backing.info, + deadline, + backing_bytes: backing.heap_bytes()?, + completed: OnceLock::new(), + storage: RwLock::new(Some(backing)), + progress: Mutex::new(Progress { + motion: 0, + filtered: 0, + outcome: Outcome::Building, + #[cfg(test)] + waits: 0, + }), + changed: Condvar::new(), + #[cfg(test)] + observing: Condvar::new(), + }); + Ok((Arc::clone(&store), RowPublisher { store })) + } + + pub(crate) fn bytes(&self) -> Result { + Self::record_bytes() + .checked_add(self.backing_bytes) + .ok_or(Error::Limit("reconstruction_memory")) + } + + pub(crate) fn picture(&self) -> Option<&RetainedPicture> { + (self.progress().ok()?.outcome == Outcome::Complete) + .then(|| self.completed.get()) + .flatten() + } + + fn progress(&self) -> Result> { + match self.progress.lock() { + Ok(progress) => Ok(progress), + Err(error) => { + error.into_inner().outcome = Outcome::Failed; + self.changed.notify_all(); + Err(Error::Internal("reference_row_poison")) + } + } + } + + pub(crate) fn fail(&self) { + let mut progress = self + .progress + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if progress.outcome != Outcome::Complete { + progress.outcome = Outcome::Failed; + } + self.changed.notify_all(); + } + + fn check_identity(&self, identity: ReferenceIdentity, generation: u64) -> Result<()> { + if self.identity.index != identity.index + || self.identity.poc != identity.poc + || self.generation != generation + { + return Err(Error::InvalidData("reference_row_identity")); + } + Ok(()) + } + + pub(crate) fn info(&self, identity: ReferenceIdentity, generation: u64) -> Result { + self.check_identity(identity, generation)?; + if self.progress()?.outcome == Outcome::Failed { + return Err(Error::Internal("reference_row_failed")); + } + Ok(self.info) + } + + fn required(&self, support: ReadSupport) -> Result { + let (last, scale) = match support { + ReadSupport::Motion { row } => (row, 1), + ReadSupport::Samples { + component: 0, + last_row, + } => (last_row, 1), + ReadSupport::Samples { + component: 1 | 2, + last_row, + } => (last_row, 2), + _ => return Err(Error::InvalidData("inter_reference_plane")), + }; + last.checked_add(1) + .and_then(|rows| rows.checked_mul(scale)) + .filter(|rows| *rows <= self.info.height as usize) + .ok_or(Error::InvalidData("inter_reference_rows")) + } + + fn ready(progress: &Progress, support: ReadSupport, required: usize) -> bool { + progress.outcome == Outcome::Complete + || progress.outcome == Outcome::Building + && match support { + ReadSupport::Motion { .. } => progress.motion >= required, + ReadSupport::Samples { .. } => progress.filtered >= required, + } + } + + pub(crate) fn read( + &self, + identity: ReferenceIdentity, + generation: u64, + support: ReadSupport, + ) -> Result> { + let required = self.required(support)?; + loop { + self.check_identity(identity, generation)?; + let mut progress = self.progress()?; + while progress.outcome != Outcome::Failed && !Self::ready(&progress, support, required) + { + #[cfg(test)] + { + progress.waits += 1; + self.observing.notify_all(); + } + progress = if let Some(deadline) = self.deadline { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .ok_or(Error::Limit("decode_deadline"))?; + match self.changed.wait_timeout(progress, remaining) { + Ok((progress, _)) => progress, + Err(error) => { + error.into_inner().0.outcome = Outcome::Failed; + self.changed.notify_all(); + return Err(Error::Internal("reference_row_poison")); + } + } + } else { + match self.changed.wait(progress) { + Ok(progress) => progress, + Err(error) => { + error.into_inner().outcome = Outcome::Failed; + self.changed.notify_all(); + return Err(Error::Internal("reference_row_poison")); + } + } + }; + } + if progress.outcome == Outcome::Failed { + return Err(Error::Internal("reference_row_failed")); + } + if progress.outcome == Outcome::Complete { + return self + .completed + .get() + .map(|picture| ReferenceRead::new(picture, support)) + .ok_or(Error::Internal("reference_row_storage")); + } + drop(progress); + let storage = match self.storage.read() { + Ok(storage) => storage, + Err(error) => { + drop(error.into_inner()); + self.fail(); + return Err(Error::Internal("reference_row_poison")); + } + }; + let progress = self.progress()?; + self.check_identity(identity, generation)?; + if progress.outcome == Outcome::Failed { + return Err(Error::Internal("reference_row_failed")); + } + if Self::ready(&progress, support, required) && progress.outcome == Outcome::Building { + if storage.is_none() { + return Err(Error::Internal("reference_row_storage")); + } + drop(progress); + return Ok(ReferenceRead::guarded(storage, support)); + } + } + } + + fn commit( + &self, + work: impl FnOnce(&mut RetainedPicture, &Progress) -> Result<(usize, usize)>, + ) -> Result<()> { + let mut storage = match self.storage.write() { + Ok(storage) => storage, + Err(error) => { + drop(error.into_inner()); + self.fail(); + return Err(Error::Internal("reference_row_poison")); + } + }; + let progress = self.progress()?; + if progress.outcome != Outcome::Building { + return Err(Error::Internal("reference_row_failed")); + } + let frontiers = work( + storage + .as_mut() + .ok_or(Error::Internal("reference_row_storage"))?, + &progress, + )?; + drop(progress); + drop(storage); + let mut progress = self.progress()?; + if progress.outcome != Outcome::Building { + return Err(Error::Internal("reference_row_failed")); + } + progress.motion = frontiers.0; + progress.filtered = frontiers.1; + self.changed.notify_all(); + Ok(()) + } + + fn motion(&self, rows: core::ops::Range, cells: &[Cell]) -> Result<()> { + self.commit(|picture, progress| { + let width = self.info.width as usize / 4; + let ctb = 1usize << self.info.ctb_log2; + if rows.start != progress.motion + || rows.end <= rows.start + || rows.end != (rows.start + ctb).min(self.info.height as usize) + || cells.len() != (rows.end - rows.start) / 4 * width + || cells.iter().any(|cell| cell.reconstructed != 7) + { + return Err(Error::InvalidData("reference_motion_commit")); + } + picture + .cells + .get_mut(rows.start / 4 * width..rows.end / 4 * width) + .ok_or(Error::InvalidData("reference_motion_commit"))? + .copy_from_slice(cells); + Ok((rows.end, progress.filtered)) + }) + } + + fn samples( + &self, + origin: usize, + dirty: core::ops::Range, + filtered: usize, + planes: &[Vec; 3], + ) -> Result<()> { + self.commit(|picture, progress| { + let ctb = 1usize << self.info.ctb_log2; + if origin % 4 != 0 + || dirty.start != progress.filtered + || dirty.start < origin + || dirty.end != progress.motion + || filtered != (progress.filtered + ctb).min(self.info.height as usize) + || filtered > dirty.end + || filtered <= progress.filtered + { + return Err(Error::InvalidData("reference_sample_commit")); + } + for component in 0..3 { + let scale = if component == 0 { 1 } else { 2 }; + let width = self.info.width as usize / scale; + planes + .get(component) + .and_then(|plane| { + plane.get( + (dirty.start - origin) / scale * width + ..(dirty.end - origin) / scale * width, + ) + }) + .ok_or(Error::InvalidData("reference_sample_commit"))?; + } + for (component, (source, destination)) in + planes.iter().zip(&mut picture.planes).enumerate() + { + let scale = if component == 0 { 1 } else { 2 }; + let width = self.info.width as usize / scale; + destination + .get_mut(dirty.start / scale * width..dirty.end / scale * width) + .ok_or(Error::InvalidData("reference_sample_commit"))? + .copy_from_slice( + source + .get( + (dirty.start - origin) / scale * width + ..(dirty.end - origin) / scale * width, + ) + .ok_or(Error::InvalidData("reference_sample_commit"))?, + ); + } + Ok((progress.motion, filtered)) + }) + } +} + +impl crate::reconstruction::raster::Sink for RowPublisher { + fn motion(&mut self, band: crate::reconstruction::raster::MotionBand<'_>) -> Result<()> { + self.motion(band.rows, band.cells) + } + + fn samples(&mut self, band: crate::reconstruction::raster::SampleBand<'_>) -> Result<()> { + self.samples(band.origin, band.dirty, band.filtered_through, band.planes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::reconstruction::{Memory, ReferenceState}; + use std::sync::mpsc; + use std::time::{Duration, Instant}; + + const IDENTITY: ReferenceIdentity = ReferenceIdentity { + index: 0, + poc: 0, + long_term: false, + }; + const GENERATION: u64 = 7; + + fn fixture() -> Result<(Arc, RowPublisher, RetainedPicture)> { + fixture_deadline(Some(Instant::now() + Duration::from_secs(3))) + } + + fn fixture_deadline( + deadline: Option, + ) -> Result<(Arc, RowPublisher, RetainedPicture)> { + let info = PictureInfo { + width: 32, + height: 32, + visible_width: 32, + visible_height: 32, + bit_depth: 8, + profile_idc: 1, + ctb_log2: 4, + tiles: false, + wpp: false, + conformance_window: [0; 4], + vui: Default::default(), + }; + let mut backing = RetainedPicture::for_test(0, info, None); + backing.reference = Some(ReferenceState { + poc: 0, + references: Default::default(), + }); + for plane in &mut backing.planes { + plane.fill(u16::MAX); + } + for cell in &mut backing.cells { + cell.reconstructed = 0; + } + let mut memory = Memory::new(backing.heap_bytes()?, usize::MAX)?; + let (store, publisher) = + RowStore::new(IDENTITY, GENERATION, backing, deadline, &mut memory)?; + assert_eq!(store.bytes()?, memory.peak()); + Ok((store, publisher, RetainedPicture::for_test(0, info, None))) + } + + fn await_waiters(store: &RowStore, count: usize) { + let progress = store.progress.lock().unwrap(); + let (progress, _) = store + .observing + .wait_timeout_while(progress, Duration::from_secs(1), |progress| { + progress.waits < count + }) + .unwrap(); + assert!(progress.waits >= count); + } + + #[test] + fn scoped_row_reads_separate_motion_and_chroma_and_pin_the_committed_prefix() -> Result<()> { + let (store, mut publisher, source) = fixture()?; + publisher.motion(0..16, &source.cells[..32 * 16 / 16])?; + publisher.motion(16..32, &source.cells[32 * 16 / 16..])?; + publisher.samples(0, 0..32, 16, &source.planes)?; + assert!( + store + .read(IDENTITY, GENERATION, ReadSupport::Motion { row: 17 })? + .motion(0, 17)? + .motion + .is_none() + ); + let prefix = store.read( + IDENTITY, + GENERATION, + ReadSupport::Samples { + component: 0, + last_row: 15, + }, + )?; + let pointer = store.storage.read().unwrap().as_ref().unwrap().planes[0].as_ptr(); + let result = std::thread::scope(|scope| -> Result { + let store = store.as_ref(); + let (ready, received) = mpsc::channel(); + let reader = scope.spawn(move || { + let result = store + .read( + IDENTITY, + GENERATION, + ReadSupport::Samples { + component: 1, + last_row: 11, + }, + ) + .and_then(|read| read.plane().map(|_| ())); + ready.send(result).unwrap(); + }); + await_waiters(store, 1); + assert!(received.try_recv().is_err()); + let window: [Vec; 3] = core::array::from_fn(|component| { + let scale = if component == 0 { 1 } else { 2 }; + source.planes[component][12 / scale * (32 / scale)..].to_vec() + }); + let (enter, entered) = mpsc::channel(); + let (committed, committed_rx) = mpsc::channel(); + let writer = scope.spawn(move || -> Result { + enter.send(()).unwrap(); + publisher.samples(12, 16..32, 32, &window)?; + committed.send(()).unwrap(); + Ok(publisher) + }); + entered.recv_timeout(Duration::from_secs(1)).unwrap(); + assert!(matches!( + committed_rx.recv_timeout(Duration::from_millis(30)), + Err(mpsc::RecvTimeoutError::Timeout) + )); + drop(prefix); + committed_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + received.recv_timeout(Duration::from_secs(1)).unwrap()?; + reader.join().unwrap(); + writer.join().unwrap() + })?; + result.finish(Ok)?; + let picture = store.picture().unwrap(); + assert_eq!(picture.planes, source.planes); + assert_eq!(picture.planes[0].as_ptr(), pointer); + assert!( + store + .read( + IDENTITY, + GENERATION, + ReadSupport::Samples { + component: 2, + last_row: 15 + } + )? + .plane() + .is_ok() + ); + Ok(()) + } + + #[test] + fn completion_moves_the_owner_outside_the_lock_and_wakes_transition_readers() -> Result<()> { + let (store, mut publisher, source) = fixture()?; + publisher.motion(0..16, &source.cells[..32])?; + publisher.motion(16..32, &source.cells[32..])?; + publisher.samples(0, 0..32, 16, &source.planes)?; + publisher.samples(0, 16..32, 32, &source.planes)?; + let pointer = store.storage.read().unwrap().as_ref().unwrap().planes[0].as_ptr() as usize; + std::thread::scope(|scope| -> Result<()> { + let (entered, entering) = mpsc::channel(); + let (release, released) = mpsc::channel(); + let writer = scope.spawn(move || { + publisher.finish(|picture| { + entered.send(()).unwrap(); + released.recv_timeout(Duration::from_secs(1)).unwrap(); + Ok(picture) + }) + }); + entering.recv_timeout(Duration::from_secs(1)).unwrap(); + assert!(store.storage.try_read().unwrap().is_none()); + assert!(store.picture().is_none()); + let reader = scope.spawn(|| { + store + .read(IDENTITY, GENERATION, ReadSupport::Motion { row: 0 }) + .map(|_| ()) + }); + await_waiters(&store, 1); + release.send(()).unwrap(); + writer.join().unwrap()?; + reader.join().unwrap()?; + Ok(()) + })?; + assert_eq!( + store.picture().unwrap().planes[0].as_ptr() as usize, + pointer + ); + Ok(()) + } + + #[test] + fn dropping_the_only_publisher_wakes_motion_and_sample_waiters() -> Result<()> { + let (store, publisher, _) = fixture()?; + std::thread::scope(|scope| { + let store = store.as_ref(); + let (done, receiver) = mpsc::channel(); + let first = done.clone(); + scope.spawn(move || { + first + .send( + store + .read(IDENTITY, GENERATION, ReadSupport::Motion { row: 31 }) + .map(|_| ()), + ) + .unwrap(); + }); + scope.spawn(move || { + done.send( + store + .read( + IDENTITY, + GENERATION, + ReadSupport::Samples { + component: 0, + last_row: 31, + }, + ) + .map(|_| ()), + ) + .unwrap(); + }); + await_waiters(store, 2); + drop(publisher); + for _ in 0..2 { + assert!(matches!( + receiver.recv_timeout(Duration::from_secs(1)).unwrap(), + Err(Error::Internal("reference_row_failed")) + )); + } + }); + Ok(()) + } + + #[test] + fn invalid_commits_latch_failure_without_overwriting_published_samples() -> Result<()> { + let (store, mut publisher, source) = fixture()?; + assert!(matches!( + store.info(IDENTITY, GENERATION + 1), + Err(Error::InvalidData("reference_row_identity")) + )); + assert!(matches!( + store.read( + IDENTITY, + GENERATION, + ReadSupport::Samples { + component: 3, + last_row: 0 + } + ), + Err(Error::InvalidData("inter_reference_plane")) + )); + publisher.motion(0..16, &source.cells[..32])?; + publisher.motion(16..32, &source.cells[32..])?; + publisher.samples(0, 0..32, 16, &source.planes)?; + assert!(matches!( + publisher.samples(0, 0..32, 32, &source.planes), + Err(Error::InvalidData("reference_sample_commit")) + )); + assert!(matches!( + store.read(IDENTITY, GENERATION, ReadSupport::Motion { row: 0 }), + Err(Error::Internal("reference_row_failed")) + )); + assert_eq!( + store.storage.read().unwrap().as_ref().unwrap().planes, + source.planes + ); + Ok(()) + } + + #[test] + fn row_wait_deadlines_and_storage_poison_report_their_original_failures() -> Result<()> { + let (store, _publisher, _) = fixture_deadline(Some(Instant::now()))?; + assert!(matches!( + store.read(IDENTITY, GENERATION, ReadSupport::Motion { row: 0 }), + Err(Error::Limit("decode_deadline")) + )); + let (store, mut publisher, source) = fixture()?; + publisher.motion(0..16, &source.cells[..32])?; + publisher.samples(0, 0..16, 16, &source.planes)?; + std::thread::scope(|scope| { + assert!( + scope + .spawn(|| { + let _guard = store.storage.write().unwrap(); + panic!("row storage poison"); + }) + .join() + .is_err() + ); + }); + assert!(matches!( + store.read(IDENTITY, GENERATION, ReadSupport::Motion { row: 0 }), + Err(Error::Internal("reference_row_poison")) + )); + assert!(matches!( + store.read( + IDENTITY, + GENERATION, + ReadSupport::Samples { + component: 0, + last_row: 0 + } + ), + Err(Error::Internal("reference_row_failed")) + )); + Ok(()) + } +} diff --git a/crates/coreav-hevc/src/reconstruction/syntax.rs b/crates/coreav-hevc/src/reconstruction/syntax.rs index 0bfc301f1..277d08d0e 100644 --- a/crates/coreav-hevc/src/reconstruction/syntax.rs +++ b/crates/coreav-hevc/src/reconstruction/syntax.rs @@ -17,6 +17,7 @@ use coreav_kernels_hevc::{ }; #[derive(Clone, Copy)] +#[cfg_attr(test, derive(Debug, PartialEq, Eq))] pub(super) struct Contexts { split_cu: [Context; 3], bypass: Context, @@ -34,6 +35,7 @@ pub(super) struct Contexts { } #[derive(Clone, Copy)] +#[cfg_attr(test, derive(Debug, PartialEq, Eq))] struct InterContexts { skip: [Context; 3], pred_mode: Context, @@ -114,6 +116,7 @@ pub(super) struct Checkpoint { } #[derive(Clone, Copy)] +#[cfg_attr(test, derive(Debug, PartialEq, Eq))] pub(super) struct Qp { current: i32, predictor: i32, @@ -572,7 +575,7 @@ impl Frame { pub(crate) fn decode_full( &mut self, input: SliceInput<'_>, - dpb: &crate::dpb::Dpb, + dpb: &dyn crate::dpb::ReferenceProvider, remaining: &mut u64, fixed: usize, limit: usize, @@ -595,13 +598,47 @@ impl Frame { fn decode_common( &mut self, input: SliceInput<'_>, - dpb: Option<&crate::dpb::Dpb>, + dpb: Option<&dyn crate::dpb::ReferenceProvider>, remaining: &mut u64, fixed: usize, limit: usize, threads: usize, mut cache: Option<&mut super::Cache>, ) -> Result { + let (ranges, fixed, prior_peak) = self.prepare_slice( + &input, + fixed, + limit, + match &mut cache { + Some(cache) => Some(&mut **cache), + None => None, + }, + )?; + if let Some(peak) = self.decode_wpp( + &input, + dpb.and_then(|provider| provider.dpb()), + remaining, + &ranges, + fixed, + limit, + threads, + cache, + )? { + return Ok(peak.max(prior_peak)); + } + self.traverse_slice(&input, &ranges, remaining, |frame, slice, cabac| { + frame.decode_ctu_references(&input, dpb, slice, cabac) + })?; + Ok(prior_peak) + } + + pub(super) fn prepare_slice( + &self, + input: &SliceInput<'_>, + fixed: usize, + limit: usize, + reclaim: Option<&mut dyn super::ReclaimMemory>, + ) -> Result<(Vec>, usize, usize)> { if self.index != input.index || self.info != input.picture || self.hash_seen @@ -623,15 +660,8 @@ impl Frame { if count > (total_ctus - self.next_ctu) as usize { return Err(Error::InvalidData("intra_substream_count")); } - let (ranges, fixed, prior_peak) = { - let mut memory = Memory::with_reclaim( - fixed, - limit, - match &mut cache { - Some(cache) => Some(&mut **cache), - None => None, - }, - )?; + let result = { + let mut memory = Memory::with_reclaim(fixed, limit, reclaim)?; memory.charge(self.heap_bytes()?)?; memory.charge(input.scratch_bytes)?; memory.charge( @@ -652,11 +682,17 @@ impl Frame { )?; (ranges, memory.used, memory.peak()) }; - if let Some(peak) = self.decode_wpp( - &input, dpb, remaining, &ranges, fixed, limit, threads, cache, - )? { - return Ok(peak.max(prior_peak)); - } + Ok(result) + } + + pub(super) fn traverse_slice( + &mut self, + input: &SliceInput<'_>, + ranges: &[Range], + remaining: &mut u64, + mut decode: impl FnMut(&mut Self, u32, &mut Cabac<'_, '_>) -> Result, + ) -> Result<()> { + let total_ctus = input.layout.width * input.layout.height; if !input.header.dependent { self.contexts = Contexts::new(input.syntax.qp_y, input.syntax.initialization); self.qp = Qp::new(input.syntax.qp_y); @@ -677,14 +713,14 @@ impl Frame { .ok_or(Error::InvalidData("intra_substream_range"))?; let mut cabac = Cabac::new(bytes, remaining)?; loop { - let location = self.decode_ctu(&input, dpb, slice, &mut cabac)?; + let location = decode(self, slice, &mut cabac)?; if cabac.terminate()? { if range_index + 1 != ranges.len() { return Err(Error::InvalidData("intra_unconsumed_substreams")); } end_range(&mut cabac, bytes.len(), true)?; self.segment_closed = true; - return Ok(prior_peak); + return Ok(()); } if self.next_ctu == total_ctus { return Err(Error::InvalidData("intra_picture_termination")); @@ -715,6 +751,32 @@ impl Frame { dpb: Option<&crate::dpb::Dpb>, slice: u32, cabac: &mut Cabac<'_, '_>, + ) -> Result { + self.decode_ctu_references( + input, + dpb.map(|dpb| dpb as &dyn crate::dpb::ReferenceProvider), + slice, + cabac, + ) + } + + fn decode_ctu_references( + &mut self, + input: &SliceInput<'_>, + dpb: Option<&dyn crate::dpb::ReferenceProvider>, + slice: u32, + cabac: &mut Cabac<'_, '_>, + ) -> Result { + self.decode_ctu_rows(input, dpb, slice, cabac, super::rows::SampleRows::full()) + } + + pub(super) fn decode_ctu_rows( + &mut self, + input: &SliceInput<'_>, + dpb: Option<&dyn crate::dpb::ReferenceProvider>, + slice: u32, + cabac: &mut Cabac<'_, '_>, + rows: super::rows::SampleRows<'_>, ) -> Result { self.check_deadline()?; let location = input.layout.at_tile_scan(self.next_ctu)?; @@ -722,9 +784,10 @@ impl Frame { if let Some(hook) = &self.test_hook { hook.compute(location.y, location.x)?; } - self.start_ctu(input, location, slice)?; + self.start_ctu(input, location, slice, rows)?; self.read_sao(input, location, slice, cabac)?; Walker { + rows, frame: self, input, location, @@ -753,13 +816,20 @@ impl Frame { Ok((y / 4) * (self.info.width as usize / 4) + x / 4) } + #[cfg(test)] pub(super) fn cell(&self, x: usize, y: usize) -> Result<&Cell> { self.cells .get(self.cell_index(x, y)?) .ok_or(Error::InvalidData("intra_cell_address")) } - fn start_ctu(&mut self, input: &SliceInput<'_>, location: Location, slice: u32) -> Result<()> { + fn start_ctu( + &mut self, + input: &SliceInput<'_>, + location: Location, + slice: u32, + rows: super::rows::SampleRows<'_>, + ) -> Result<()> { let [tile_x, tile_y] = location.tile_origin; if location.x == tile_x && location.y == tile_y { self.contexts = Contexts::new(input.syntax.qp_y, input.syntax.initialization); @@ -773,8 +843,9 @@ impl Frame { .tile_size .first() .ok_or(Error::InvalidData("tile_grid"))? - && self + && rows .cell( + self, ((location.x + 1) << input.sps.ctb_log2) as usize, ((location.y - 1) << input.sps.ctb_log2) as usize, )? @@ -816,14 +887,23 @@ pub(super) struct Walker<'f, 'i, 'data> { pub(super) input: &'i SliceInput<'data>, pub(super) location: Location, pub(super) slice: u32, - pub(super) dpb: Option<&'i crate::dpb::Dpb>, + pub(super) dpb: Option<&'i dyn crate::dpb::ReferenceProvider>, + pub(super) rows: super::rows::SampleRows<'i>, } impl Walker<'_, '_, '_> { + pub(super) fn cell_index(&self, x: usize, y: usize) -> Result { + self.rows.cell_index(self.frame, x, y) + } + + pub(super) fn cell(&self, x: usize, y: usize) -> Result<&Cell> { + self.rows.cell(self.frame, x, y) + } + fn protect(&mut self, area: Area) -> Result<()> { for y in (area.y..area.y + area.side()).step_by(4) { for x in (area.x..area.x + area.side()).step_by(4) { - let index = self.frame.cell_index(x, y)?; + let index = self.cell_index(x, y)?; self.frame .cells .get_mut(index) @@ -841,7 +921,11 @@ impl Walker<'_, '_, '_> { if x >= self.frame.info.width as usize || y >= self.frame.info.height as usize { return Ok(None); } - let cell = *self.frame.cell(x, y)?; + let cell = match self.frame.cells.get(self.cell_index(x, y)?) { + Some(cell) => *cell, + None if y >> self.input.sps.ctb_log2 > self.location.y as usize => return Ok(None), + None => return Err(Error::InvalidData("intra_cell_address")), + }; if cell.slice != self.slice || cell.mode.is_none() && cell.motion.is_none() && cell.reconstructed == 0 { @@ -930,7 +1014,7 @@ impl Walker<'_, '_, '_> { fn initialize_cells(&mut self, area: Area, depth: u8) -> Result<()> { for y in (area.y..area.y + area.side()).step_by(4) { for x in (area.x..area.x + area.side()).step_by(4) { - let index = self.frame.cell_index(x, y)?; + let index = self.cell_index(x, y)?; let cell = self .frame .cells @@ -960,7 +1044,7 @@ impl Walker<'_, '_, '_> { fn set_mode(&mut self, area: Area, mode: IntraMode) -> Result<()> { for y in (area.y..area.y + area.side()).step_by(4) { for x in (area.x..area.x + area.side()).step_by(4) { - let index = self.frame.cell_index(x, y)?; + let index = self.cell_index(x, y)?; self.frame .cells .get_mut(index) @@ -1072,7 +1156,6 @@ impl Walker<'_, '_, '_> { 4 }; let luma = self - .frame .cell(area.x, area.y)? .mode .ok_or(Error::InvalidData("intra_luma_mode"))? @@ -1118,7 +1201,7 @@ impl Walker<'_, '_, '_> { if skipped { for y in (area.y..area.y + area.side()).step_by(4) { for x in (area.x..area.x + area.side()).step_by(4) { - let index = self.frame.cell_index(x, y)?; + let index = self.cell_index(x, y)?; self.frame .cells .get_mut(index) @@ -1188,7 +1271,7 @@ impl Walker<'_, '_, '_> { .syntax .inter .ok_or(Error::InvalidData("inter_slice"))?; - let cell = *self.frame.cell(cu.x, cu.y)?; + let cell = *self.cell(cu.x, cu.y)?; let contexts = self .frame .contexts @@ -1269,6 +1352,7 @@ impl Walker<'_, '_, '_> { .ok_or(Error::InvalidData("intra_pcm_depth"))?; let side = area.side() / scale; let width = self.frame.info.width as usize / scale; + let origin = self.rows.offset(component, width)?; let plane = self .frame .planes @@ -1278,7 +1362,11 @@ impl Walker<'_, '_, '_> { for x in 0..side { let sample = (cabac.raw_bits(depth)? << shift) as u16; *plane - .get_mut((area.y / scale + y) * width + area.x / scale + x) + .get_mut( + ((area.y / scale + y) * width + area.x / scale + x) + .checked_sub(origin) + .ok_or(Error::InvalidData("intra_sample_address"))?, + ) .ok_or(Error::InvalidData("intra_sample_address"))? = sample; } } @@ -1336,7 +1424,7 @@ impl Walker<'_, '_, '_> { self.frame.stats.nonzero_qp_deltas += u64::from(delta != 0); for y in (area.y..area.y + area.side()).step_by(4) { for x in (area.x..area.x + area.side()).step_by(4) { - let index = self.frame.cell_index(x, y)?; + let index = self.cell_index(x, y)?; let cell = self .frame .cells @@ -1410,7 +1498,7 @@ impl Walker<'_, '_, '_> { if luma || chroma.into_iter().any(|value| value) { self.delta_qp(cabac, cu.area)?; } - let mode = self.frame.cell(area.x, area.y)?.mode; + let mode = self.cell(area.x, area.y)?.mode; if intra && mode.is_none() { return Err(Error::InvalidData("intra_luma_mode")); } @@ -1485,14 +1573,13 @@ impl Walker<'_, '_, '_> { } }; if available { - Some( - *self - .frame - .planes - .get(block.component) - .and_then(|plane| plane.get(ny * width + nx)) - .ok_or(Error::InvalidData("intra_reference_sample"))?, - ) + Some(self.rows.sample( + &self.frame.planes, + block.component, + width, + nx, + ny, + )?) } else { None } @@ -1512,7 +1599,7 @@ impl Walker<'_, '_, '_> { fn mark(&mut self, luma_area: Area, mask: u8) -> Result<()> { for y in (luma_area.y..luma_area.y + luma_area.side()).step_by(4) { for x in (luma_area.x..luma_area.x + luma_area.side()).step_by(4) { - let index = self.frame.cell_index(x, y)?; + let index = self.cell_index(x, y)?; let cell = self .frame .cells @@ -1609,13 +1696,16 @@ impl Walker<'_, '_, '_> { } else { let scale = if block.component == 0 { 1 } else { 2 }; let width = self.frame.info.width as usize / scale; + let origin = self.rows.offset(block.component, width)?; let plane = self .frame .planes .get(block.component) .ok_or(Error::InvalidData("inter_plane"))?; for y in 0..side { - let source = (block.area.y + y) * width + block.area.x; + let source = ((block.area.y + y) * width + block.area.x) + .checked_sub(origin) + .ok_or(Error::InvalidData("inter_sample_address"))?; self.frame .scratch .prediction @@ -1684,13 +1774,16 @@ impl Walker<'_, '_, '_> { )?; let scale = if block.component == 0 { 1 } else { 2 }; let width = self.frame.info.width as usize / scale; + let origin = self.rows.offset(block.component, width)?; let plane = self .frame .planes .get_mut(block.component) .ok_or(Error::InvalidData("intra_plane"))?; for y in 0..side { - let dest = (block.area.y + y) * width + block.area.x; + let dest = ((block.area.y + y) * width + block.area.x) + .checked_sub(origin) + .ok_or(Error::InvalidData("intra_sample_address"))?; plane .get_mut(dest..dest + side) .ok_or(Error::InvalidData("intra_sample_address"))? @@ -1713,7 +1806,7 @@ impl Walker<'_, '_, '_> { if block.component == 0 && block.coded { for y in (block.area.y..block.area.y + side).step_by(4) { for x in (block.area.x..block.area.x + side).step_by(4) { - let index = self.frame.cell_index(x, y)?; + let index = self.cell_index(x, y)?; self.frame .cells .get_mut(index) @@ -2117,6 +2210,7 @@ mod tests { picture: PictureInfo, } + #[derive(Clone, Copy)] struct Segment<'a> { rbsp: &'a [u8], original: &'a [u8], @@ -2305,6 +2399,582 @@ mod tests { .ok_or(Error::InvalidData("test_cell")) } + fn raster_matches_serial(inputs: &[SliceInput<'_>], budget: u64) -> Result<()> { + use super::super::raster::tests::{assert_entropy, producer as make_producer}; + let input = inputs.first().ok_or(Error::Internal("test_raster_input"))?; + let dpb = crate::dpb::Dpb::new_reconstruction(); + let mut serial = Frame::new_full(input, 0, Default::default(), 0, usize::MAX, None)?; + let (mut producer, mut sink) = make_producer(input, 0, Default::default())?; + let bytes = producer.heap_bytes()?; + let mut expected_budget = budget; + let mut actual_budget = budget; + let reserved = crate::SeiInfo { + reserved_hashes: 1, + ..Default::default() + }; + assert_eq!( + producer.hash(reserved, [&[], &[], &[]]), + serial.hash(reserved, 0, usize::MAX).map(|_| ()) + ); + for input in inputs { + let expected = + serial.decode_full(*input, &dpb, &mut expected_budget, 0, usize::MAX, 1, None); + let actual = + producer.decode(*input, &dpb, &mut actual_budget, 0, usize::MAX, &mut sink); + assert_eq!(actual.map(|_| ()), expected.map(|_| ())); + assert_eq!(actual_budget, expected_budget); + assert_entropy(&producer, &serial); + assert_eq!(producer.heap_bytes()?, bytes); + assert_eq!( + sink.motion, + ((serial.next_ctu / input.layout.width) as usize * (1 << input.sps.ctb_log2)) + .min(input.picture.height as usize) + ); + if expected.is_err() { + assert!(producer.finish().is_err()); + return Ok(()); + } + } + assert!(serial.complete()); + serial.finalize(0, usize::MAX)?; + let planes = sink.picture.planes.each_ref().map(|plane| plane.as_slice()); + use md5::{Digest, Md5}; + let hashes = planes.map(|plane| { + let mut hash = Md5::new(); + for sample in plane { + let bytes = sample.to_le_bytes(); + hash.update(&bytes[..if input.picture.bit_depth == 8 { 1 } else { 2 }]); + } + hash.finalize().into() + }); + let hash = crate::SeiInfo { + picture_hash: Some(crate::PictureHash::Md5(hashes)), + reserved_hashes: 1, + ..Default::default() + }; + producer.hash(hash, planes)?; + serial.hash(hash, 0, usize::MAX)?; + assert_eq!( + producer.hash(reserved, planes), + serial.hash(reserved, 0, usize::MAX).map(|_| ()) + ); + let addresses = sink.picture.planes.each_ref().map(|plane| plane.as_ptr()); + let actual = sink.retain(producer.finish()?)?; + assert_eq!( + actual.planes.each_ref().map(|plane| plane.as_ptr()), + addresses + ); + assert_eq!(actual.planes, serial.planes); + assert_eq!(actual.stats, serial.stats); + assert_eq!(actual.inter_stats, serial.inter_stats); + assert_eq!( + actual + .cells + .iter() + .map(|c| (c.mode, c.motion, c.qp, c.slice, c.reconstructed)) + .collect::>(), + serial + .cells + .iter() + .map(|c| (c.mode, c.motion, c.qp, c.slice, c.reconstructed)) + .collect::>() + ); + Ok(()) + } + + #[test] + fn raster_full_frame_conversion_preserves_backing_and_cold_fallback() -> Result<()> { + let fixture = fixture(16, 64, 8, false, 1)?; + let mut rbsp = vec![0x80]; + rbsp.extend(pcm_body( + &[[0x86, 0x80], [0x8b, 0x80], [0x91, 0x80], [0x96, 0x80]], + [0xfe, 0x80], + &[34, 51, 68, 85], + )?); + let original = escaped(&rbsp); + super::super::raster::tests::conversion(fixture.input(segment(&rbsp, &original, &[]))) + } + + #[test] + fn raster_intra_top_border_precedes_vertical_deblocking() -> Result<()> { + for depth in [8, 10] { + let fixture = fixture(16, 32, depth, false, 1)?; + let rbsp = [0x80, 0x88, 0x58, 0x13, 0xfe]; + let original = escaped(&rbsp); + let mut tail = segment(&rbsp, &original, &[]); + tail.address = 1; + let mut input = fixture.input(tail); + input.syntax.deblock_disabled = false; + super::super::raster::tests::intra_border(input)?; + } + Ok(()) + } + + #[test] + fn raster_resumes_one_entropy_cursor_across_rows_and_split_segments() -> Result<()> { + for ctb in [16u32, 32, 64] { + for depth in [8, 10] { + let mut fixture = fixture(ctb, ctb * 4, depth, false, 1)?; + fixture.sps.ctb_log2 = ctb.trailing_zeros(); + fixture.sps.min_cb_log2 = ctb.trailing_zeros(); + fixture.sps.ctu_count = 4; + fixture.sps.pcm = Some(Pcm { + luma_depth: depth, + chroma_depth: depth, + min_log2: ctb.trailing_zeros(), + max_log2: ctb.trailing_zeros(), + loop_filter_disabled: false, + }); + fixture.picture.ctb_log2 = ctb.trailing_zeros(); + fixture.layout = Layout::new(&fixture.sps, &fixture.pps)?; + let mut rbsp = vec![0x80]; + for (start, value) in [0x86, 0x8b, 0x91, 0x96].into_iter().zip([34, 51, 68, 85]) { + rbsp.extend([start, 0x80]); + let mut raw = params::tests::Writer::default(); + for _ in 0..ctb * ctb * 3 / 2 { + raw.u(depth, value << (depth - 8)); + } + rbsp.extend(raw.bytes()); + } + rbsp.extend([0xfe, 0x80]); + let original = escaped(&rbsp); + let input = fixture.input(segment(&rbsp, &original, &[])); + raster_matches_serial(&[input], 1_000_000)?; + for budget in [0, 2, 3, 5, 8, 11] { + raster_matches_serial(&[input], budget)?; + } + let mut truncated = rbsp.clone(); + truncated.truncate(truncated.len() - 100); + let truncated_nal = escaped(&truncated); + raster_matches_serial( + &[fixture.input(segment(&truncated, &truncated_nal, &[]))], + 1_000_000, + )?; + } + } + for dependent in [false, true] { + let fixture = fixture(32, 32, 8, false, 1)?; + let mut first = vec![0x80]; + first.extend(pcm_body(&[[0x86, 0x80]], [0xfe, 0x80], &[34])?); + let first_nal = escaped(&first); + let mut second = vec![0x80]; + second.extend(pcm_body( + &[ + [if dependent { 0x8c } else { 0x86 }, 0x80], + [if dependent { 0x91 } else { 0x8b }, 0x80], + [if dependent { 0x96 } else { 0x91 }, 0x80], + ], + [0xfe, 0x80], + &[51, 68, 85], + )?); + let second_nal = escaped(&second); + let mut tail = segment(&second, &second_nal, &[]); + tail.address = 1; + tail.dependent = dependent; + tail.independent = if dependent { 0 } else { 1 }; + if dependent { + super::super::raster::tests::qp_continuation( + fixture.input(segment(&first, &first_nal, &[])), + fixture.input(tail), + )?; + } + raster_matches_serial( + &[ + fixture.input(segment(&first, &first_nal, &[])), + fixture.input(tail), + ], + 1_000_000, + )?; + } + Ok(()) + } + + #[test] + fn raster_later_segment_validation_precedes_layout_eligibility() -> Result<()> { + use super::super::raster::tests::producer as make_producer; + let fixture = fixture(32, 32, 8, false, 1)?; + let changed = self::fixture(32, 32, 8, true, 1)?; + let mut rbsp = vec![0x80]; + rbsp.extend(pcm_body(&[[0x86, 0x80]], [0xfe, 0x80], &[34])?); + let original = escaped(&rbsp); + let first = fixture.input(segment(&rbsp, &original, &[])); + let dpb = crate::dpb::Dpb::new_reconstruction(); + for invalid_inter in [false, true] { + let mut serial = Frame::new_full(&first, 0, Default::default(), 0, usize::MAX, None)?; + let (mut producer, mut sink) = make_producer(&first, 0, Default::default())?; + let mut expected_budget = 1000; + let mut actual_budget = 1000; + serial.decode_full(first, &dpb, &mut expected_budget, 0, usize::MAX, 1, None)?; + producer.decode(first, &dpb, &mut actual_budget, 0, usize::MAX, &mut sink)?; + let mut later = segment(&rbsp, &original, &[]); + later.address = 1; + later.dependent = true; + let mut input = changed.input(later); + if invalid_inter { + input.syntax.kind = 3; + } + let expected = serial + .decode_full(input, &dpb, &mut expected_budget, 0, usize::MAX, 1, None) + .map(|_| ()); + assert_eq!( + expected, + Err(Error::InvalidData(if invalid_inter { + "inter_segment_state" + } else { + "selected_segment_state" + })) + ); + assert_eq!( + producer + .decode(input, &dpb, &mut actual_budget, 0, usize::MAX, &mut sink) + .map(|_| ()), + expected + ); + assert_eq!(actual_budget, expected_budget); + assert_eq!((sink.motion, sink.filtered), (0, 0)); + assert!(producer.finish().is_err()); + } + Ok(()) + } + + #[test] + fn raster_deadline_and_commit_failures_are_terminal_without_a_final_flush() -> Result<()> { + use super::super::raster::{ + MotionBand, SampleBand, Sink, tests::producer as make_producer, + }; + let fixture = fixture(16, 32, 8, false, 1)?; + let mut rbsp = vec![0x80]; + rbsp.extend(pcm_body( + &[[0x86, 0x80], [0x8b, 0x80]], + [0xfe, 0x80], + &[34, 51], + )?); + let original = escaped(&rbsp); + let input = fixture.input(segment(&rbsp, &original, &[])); + let dpb = crate::dpb::Dpb::new_reconstruction(); + let (mut producer, mut sink) = make_producer(&input, 0, Default::default())?; + let mut serial = Frame::new_full(&input, 0, Default::default(), 0, usize::MAX, None)?; + let deadline = Some(std::time::Instant::now()); + super::super::raster::tests::set_deadline(&mut producer, deadline); + serial.deadline = deadline; + let mut expected_budget = 1000; + let mut actual_budget = 1000; + assert_eq!( + producer + .decode(input, &dpb, &mut actual_budget, 0, usize::MAX, &mut sink) + .map(|_| ()), + serial + .decode_full(input, &dpb, &mut expected_budget, 0, usize::MAX, 1, None) + .map(|_| ()) + ); + assert_eq!(actual_budget, expected_budget); + assert_eq!((sink.motion, sink.filtered), (0, 0)); + assert!(producer.finish().is_err()); + struct Refuse { + motion: usize, + samples: usize, + fail_motion: bool, + } + impl Sink for Refuse { + fn motion(&mut self, _: MotionBand<'_>) -> Result<()> { + self.motion += 1; + if self.fail_motion { + Err(Error::Internal("test_commit")) + } else { + Ok(()) + } + } + fn samples(&mut self, _: SampleBand<'_>) -> Result<()> { + self.samples += 1; + Err(Error::Internal("test_commit")) + } + } + for fail_motion in [false, true] { + let (mut producer, _backing) = make_producer(&input, 0, Default::default())?; + let mut sink = Refuse { + motion: 0, + samples: 0, + fail_motion, + }; + assert_eq!( + producer.decode(input, &dpb, &mut 1000, 0, usize::MAX, &mut sink), + Err(Error::Internal("test_commit")) + ); + assert_eq!( + (sink.motion, sink.samples), + if fail_motion { (1, 0) } else { (2, 1) } + ); + assert_eq!( + producer.decode(input, &dpb, &mut 1000, 0, usize::MAX, &mut sink), + Err(Error::InvalidData("selected_segment_state")) + ); + assert!(producer.finish().is_err()); + } + Ok(()) + } + + #[test] + fn raster_uses_real_intra_inter_and_hash_failure_paths() -> Result<()> { + use super::super::raster::tests::producer as make_producer; + for depth in [8, 10] { + for (initialization, body) in [ + (Initialization::P, [0xfe, 0xe0]), + (Initialization::B, [0x3c, 0x07]), + ] { + let mut fixture = fixture(8, 8, depth, false, 1)?; + fixture.sps.min_cb_log2 = 3; + fixture.sps.pcm = None; + fixture.sps.conformance_window = [0; 4]; + fixture.pps.qp_delta_enabled = true; + let rbsp = [0x80, body[0], body[1]]; + let original = escaped(&rbsp); + let input = inter_input(&fixture, &rbsp, &original, initialization); + let dpb = reference_dpb(&input)?; + let mut serial = + Frame::new_full(&input, 1, dpb.current_references(), 0, usize::MAX, None)?; + let (mut producer, mut sink) = make_producer(&input, 1, dpb.current_references())?; + serial.decode_full(input, &dpb, &mut 1000, 0, usize::MAX, 1, None)?; + serial.finalize(0, usize::MAX)?; + producer.decode(input, &dpb, &mut 1000, 0, usize::MAX, &mut sink)?; + let hash = crate::SeiInfo { + picture_hash: Some(crate::PictureHash::Checksum([[0; 4]; 3])), + ..Default::default() + }; + assert_eq!( + producer.hash( + hash, + sink.picture.planes.each_ref().map(|plane| plane.as_slice()) + ), + serial.hash(hash, 0, usize::MAX).map(|_| ()) + ); + let actual = sink.retain(producer.finish()?)?; + assert_eq!(actual.planes, serial.planes); + assert_eq!(actual.inter_stats, serial.inter_stats); + } + let mut fixture = fixture(8, 8, depth, false, 1)?; + fixture.sps.min_cb_log2 = 3; + fixture.sps.pcm = None; + let rbsp = [0x80, 0x88, 0x58, 0x13, 0xfe]; + let original = escaped(&rbsp); + raster_matches_serial(&[fixture.input(segment(&rbsp, &original, &[]))], 1000)?; + } + Ok(()) + } + + #[test] + fn private_row_windows_keep_pcm_cursor_samples_and_failure_prefix() -> Result<()> { + for depth in [8, 10] { + for truncated in [false, true] { + let fixture = fixture(16, 64, depth, false, 1)?; + let mut raw = params::tests::Writer::default(); + for _ in 0..384 { + raw.u(depth, (1 << depth) - 7); + } + let mut body = vec![0x86, 0x80]; + body.extend(raw.bytes()); + body.extend([0xfe, 0x80]); + if truncated { + body.truncate(body.len() / 2); + } + let input = fixture.input(segment(&body, &body, &[])); + let mut serial = Frame::new(&input, 0, usize::MAX)?; + let mut private = Frame::new(&input, 0, usize::MAX)?; + for (component, plane) in private.planes.iter_mut().enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + *plane = plane[16 / scale * (16 / scale)..48 / scale * (16 / scale)].to_vec(); + } + private.cells = private.cells[4 * 4..12 * 4].to_vec(); + serial.next_ctu = 2; + private.next_ctu = 2; + let mut serial_budget = 1000; + let mut private_budget = 1000; + let mut serial_bins = Cabac::new(&body, &mut serial_budget)?; + let mut private_bins = Cabac::new(&body, &mut private_budget)?; + let expected = serial.decode_ctu_references(&input, None, 1, &mut serial_bins); + let actual = private.decode_ctu_rows( + &input, + None, + 1, + &mut private_bins, + super::super::rows::SampleRows { + origin: 16, + above: None, + above_y: 0, + }, + ); + assert_eq!(actual, expected); + assert_eq!(actual.is_err(), truncated); + assert_eq!(serial_bins.position(), private_bins.position()); + assert_eq!(serial_budget, private_budget); + assert_eq!(serial.next_ctu, private.next_ctu); + assert_eq!(serial.stats, private.stats); + for (component, plane) in private.planes.iter().enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + assert_eq!( + plane, + &serial.planes[component] + [16 / scale * (16 / scale)..48 / scale * (16 / scale)] + ); + } + assert_eq!( + private + .cells + .iter() + .map(|c| (c.mode, c.reconstructed, c.slice)) + .collect::>(), + serial.cells[4 * 4..12 * 4] + .iter() + .map(|c| (c.mode, c.reconstructed, c.slice)) + .collect::>() + ); + } + } + Ok(()) + } + + #[test] + fn private_row_intra_reads_use_unfiltered_top_border_and_local_cells() -> Result<()> { + let fixture = fixture(32, 64, 8, false, 1)?; + let input = fixture.input(segment(&[], &[], &[])); + let mut serial = Frame::new(&input, 0, usize::MAX)?; + let mut private = Frame::new(&input, 0, usize::MAX)?; + for (component, plane) in serial.planes.iter_mut().enumerate() { + let width = if component == 0 { 32 } else { 16 }; + for (index, sample) in plane.iter_mut().enumerate() { + *sample = (index % width + 23 + component * 11) as u16; + } + } + for cell in serial.cells.iter_mut().take(12 * 8) { + cell.mode = Some(IntraMode::new(1)?); + cell.reconstructed = 7; + cell.slice = 1; + } + for (component, plane) in private.planes.iter_mut().enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + *plane = serial.planes[component][16 / scale * (32 / scale)..48 / scale * (32 / scale)] + .to_vec(); + plane[(31 / scale - 16 / scale) * (32 / scale) + ..(32 / scale - 16 / scale) * (32 / scale)] + .fill(255); + } + private.cells = serial.cells[4 * 8..12 * 8].to_vec(); + let above = core::array::from_fn(|component| { + let width = if component == 0 { 32 } else { 16 }; + (0..width) + .map(|x| (x + 23 + component * 11) as u16) + .collect::>() + }); + let mode = IntraMode::new(1)?; + for component in 0..3 { + let scale = if component == 0 { 1 } else { 2 }; + let block = || ReconstructionBlock { + area: Area { + x: 16 / scale, + y: 32 / scale, + log2: 4 - u32::from(component != 0), + }, + component, + mode: Some(mode), + coded: false, + bypass: false, + }; + Walker { + frame: &mut serial, + input: &input, + location: fixture.layout.at_raster(5)?, + slice: 1, + dpb: None, + rows: super::super::rows::SampleRows::full(), + } + .references(&block())?; + let mut walker = Walker { + frame: &mut private, + input: &input, + location: fixture.layout.at_raster(5)?, + slice: 1, + dpb: None, + rows: super::super::rows::SampleRows { + origin: 16, + above: Some(&above), + above_y: 32, + }, + }; + walker.references(&block())?; + assert_eq!(walker.frame.scratch.references, serial.scratch.references); + assert_eq!(walker.cell(16, 32)?.slice, 1); + walker.protect(Area { + x: 16, + y: 32, + log2: 4, + })?; + assert!(walker.cell(16, 32)?.protected); + if component == 2 { + walker.frame.cells.clear(); + assert!(walker.neighbour(Some(16), Some(48))?.is_none()); + assert!(matches!( + walker.neighbour(Some(16), Some(32)), + Err(Error::InvalidData("intra_cell_address")) + )); + } + } + Ok(()) + } + + #[test] + fn private_row_wpp_above_right_uses_the_window_with_and_without_a_checkpoint() -> Result<()> { + let fixture = fixture(32, 64, 8, true, 1)?; + let mut body = vec![0x86, 0x80]; + body.extend(core::iter::repeat_n(42, 384)); + body.extend([0xfe, 0x80]); + let input = fixture.input(segment(&body, &body, &[])); + for saved in [false, true] { + let mut serial = Frame::new(&input, 0, usize::MAX)?; + let mut private = Frame::new(&input, 0, usize::MAX)?; + cell_mut(&mut serial, 16, 16)?.slice = 1; + private.cells = serial.cells[4 * 8..12 * 8].to_vec(); + for (component, plane) in private.planes.iter_mut().enumerate() { + let scale = if component == 0 { 1 } else { 2 }; + *plane = plane[16 / scale * (32 / scale)..48 / scale * (32 / scale)].to_vec(); + } + for frame in [&mut serial, &mut private] { + frame.next_ctu = 4; + frame.wpp = saved.then_some(Checkpoint { + contexts: Contexts::new(26, Initialization::I), + tile: 0, + row: 1, + slice: 1, + }); + } + let mut serial_budget = 1000; + let mut private_budget = 1000; + let mut serial_bins = Cabac::new(&body, &mut serial_budget)?; + let mut private_bins = Cabac::new(&body, &mut private_budget)?; + let expected = serial.decode_ctu_references(&input, None, 1, &mut serial_bins); + let actual = private.decode_ctu_rows( + &input, + None, + 1, + &mut private_bins, + super::super::rows::SampleRows { + origin: 16, + above: None, + above_y: 0, + }, + ); + assert_eq!(actual, expected); + if saved { + assert!(actual.is_ok()); + } else { + assert_eq!(actual, Err(Error::InvalidData("intra_wpp_checkpoint"))); + } + assert_eq!(private.stats, serial.stats); + assert_eq!(private.contexts.split_cu, serial.contexts.split_cu); + assert_eq!(private_bins.position(), serial_bins.position()); + assert_eq!(private_budget, serial_budget); + } + Ok(()) + } + fn inter_input<'a>( fixture: &'a Fixture, rbsp: &'a [u8], @@ -2510,6 +3180,7 @@ mod tests { plane.fill(value); } let mut walker = Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -2568,6 +3239,7 @@ mod tests { plane.fill(42); } let mut walker = Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -2705,6 +3377,7 @@ mod tests { location: fixture.layout.at_raster(0)?, slice: 1, dpb: None, + rows: super::super::rows::SampleRows::full(), }; walker.references(&ReconstructionBlock { area: Area { @@ -2762,6 +3435,7 @@ mod tests { location: fixture.layout.at_raster(0)?, slice: 1, dpb: None, + rows: super::super::rows::SampleRows::full(), }; for component in 0..3 { let scale = if component == 0 { 1 } else { 2 }; @@ -2839,6 +3513,7 @@ mod tests { plane.fill(42); } let mut walker = Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -2912,6 +3587,7 @@ mod tests { plane.fill(42); } let mut walker = Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -2984,6 +3660,7 @@ mod tests { let input = fixture.input(segment(&[], &[], &[])); let mut frame = Frame::new(&input, 0, usize::MAX)?; let mut walker = Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -3295,6 +3972,7 @@ mod tests { protected: false, }; let mut walker = Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(1)?, @@ -3345,6 +4023,7 @@ mod tests { *cell_mut(&mut frame, 8, 4)? = value; let location = fixture.layout.at_raster(0)?; let mut walker = Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location, @@ -3384,6 +4063,7 @@ mod tests { let mut budget = 5; let mut cabac = Cabac::new(&[0x17, 0], &mut budget)?; let mut walker = Walker { + rows: crate::reconstruction::rows::SampleRows::full(), frame: &mut frame, input: &input, location: fixture.layout.at_raster(0)?, @@ -3708,6 +4388,64 @@ mod tests { Ok((fixture, rbsp, lengths)) } + #[test] + fn wavefront_poisoned_worker_reuse_never_exposes_stale_prediction_samples() -> Result<()> { + use super::super::Cache; + for threads in [2, 4, 8] { + let (fixture, rbsp, lengths) = cached_wavefront(93, 8, 8)?; + let original = escaped(&rbsp); + let input = fixture.input(segment(&rbsp, &original, &lengths)); + let mut serial = Frame::new(&input, 0, usize::MAX)?; + let mut expected_budget = 10_000_000; + serial.decode(input, &mut expected_budget, 0, usize::MAX)?; + let mut cache = Cache::default(); + for reuse in [false, true] { + if reuse { + assert_eq!(cache.frames_for_test().len(), threads - 1); + for frame in cache.frames_for_test() { + for plane in &mut frame.planes { + plane.fill(255); + } + for cell in &mut frame.cells { + cell.slice = 1; + cell.reconstructed = 7; + cell.mode = Some(IntraMode::new(34)?); + cell.depth = 3; + cell.qp = 51; + cell.edges = 3; + cell.transform_edges = 3; + cell.protected = true; + } + for ctu in &mut frame.filter_ctus { + ctu.seen = true; + ctu.slice = 1; + ctu.sao[0].kind = 1; + } + } + } + let mut frame = Frame::new(&input, 0, usize::MAX)?; + let mut budget = 10_000_000; + frame.decode_common( + input, + None, + &mut budget, + 0, + usize::MAX, + threads, + Some(&mut cache), + )?; + assert_eq!(frame.workers, threads); + assert!(frame.complete()); + assert_eq!(frame.planes, serial.planes); + assert_eq!(frame.stats, serial.stats); + assert_eq!(frame.inter_stats, serial.inter_stats); + assert_eq!(budget, expected_budget); + } + assert_eq!(cache.hits(), 1); + } + Ok(()) + } + #[test] fn wavefront_warm_workers_match_cold_and_serial_across_picture_changes() -> Result<()> { use super::super::{Cache, ReclaimMemory}; @@ -4077,6 +4815,97 @@ mod tests { Ok(()) } + #[test] + fn wavefront_poisoned_reuse_preserves_partial_slice_tails_at_both_depths() -> Result<()> { + use super::super::Cache; + for depth in [8, 10] { + let (fixture, seed, seed_lengths) = cached_wavefront(110, depth, 2)?; + let seed_original = escaped(&seed); + let seed_input = fixture.input(segment(&seed, &seed_original, &seed_lengths)); + let top = pcm_body( + &[[0x86, 0x80], [0x8b, 0x80], [0x91, 0x80]], + [0xfd, 0x80], + &[34; 3], + )?; + let lengths = [top.len() as u64]; + let mut rbsp = vec![0x80]; + rbsp.extend(top); + rbsp.extend(pcm_body(&[[0x92, 0x80]], [0xfe, 0x80], &[51])?); + let original = escaped(&rbsp); + let input = fixture.input(segment(&rbsp, &original, &lengths)); + let mut tail = vec![0x80]; + tail.extend(pcm_body( + &[[0x97, 0x80], [0x9b, 0x80]], + [0xfe, 0x80], + &[68; 2], + )?); + let tail_original = escaped(&tail); + let mut tail_segment = segment(&tail, &tail_original, &[]); + tail_segment.address = 4; + tail_segment.dependent = true; + let tail_input = fixture.input(tail_segment); + let mut expected = None; + for threads in [1, 2, 4, 8] { + let mut cache = Cache::default(); + if threads > 1 { + let mut seed_frame = Frame::new(&seed_input, 0, usize::MAX)?; + seed_frame.decode_common( + seed_input, + None, + &mut 10_000_000, + 0, + usize::MAX, + threads, + Some(&mut cache), + )?; + assert_eq!(cache.frames_for_test().len(), 1); + for frame in cache.frames_for_test() { + for plane in &mut frame.planes { + plane.fill(u16::MAX); + } + for cell in &mut frame.cells { + cell.slice = 1; + cell.reconstructed = 7; + cell.mode = Some(IntraMode::new(34)?); + cell.protected = true; + } + } + } + let mut frame = Frame::new(&input, 0, usize::MAX)?; + let mut budget = 10_000_000; + frame.decode_common( + input, + None, + &mut budget, + 0, + usize::MAX, + threads, + Some(&mut cache), + )?; + assert_eq!(frame.next_ctu, 4); + assert!(!frame.complete()); + assert_eq!(cache.hits(), usize::from(threads > 1)); + frame.decode_common( + tail_input, + None, + &mut budget, + 0, + usize::MAX, + threads, + Some(&mut cache), + )?; + assert!(frame.complete()); + let actual = (frame.planes, frame.stats, frame.inter_stats, budget); + if let Some(expected) = &expected { + assert_eq!(&actual, expected); + } else { + expected = Some(actual); + } + } + } + Ok(()) + } + #[test] fn wavefront_partial_final_row_continues_as_a_dependent_segment() -> Result<()> { let fixture = fixture(48, 32, 8, true, 1)?; diff --git a/crates/coreav-hevc/src/sei.rs b/crates/coreav-hevc/src/sei.rs index 782a5bbed..20db1bf64 100644 --- a/crates/coreav-hevc/src/sei.rs +++ b/crates/coreav-hevc/src/sei.rs @@ -55,6 +55,15 @@ pub(crate) fn parse(mut bytes: &[u8], suffix: bool) -> Result { let mut hash_seen = false; while bytes != [0x80] { let kind = extended_byte(&mut bytes)?; + #[cfg(hevc_thread_diagnostics)] + match kind { + 0 => crate::thread_diagnostic!(event SeiType0), + 1 => crate::thread_diagnostic!(event SeiType1), + 5 => crate::thread_diagnostic!(event SeiType5), + 129 => crate::thread_diagnostic!(event SeiType129), + 132 => crate::thread_diagnostic!(event SeiType132), + _ => crate::thread_diagnostic!(event SeiTypeOther), + } let size = extended_byte(&mut bytes)? as usize; let payload = bytes .get(..size) diff --git a/crates/coreav-hevc/src/tests.rs b/crates/coreav-hevc/src/tests.rs index c87c64362..cf11cafe1 100644 --- a/crates/coreav-hevc/src/tests.rs +++ b/crates/coreav-hevc/src/tests.rs @@ -10,6 +10,10 @@ use pcm::{escaped, padded, pcm_fixture}; mod intra_parameters; fn synthetic_parameter_nals() -> [Vec; 3] { + synthetic_parameter_nals_with_wpp(false) +} + +pub(super) fn synthetic_parameter_nals_with_wpp(wpp: bool) -> [Vec; 3] { let mut w = params::tests::Writer::default(); w.ue(0); w.ue(0); @@ -26,8 +30,8 @@ fn synthetic_parameter_nals() -> [Vec; 3] { } w.se(0); w.se(0); - for _ in 0..7 { - w.flag(false); + for flag in [false, false, false, false, false, wpp, false] { + w.flag(flag); } w.flag(true); w.flag(true); @@ -38,9 +42,13 @@ fn synthetic_parameter_nals() -> [Vec; 3] { w.flag(false); w.flag(false); let pps = padded(w); + let mut sps = params::tests::sequence(8, false, false, false); + if wpp { + sps = replaced_ue(&sps, 8 + 96 + 1 + 3 + 13, 128); + } [ escaped(32, ¶ms::tests::video(false)), - escaped(33, ¶ms::tests::sequence(8, false, false, false)), + escaped(33, &sps), escaped(34, &pps), ] } @@ -880,7 +888,7 @@ fn reconstruction_scratch_reservations_cover_rbsp_allocation_and_actual_capacity refuse: Option, } impl parser::PictureObserver for Probe { - fn reserve_scratch(&mut self, bytes: usize, _: &dpb::Dpb) -> Result<()> { + fn reserve_scratch(&mut self, bytes: usize, _: &mut dpb::Dpb) -> Result<()> { self.calls.push(bytes); if self.refuse == Some(self.calls.len()) { return Err(Error::Limit("fixture_scratch")); @@ -890,7 +898,7 @@ fn reconstruction_scratch_reservations_cover_rbsp_allocation_and_actual_capacity fn boundary(&mut self, _: &mut dpb::Dpb, _: usize) -> Result<()> { Ok(()) } - fn slice(&mut self, _: parser::SliceInput<'_>, _: &dpb::Dpb) -> Result<()> { + fn slice(&mut self, _: parser::SliceInput<'_>, _: &mut dpb::Dpb) -> Result<()> { Ok(()) } fn sei(&mut self, _: SeiInfo, _: &dpb::Dpb, _: usize) -> Result<()> { @@ -1009,6 +1017,10 @@ fn idr_segment(first: bool) -> Vec { } fn intra_first(kind: u8) -> Vec { + intra_first_with_wpp(kind, 1, None) +} + +pub(super) fn intra_first_with_wpp(kind: u8, poc: u32, entries: Option) -> Vec { let mut w = params::tests::Writer::default(); w.flag(true); if (16..=21).contains(&kind) { @@ -1017,7 +1029,7 @@ fn intra_first(kind: u8) -> Vec { w.ue(0); w.ue(2); w.flag(true); - w.u(4, 1); + w.u(4, poc); w.flag(false); w.flag(false); for _ in 0..4 { @@ -1028,6 +1040,15 @@ fn intra_first(kind: u8) -> Vec { w.flag(false); w.se(0); w.flag(false); + if let Some(entries) = entries { + w.ue(entries); + if entries != 0 { + w.ue(0); + for _ in 0..entries { + w.u(1, 0); + } + } + } let mut rbsp = padded(w); rbsp.push(0x80); escaped(kind, &rbsp) diff --git a/crates/coreav-hevc/src/thread_diagnostics.rs b/crates/coreav-hevc/src/thread_diagnostics.rs new file mode 100644 index 000000000..8e517d79b --- /dev/null +++ b/crates/coreav-hevc/src/thread_diagnostics.rs @@ -0,0 +1,302 @@ +//! Untimed diagnostic counters and bounded read traces, absent from ordinary builds. +//! Diagnostic interfaces have no stability guarantee. + +use std::cell::RefCell; + +mod rows; +pub(crate) use rows::{ + ReadContext, ReadGuard, check_row_threads, motion_read, row_picture, sample_read, +}; +pub use rows::{RowDependency, RowPicture, RowReference, begin_rows, take_rows}; + +#[doc(hidden)] +pub struct DependencySnapshot { + pub rps: [Option<(u64, i32, bool)>; 16], + pub active: [Option<(u64, i32, bool)>; 16], +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[doc(hidden)] +pub struct DependencyOutput { + pub event: crate::OutputEvent, + pub before_picture: u64, +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct DependencyOutputs { + pub events: [Option; 32], + pub finishing: bool, + pub current_picture: Option, + pub deferred: bool, +} + +thread_local! { + static OUTPUTS: RefCell> = const { RefCell::new(None) }; +} + +pub(crate) fn capture_outputs(outputs: &mut DependencyOutputs, run: impl FnOnce() -> T) -> T { + struct Capture<'a> { + target: &'a mut DependencyOutputs, + previous: Option, + } + impl Drop for Capture<'_> { + fn drop(&mut self) { + if let Some(completed) = OUTPUTS.replace(self.previous.take()) { + *self.target = completed; + } + } + } + let previous = OUTPUTS.replace(Some(*outputs)); + let _capture = Capture { + target: outputs, + previous, + }; + run() +} + +pub(crate) fn output_picture(index: Option) { + OUTPUTS.with_borrow_mut(|outputs| { + if let Some(outputs) = outputs { + outputs.current_picture = index; + outputs.deferred = false; + } + }); +} + +pub(crate) fn output_finish() { + OUTPUTS.with_borrow_mut(|outputs| { + if let Some(outputs) = outputs { + outputs.finishing = true; + } + }); +} + +pub(crate) fn output_event( + event: crate::OutputEvent, + count: u64, + current_stored: bool, +) -> crate::Result<()> { + OUTPUTS.with_borrow_mut(|outputs| { + let Some(outputs) = outputs else { + return Ok(()); + }; + let before_picture = match outputs.current_picture { + Some(current) if outputs.deferred || event.picture_index == current => count, + Some(_) => count + .checked_sub(1) + .ok_or(crate::Error::Internal("dependency_output_order"))?, + None => count + .checked_sub(u64::from(!outputs.finishing && !current_stored)) + .ok_or(crate::Error::Internal("dependency_output_order"))?, + }; + *outputs + .events + .iter_mut() + .find(|entry| entry.is_none()) + .ok_or(crate::Error::Limit("dependency_output_count"))? = Some(DependencyOutput { + event, + before_picture, + }); + if outputs.current_picture == Some(event.picture_index) { + outputs.deferred = true; + } + Ok(()) + }) +} + +const LABELS: [&str; 35] = [ + "barrier_workers_disabled", + "barrier_wpp", + "barrier_output_random_access", + "barrier_output_capacity", + "barrier_output_reorder", + "barrier_output_latency", + "barrier_reference_unavailable", + "barrier_memory_input", + "barrier_memory_scratch", + "barrier_memory_frame", + "barrier_queue_budget", + "barrier_queue_memory", + "barrier_snapshot_allocation", + "barrier_window_full", + "barrier_duplicate_hash", + "barrier_worker_spawn", + "event_picture_i", + "event_picture_p", + "event_picture_b", + "event_prefix_sei", + "event_suffix_sei_informational", + "event_suffix_sei_hash", + "event_parameter_set", + "event_queue_admitted", + "event_spawned_job", + "event_caller_job", + "event_forced_flush", + "event_reconcile", + "event_sequence_end", + "event_sei_type_0", + "event_sei_type_1", + "event_sei_type_5", + "event_sei_type_129", + "event_sei_type_132", + "event_sei_type_other", +]; + +pub(crate) enum Barrier { + WorkersDisabled, + Wpp, + OutputRandomAccess, + OutputCapacity, + OutputReorder, + OutputLatency, + ReferenceUnavailable, + MemoryInput, + MemoryScratch, + MemoryFrame, + QueueBudget, + QueueMemory, + SnapshotAllocation, + WindowFull, + DuplicateHash, + WorkerSpawn, +} + +pub(crate) enum Event { + PictureI, + PictureP, + PictureB, + PrefixSei, + SuffixSeiInformational, + SuffixSeiHash, + ParameterSet, + QueueAdmitted, + SpawnedJob, + CallerJob, + ForcedFlush, + Reconcile, + SequenceEnd, + SeiType0, + SeiType1, + SeiType5, + SeiType129, + SeiType132, + SeiTypeOther, +} + +struct Counters { + picture: Option, + seen: u32, + counts: [u64; LABELS.len()], +} + +impl Default for Counters { + fn default() -> Self { + Self { + picture: None, + seen: 0, + counts: [0; LABELS.len()], + } + } +} + +thread_local! { + static COUNTERS: RefCell = RefCell::new(Counters::default()); +} + +pub(crate) fn picture(index: u64) { + COUNTERS.with_borrow_mut(|counters| { + if counters.picture != Some(index) { + counters.picture = Some(index); + counters.seen = 0; + } + }); +} + +pub(crate) fn barrier(reason: Barrier) { + COUNTERS.with_borrow_mut(|counters| { + let index = reason as usize; + let mask = 1 << index; + if counters.picture.is_none() || counters.seen & mask != 0 { + return; + } + counters.seen |= mask; + if let Some(count) = counters.counts.get_mut(index) { + *count = count.saturating_add(1); + } + }); +} + +pub(crate) fn event(reason: Event) { + COUNTERS.with_borrow_mut(|counters| { + if let Some(count) = counters.counts.get_mut(16 + reason as usize) { + *count = count.saturating_add(1); + } + }); +} + +/// Takes and resets profiling counters; barrier reasons may overlap per picture. +pub fn take() -> [(&'static str, u64); 35] { + COUNTERS.with_borrow_mut(|counters| { + let mut counts = std::mem::take(counters).counts.into_iter(); + LABELS.map(|label| (label, counts.next().unwrap_or(0))) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn output_event_refuses_an_impossible_zero_count() { + let event = crate::OutputEvent { + poc: 0, + picture_index: 0, + width: 8, + height: 8, + bit_depth: 8, + }; + for current_picture in [None, Some(1)] { + let mut outputs = DependencyOutputs { + current_picture, + ..DependencyOutputs::default() + }; + let result = capture_outputs(&mut outputs, || output_event(event, 0, false)); + assert_eq!( + result, + Err(crate::Error::Internal("dependency_output_order")) + ); + assert!(outputs.events.iter().all(Option::is_none)); + } + } + + #[test] + fn barriers_are_per_picture_while_events_count_every_operation() { + take(); + barrier(Barrier::QueueMemory); + picture(2); + barrier(Barrier::QueueMemory); + barrier(Barrier::QueueMemory); + event(Event::QueueAdmitted); + event(Event::QueueAdmitted); + picture(3); + barrier(Barrier::QueueMemory); + let values = take(); + assert_eq!( + values + .iter() + .filter(|(name, _)| *name == "barrier_queue_memory") + .map(|(_, count)| *count) + .sum::(), + 2 + ); + assert_eq!( + values + .iter() + .filter(|(name, _)| *name == "event_queue_admitted") + .map(|(_, count)| *count) + .sum::(), + 2 + ); + assert!(take().into_iter().all(|(_, count)| count == 0)); + } +} diff --git a/crates/coreav-hevc/src/thread_diagnostics/rows.rs b/crates/coreav-hevc/src/thread_diagnostics/rows.rs new file mode 100644 index 000000000..32f70d55b --- /dev/null +++ b/crates/coreav-hevc/src/thread_diagnostics/rows.rs @@ -0,0 +1,222 @@ +use std::cell::RefCell; + +use crate::{Error, PictureInfo, Result, dpb::ReferenceIdentity}; + +#[derive(Clone, Copy, Default)] +#[doc(hidden)] +pub struct RowReference { + pub index: u64, + pub poc: i32, + pub sample_row: Option, + pub motion_row: Option, +} + +#[derive(Default)] +#[doc(hidden)] +pub struct RowDependency { + pub references: [Option; 16], + pub sample_calls: u64, + pub motion_calls: u64, +} + +#[doc(hidden)] +pub struct RowPicture { + pub index: u64, + pub poc: i32, + pub info: PictureInfo, + pub rows: Vec, +} + +#[derive(Clone, Copy)] +pub(crate) struct ReadContext { + pub picture: u64, + pub row: usize, + pub reference: ReferenceIdentity, + pub reference_ctb_log2: u32, + pub scale: usize, +} + +struct Recorder { + pictures: [Option; 32], + context: Option, + error: Option, +} + +thread_local! { + static ROWS: RefCell> = const { RefCell::new(None) }; +} + +#[doc(hidden)] +pub fn begin_rows() -> Result<()> { + ROWS.with_borrow_mut(|state| { + if state.is_some() { + return Err(Error::InvalidData("row_trace_active")); + } + *state = Some(Recorder { + pictures: [const { None }; 32], + context: None, + error: None, + }); + Ok(()) + }) +} + +#[doc(hidden)] +pub fn take_rows(max_workers: usize) -> Result> { + let state = ROWS.take().ok_or(Error::InvalidData("row_trace_missing"))?; + if max_workers != 1 { + return Err(Error::Unsupported("row_trace_workers")); + } + if let Some(error) = state.error { + return Err(error); + } + let mut pictures = Vec::new(); + pictures + .try_reserve_exact(32) + .map_err(|_| Error::Limit("row_trace_memory"))?; + pictures.extend(state.pictures.into_iter().flatten()); + Ok(pictures) +} + +pub(crate) fn check_row_threads(threads: usize) -> Result<()> { + ROWS.with_borrow_mut(|state| { + let Some(state) = state else { + return Ok(()); + }; + if threads == 1 { + return Ok(()); + } + state.error = Some(Error::Unsupported("row_trace_workers")); + Err(Error::Unsupported("row_trace_workers")) + }) +} + +pub(crate) fn row_picture(index: u64, poc: i32, info: PictureInfo) -> Result<()> { + ROWS.with_borrow_mut(|state| { + let Some(state) = state else { + return Ok(()); + }; + if info.wpp { + return Err(Error::Unsupported("row_trace_wpp")); + } + let count = info.height.div_ceil(1 << info.ctb_log2) as usize; + if count == 0 || count > 2048 { + return Err(Error::Limit("row_trace_rows")); + } + let slot = state + .pictures + .get_mut(index as usize) + .ok_or(Error::Limit("row_trace_pictures"))?; + if slot.is_some() { + return Err(Error::InvalidData("row_trace_duplicate_picture")); + } + let mut rows = Vec::new(); + rows.try_reserve_exact(count) + .map_err(|_| Error::Limit("row_trace_memory"))?; + rows.resize_with(count, RowDependency::default); + *slot = Some(RowPicture { + index, + poc, + info, + rows, + }); + Ok(()) + }) +} + +pub(crate) struct ReadGuard { + previous: Option, + active: bool, +} + +impl ReadGuard { + pub(crate) fn new(context: ReadContext) -> Self { + ROWS.with_borrow_mut(|state| match state { + Some(state) => Self { + previous: state.context.replace(context), + active: true, + }, + None => Self { + previous: None, + active: false, + }, + }) + } +} + +impl Drop for ReadGuard { + fn drop(&mut self) { + if self.active { + ROWS.with_borrow_mut(|state| { + if let Some(state) = state { + state.context = self.previous; + } + }); + } + } +} + +fn record(last: usize, samples: bool) { + ROWS.with_borrow_mut(|state| { + let Some(state) = state else { + return; + }; + if state.error.is_some() { + return; + } + let result = (|| { + let context = state.context.ok_or(Error::Internal("row_trace_context"))?; + let row = state + .pictures + .get_mut(context.picture as usize) + .and_then(Option::as_mut) + .and_then(|picture| picture.rows.get_mut(context.row)) + .ok_or(Error::Internal("row_trace_row"))?; + let at = row + .references + .iter() + .position(|entry| entry.is_none_or(|entry| entry.index == context.reference.index)) + .ok_or(Error::Limit("row_trace_references"))?; + let reference = row + .references + .get_mut(at) + .ok_or(Error::Internal("row_trace_reference"))? + .get_or_insert(RowReference { + index: context.reference.index, + poc: context.reference.poc, + ..RowReference::default() + }); + if reference.poc != context.reference.poc { + return Err(Error::Internal("row_trace_reference_poc")); + } + let maximum = if samples { + last.checked_mul(context.scale) + .and_then(|value| value.checked_add(context.scale - 1)) + .ok_or(Error::Limit("row_trace_coordinate"))? + } else { + last + } >> context.reference_ctb_log2; + let (bound, calls) = if samples { + (&mut reference.sample_row, &mut row.sample_calls) + } else { + (&mut reference.motion_row, &mut row.motion_calls) + }; + *bound = Some(bound.map_or(maximum, |value| value.max(maximum))); + *calls = calls + .checked_add(1) + .ok_or(Error::Limit("row_trace_calls"))?; + Ok(()) + })(); + if let Err(error) = result { + state.error = Some(error); + } + }); +} + +pub(crate) fn sample_read(_first: usize, last: usize) { + record(last, true); +} + +pub(crate) fn motion_read(last: usize) { + record(last, false); +} diff --git a/crates/coreav-hevc/test-support/pcm.rs b/crates/coreav-hevc/test-support/pcm.rs index ada002281..b7fafa918 100644 --- a/crates/coreav-hevc/test-support/pcm.rs +++ b/crates/coreav-hevc/test-support/pcm.rs @@ -24,7 +24,7 @@ pub(super) struct PcmFixture { pub(super) planes: [Vec; 3], } -fn fixture_profile(writer: &mut Writer, depth: u32) { +pub(super) fn fixture_profile(writer: &mut Writer, depth: u32) { writer.u(2, 0); writer.flag(false); writer.u(5, if depth == 10 { 2 } else { 1 }); diff --git a/crates/coreav-hevc/test-support/raster.rs b/crates/coreav-hevc/test-support/raster.rs new file mode 100644 index 000000000..20ef1038e --- /dev/null +++ b/crates/coreav-hevc/test-support/raster.rs @@ -0,0 +1,111 @@ +use super::{Result, pcm, writer::Writer}; + +fn profile(writer: &mut Writer, rows: u32) { + writer.u(2, 0); + writer.flag(false); + writer.u(5, 1); + writer.u(32, 0); + writer.u(4, 0); + writer.u(32, 0); + writer.u(12, 0); + writer.u(8, if rows <= 256 { 120 } else { 150 }); +} + +pub(super) fn stream(predicted: bool) -> Result>> { + stream_geometry(predicted, 32, 4) +} + +pub(super) fn stream_geometry(predicted: bool, rows: u32, dpb: u32) -> Result>> { + assert!((1..=512).contains(&rows)); + assert!((1..=16).contains(&dpb)); + let fixture = pcm::pcm_fixture(8, 8, false)?; + let mut vps = Writer::default(); + vps.u(4, 0); + vps.flag(true); + vps.flag(true); + vps.u(6, 0); + vps.u(3, 0); + vps.flag(true); + vps.u(16, 0xffff); + profile(&mut vps, rows); + vps.flag(false); + for value in [dpb - 1, dpb - 1, 0] { + vps.ue(value); + } + vps.u(6, 0); + vps.ue(0); + vps.flag(false); + vps.flag(false); + let mut sps = Writer::default(); + sps.u(4, 0); + sps.u(3, 0); + sps.flag(true); + profile(&mut sps, rows); + for value in [0, 1, 16, rows * 16] { + sps.ue(value); + } + sps.flag(false); + sps.ue(0); + sps.ue(0); + sps.ue(0); + sps.flag(false); + for value in [dpb - 1, dpb - 1, 0, 1, 0, 0, 1, 0, 0] { + sps.ue(value); + } + for flag in [false, false, false, true] { + sps.flag(flag); + } + sps.u(4, 7); + sps.u(4, 7); + sps.ue(1); + sps.ue(0); + sps.flag(true); + sps.ue(0); + for _ in 0..5 { + sps.flag(false); + } + let mut nals = vec![ + pcm::escaped(32, &vps.rbsp()), + pcm::escaped(33, &sps.rbsp()), + fixture.sets[2].clone(), + ]; + for picture in 0..2 { + let inter = predicted && picture != 0; + for row in 0..rows { + let mut header = Writer::default(); + header.flag(row == 0); + if !inter { + header.flag(false); + } + header.ue(0); + if row != 0 { + header.u(rows.next_power_of_two().trailing_zeros(), row); + } + header.ue(if inter { 1 } else { 2 }); + if inter { + header.u(4, 1); + header.flag(false); + header.ue(1); + header.ue(0); + header.ue(0); + header.flag(true); + header.flag(false); + header.ue(4); + } + header.se(0); + let mut rbsp = header.rbsp(); + if inter { + rbsp.extend([0xfe, 0xe0]); + } else { + rbsp.extend([0x86, 0x80]); + rbsp.extend(core::iter::repeat_n( + (37 + picture * 5 + row) as u8, + 16 * 16 * 3 / 2, + )); + rbsp.extend([0xfe, 0x80]); + } + nals.push(pcm::escaped(if inter { 1 } else { 19 }, &rbsp)); + } + } + Ok(nals) +} diff --git a/crates/coreav-hevc/test-support/writer.rs b/crates/coreav-hevc/test-support/writer.rs index 487c62904..3bf41c525 100644 --- a/crates/coreav-hevc/test-support/writer.rs +++ b/crates/coreav-hevc/test-support/writer.rs @@ -48,7 +48,7 @@ impl Writer { pub(crate) fn rbsp(mut self) -> Vec { self.flag(true); - while !self.len().is_multiple_of(8) { + while self.len() & 7 != 0 { self.flag(false); } self.bytes() diff --git a/crates/coreav-kernels-hevc/Cargo.toml b/crates/coreav-kernels-hevc/Cargo.toml index bc5d72837..a26402955 100644 --- a/crates/coreav-kernels-hevc/Cargo.toml +++ b/crates/coreav-kernels-hevc/Cargo.toml @@ -18,6 +18,7 @@ test-support = [] [lints.rust] unsafe_code = "deny" missing_docs = "warn" +unexpected_cfgs = { level = "warn", check-cfg = ["cfg(hevc_thread_diagnostics)"] } [lints.clippy] all = { level = "warn", priority = -1 } diff --git a/crates/coreav-kernels-hevc/src/inter.rs b/crates/coreav-kernels-hevc/src/inter.rs index 1d5ee8ec2..666f9e302 100644 --- a/crates/coreav-kernels-hevc/src/inter.rs +++ b/crates/coreav-kernels-hevc/src/inter.rs @@ -22,20 +22,48 @@ pub struct RefPlane<'a> { stride: usize, width: usize, height: usize, + #[cfg(hevc_thread_diagnostics)] + read_observer: Option, } impl<'a> RefPlane<'a> { /// Checks positive dimensions up to i32::MAX, stride and the last active sample. /// Sample depth is checked only over the interpolation support at invocation. pub fn new(data: &'a [u16], stride: usize, width: usize, height: usize) -> Result { - check_rectangle(data.len(), stride, width, height)?; + Self::prefix(data, stride, width, height, height) + } + + #[doc(hidden)] + pub fn prefix( + data: &'a [u16], + stride: usize, + width: usize, + height: usize, + rows: usize, + ) -> Result { + if height > i32::MAX as usize { + return Err(Error::Buffer); + } + if rows > height { + return Err(Error::Rectangle); + } + check_rectangle(data.len(), stride, width, rows)?; Ok(Self { data, stride, width, height, + #[cfg(hevc_thread_diagnostics)] + read_observer: None, }) } + + #[cfg(hevc_thread_diagnostics)] + #[doc(hidden)] + pub fn diagnostic_reads(mut self, observer: fn(usize, usize)) -> Self { + self.read_observer = Some(observer); + self + } } /// A resolved signed motion vector, without reference-list or candidate syntax. @@ -62,6 +90,73 @@ pub struct InterPrediction { pub motion: MotionVector, } +#[doc(hidden)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InterSupport { + start: [i64; 2], + phase: [i16; 2], + first: [usize; 2], + last: [usize; 2], +} + +impl InterSupport { + pub fn rows(self) -> core::ops::RangeInclusive { + self.first[1]..=self.last[1] + } +} + +impl InterPrediction { + #[doc(hidden)] + pub fn support( + self, + reference_width: usize, + reference_height: usize, + width: usize, + height: usize, + ) -> Result { + if self.x > i32::MAX as usize + || self.y > i32::MAX as usize + || reference_width == 0 + || reference_height == 0 + || reference_width > i32::MAX as usize + || reference_height > i32::MAX as usize + { + return Err(Error::Buffer); + } + let (bits, first_tap, taps, limit) = match self.component { + Component::Luma => (2, -3_i64, 8_i64, 64), + Component::Chroma => (3, -1_i64, 4_i64, 32), + }; + if width == 0 || height == 0 || width > limit || height > limit { + return Err(Error::Rectangle); + } + let mask = (1 << bits) - 1; + let axes = [ + (self.x, self.motion.x, width, reference_width), + (self.y, self.motion.y, height, reference_height), + ] + .map(|(origin, motion, output, reference)| { + let phase = motion & mask; + let base = origin as i64 + i64::from(motion >> bits); + let start = base + if phase == 0 { 0 } else { first_tap }; + let end = base + output as i64 - 1 + if phase == 0 { 0 } else { first_tap + taps - 1 }; + ( + start, + phase, + start.clamp(0, reference as i64 - 1) as usize, + end.clamp(0, reference as i64 - 1) as usize, + ) + }); + let [(xs, xp, xf, xl), (ys, yp, yf, yl)] = axes; + Ok(InterSupport { + start: [xs, ys], + phase: [xp, yp], + first: [xf, yf], + last: [xl, yl], + }) + } +} + fn coefficients(component: Component, phase: i16) -> Result<&'static [i32], Error> { let index = usize::try_from(phase - 1).map_err(|_| Error::Buffer)?; match component { @@ -151,6 +246,280 @@ pub fn interpolate_inter_scalar( interpolate(config, reference, output, false, None) } +#[cfg(test)] +mod support_tests { + use super::*; + + #[test] + fn clipped_support_rows_are_exact_at_both_depths_and_picture_edges() -> Result<(), Error> { + for bits in [8, 10] { + for (component, automatic) in [ + (Component::Luma, false), + (Component::Luma, true), + (Component::Chroma, false), + (Component::Chroma, true), + ] { + for y in [0, 8, 31] { + for motion in [ + i16::MIN, + -9, + -8, + -7, + -4, + -3, + -1, + 0, + 1, + 3, + 4, + 7, + 8, + 9, + i16::MAX, + ] { + let config = InterPrediction { + depth: BitDepth::new(bits)?, + component, + x: 8, + y, + motion: MotionVector { + x: motion, + y: motion, + }, + }; + let rows = config.support(32, 32, 8, 4)?.rows(); + let source = [42u16; 32 * 32]; + let mut expected = [0; 8 * 4]; + interpolate( + config, + RefPlane::new(&source, 32, 32, 32)?, + RectMut::new(&mut expected, 8, 8, 4)?, + automatic, + None, + )?; + for row in 0..32 { + let mut source = [42u16; 32 * 32]; + source[row * 32..(row + 1) * 32].fill(config.depth.max_sample() + 1); + let mut output = [7; 8 * 4]; + let result = interpolate( + config, + RefPlane::new(&source, 32, 32, 32)?, + RectMut::new(&mut output, 8, 8, 4)?, + automatic, + None, + ); + if rows.contains(&row) { + assert_eq!(result, Err(Error::Sample)); + assert_eq!(output, [7; 8 * 4]); + } else { + result?; + assert_eq!(output, expected); + } + } + } + } + } + } + Ok(()) + } + + #[test] + fn support_retains_fractional_taps_even_when_an_endpoint_coefficient_is_zero() + -> Result<(), Error> { + for (component, motion, first, last) in [ + (Component::Luma, 0, 8, 11), + (Component::Luma, 1, 5, 15), + (Component::Luma, 2, 5, 15), + (Component::Luma, 3, 5, 15), + (Component::Luma, -1, 4, 14), + (Component::Luma, -4, 7, 10), + (Component::Chroma, 1, 7, 13), + (Component::Chroma, 7, 7, 13), + (Component::Chroma, -1, 6, 12), + (Component::Luma, i16::MIN, 0, 0), + (Component::Luma, i16::MAX, 31, 31), + ] { + let config = InterPrediction { + depth: BitDepth::new(8)?, + component, + x: 8, + y: 8, + motion: MotionVector { x: 0, y: motion }, + }; + assert_eq!(config.support(32, 32, 8, 4)?.rows(), first..=last); + } + Ok(()) + } + + #[test] + fn prefix_references_keep_full_picture_clipping_and_refuse_unpublished_rows() + -> Result<(), Error> { + let source = [42u16; 32 * 32]; + for component in [Component::Luma, Component::Chroma] { + for motion in [i16::MIN, -1, 0, 1, i16::MAX] { + for y in [0, 8, 31] { + let config = InterPrediction { + depth: BitDepth::new(8)?, + component, + x: 8, + y, + motion: MotionVector { x: 0, y: motion }, + }; + let rows = config.support(32, 32, 8, 4)?.rows(); + let available = *rows.end() + 1; + let mut expected = [0; 32]; + interpolate( + config, + RefPlane::new(&source, 32, 32, 32)?, + RectMut::new(&mut expected, 8, 8, 4)?, + false, + None, + )?; + for borrowed in [false, true] { + let mut row_storage = [i32::MIN; 512]; + let mut actual = [0; 32]; + interpolate( + config, + RefPlane::prefix(&source[..available * 32], 32, 32, 32, available)?, + RectMut::new(&mut actual, 8, 8, 4)?, + false, + borrowed.then_some(row_storage.as_mut_slice()), + )?; + assert_eq!(actual, expected); + if available > 1 { + let mut actual = [99; 32]; + let before = row_storage; + assert_eq!( + interpolate( + config, + RefPlane::prefix( + &source[..(available - 1) * 32], + 32, + 32, + 32, + available - 1 + )?, + RectMut::new(&mut actual, 8, 8, 4)?, + false, + borrowed.then_some(row_storage.as_mut_slice()), + ), + Err(Error::Buffer) + ); + assert_eq!(actual, [99; 32]); + assert_eq!(row_storage, before); + } + } + } + } + } + assert!(RefPlane::prefix(&source, 32, 32, 32, 0).is_err()); + assert!(RefPlane::prefix(&source, 32, 32, 32, 33).is_err()); + Ok(()) + } + + #[test] + fn support_refuses_invalid_geometry_without_a_reference_buffer() -> Result<(), Error> { + let config = InterPrediction { + depth: BitDepth::new(8)?, + component: Component::Luma, + x: 0, + y: 0, + motion: MotionVector { x: 0, y: 0 }, + }; + for (rw, rh, width, height, error) in [ + (0, 32, 8, 4, Error::Buffer), + (32, 0, 8, 4, Error::Buffer), + (usize::MAX, 32, 8, 4, Error::Buffer), + (32, usize::MAX, 8, 4, Error::Buffer), + (32, 32, 0, 4, Error::Rectangle), + (32, 32, 8, 0, Error::Rectangle), + (32, 32, 65, 4, Error::Rectangle), + (32, 32, 8, 65, Error::Rectangle), + ] { + assert_eq!(config.support(rw, rh, width, height), Err(error)); + } + assert_eq!( + InterPrediction { + x: usize::MAX, + ..config + } + .support(32, 32, 8, 4), + Err(Error::Buffer) + ); + assert_eq!( + InterPrediction { + component: Component::Chroma, + ..config + } + .support(32, 32, 33, 4), + Err(Error::Rectangle) + ); + Ok(()) + } +} + +#[cfg(all(test, hevc_thread_diagnostics))] +mod tests { + use super::*; + use core::sync::atomic::{AtomicUsize, Ordering}; + + static READS: AtomicUsize = AtomicUsize::new(usize::MAX); + + fn observed(first: usize, last: usize) { + READS.store(first * 64 + last, Ordering::Relaxed); + } + + #[test] + fn diagnostic_read_bounds_match_interpolation_support() -> Result<(), Error> { + for (component, mv_x, mv_y, expected) in [ + (Component::Luma, 0, 0, (8, 11)), + (Component::Luma, 1, 0, (8, 11)), + (Component::Luma, 0, 1, (5, 15)), + (Component::Luma, 0, -1, (4, 14)), + (Component::Luma, 0, -4, (7, 10)), + (Component::Chroma, 0, 1, (7, 13)), + (Component::Chroma, 0, -1, (6, 12)), + (Component::Luma, 0, i16::MIN, (0, 0)), + (Component::Luma, 0, i16::MAX, (31, 31)), + ] { + for automatic in [false, true] { + let mut source = [42u16; 32 * 32]; + let mut output = [0i32; 8 * 4]; + let config = InterPrediction { + depth: BitDepth::new(8)?, + component, + x: 8, + y: 8, + motion: MotionVector { x: mv_x, y: mv_y }, + }; + READS.store(usize::MAX, Ordering::Relaxed); + interpolate( + config, + RefPlane::new(&source, 32, 32, 32)?.diagnostic_reads(observed), + RectMut::new(&mut output, 8, 8, 4)?, + automatic, + None, + )?; + assert_eq!(READS.load(Ordering::Relaxed), expected.0 * 64 + expected.1); + *source.get_mut(expected.1 * 32 + 8).ok_or(Error::Buffer)? = 256; + let before = output; + assert_eq!( + interpolate( + config, + RefPlane::new(&source, 32, 32, 32)?.diagnostic_reads(observed), + RectMut::new(&mut output, 8, 8, 4)?, + automatic, + None, + ), + Err(Error::Sample) + ); + assert_eq!(output, before); + } + } + Ok(()) + } +} + fn interpolate( config: InterPrediction, reference: RefPlane<'_>, @@ -160,31 +529,22 @@ fn interpolate( ) -> Result<(), Error> { #[cfg(all(not(feature = "scalar"), target_arch = "x86_64"))] let vector = _automatic.then(crate::simd::Avx2::detect).flatten(); - if config.x > i32::MAX as usize || config.y > i32::MAX as usize { - return Err(Error::Buffer); + let support = config.support( + reference.width, + reference.height, + output.width, + output.height, + )?; + let [phase_x, phase_y] = support.phase; + let [x_start, y_start] = support.start; + #[cfg(hevc_thread_diagnostics)] + if let Some(observer) = reference.read_observer { + observer(*support.rows().start(), *support.rows().end()); } - let (fraction_bits, start, taps, limit) = match config.component { - Component::Luma => (2, -3_i64, 8_i64, 64), - Component::Chroma => (3, -1_i64, 4_i64, 32), - }; - if output.width > limit || output.height > limit { - return Err(Error::Rectangle); - } - let mask = (1 << fraction_bits) - 1; - let phase_x = config.motion.x & mask; - let phase_y = config.motion.y & mask; - let x_base = config.x as i64 + i64::from(config.motion.x >> fraction_bits); - let y_base = config.y as i64 + i64::from(config.motion.y >> fraction_bits); - let x_start = x_base + if phase_x == 0 { 0 } else { start }; - let y_start = y_base + if phase_y == 0 { 0 } else { start }; - let x_end = x_base + output.width as i64 - 1 + if phase_x == 0 { 0 } else { start + taps - 1 }; - let y_end = y_base + output.height as i64 - 1 + if phase_y == 0 { 0 } else { start + taps - 1 }; - for y in - y_start.clamp(0, reference.height as i64 - 1)..=y_end.clamp(0, reference.height as i64 - 1) - { - let begin = y as usize * reference.stride; - let left = x_start.clamp(0, reference.width as i64 - 1) as usize; - let right = x_end.clamp(0, reference.width as i64 - 1) as usize; + for y in support.rows() { + let begin = y * reference.stride; + let left = support.first[0]; + let right = support.last[0]; let row = reference .data .get(begin + left..=begin + right) diff --git a/crates/coreav-kernels-hevc/src/lib.rs b/crates/coreav-kernels-hevc/src/lib.rs index 70736098d..e11f57c73 100644 --- a/crates/coreav-kernels-hevc/src/lib.rs +++ b/crates/coreav-kernels-hevc/src/lib.rs @@ -56,6 +56,9 @@ pub use filter::{ #[doc(hidden)] pub use filter::{deblock_chroma_scalar, deblock_luma_scalar}; #[cfg(feature = "native-hevc")] +#[doc(hidden)] +pub use inter::InterSupport; +#[cfg(feature = "native-hevc")] pub use inter::{ InterPrediction, MotionVector, PredictionWeight, RefPlane, interpolate_inter, interpolate_inter_with_rows, predict_bi, predict_uni, weight_bi, weight_uni, diff --git a/docs/agents/ci.md b/docs/agents/ci.md index 50fc1cace..7c1d3c9c7 100644 --- a/docs/agents/ci.md +++ b/docs/agents/ci.md @@ -39,6 +39,7 @@ List development caches with `docker volume ls --filter name=coreav-dev-`. After | Rule | Enforced by | |---|---| +| HEVC diagnostic-only extraction and kernel tests execute on ordinary Linux pull requests rather than only manual measurement dispatches | The headers job compiles both owners with `hevc_thread_diagnostics` and requires exact successful header, row and kernel test witnesses with `pipefail`; `test_ordinary_pr_ci_executes_cfg_tests_and_excludes_the_private_bridge_from_releases` checks that wiring and both release-graph exclusions. Ordinary Rust types cannot enforce workflow selection | | The embedded Python command recognizes CLI-only native encoder cfg values without exposing or enabling that encoder | `coreav-py/build.rs` declares only the exact known cfg value; `cargo clippy --workspace --all-targets -- -D warnings` rejects drift in the shared command source. The Python Cargo feature interface and default dependency graph remain unchanged | | MP3 byte parity executes real generated media in required Linux CI rather than passing with an unbuilt or unselected transcode test | The explicit `check` step enables `ffmpeg-fallback`, selects the exact MP3 test after oracle qualification and requires its success witness with pipeline failure propagation. `test_required_linux_mp3_comparison_rejects_zero_tests_and_failed_commands` rejects missing, failed and unrelated tests, and a failed Cargo command that printed the witness. Cargo features and test selection are runtime CI facts, not properties of a Rust test body | | A development image verifies its Rust bootstrap before executing it | `scripts/dev/Dockerfile` selects versioned rustup-init binaries and committed architecture-specific SHA-256 values; `test_corrupt_bootstrap_is_not_executed` executes its shell step with a corrupt download for both architectures and requires checksum failure without execution. The compiler toolchain remains owned by `rust-toolchain.toml` | @@ -46,7 +47,7 @@ List development caches with `docker volume ls --filter name=coreav-dev-`. After | A credentialed manual workflow that builds a dispatched commit admits only commits on this repository's branches, so pull-request code from forks never runs beside corpus credentials | The HEVC reference workflow's admission step refuses a candidate no branch contains before the build and every credentialed step; `test_candidate_must_be_on_a_branch_before_it_builds_or_sees_credentials` pins that order. Writers already hold the repository's secrets, so this closes the only path that widens access. Workflow step order cannot be encoded in Rust types | | A daily full run skips only for a successful same-workflow main-push attempt at its exact SHA; weekly full runs never skip for coverage, and a covered daily never masquerades as missing full work | `merge-group-result` owns bounded search and revalidation; `ci_full.covers` owns complete job/proof validation shared with the pinned-claim reporter. `test_full_coverage.py` rejects wrong identities, non-successes, missing legs, reruns and malformed claims, while preserving cancellation and manual evidence. An exact daily-cron gate excludes each weekly trigger; the weekly scheduling test supplies a covering success and still requires every registered full job, and the reporter test preserves the normal failure path. Workflow selection is external to Python types; weekly full work checks drift while main stays unchanged | | Merge-group reuse proves both the actual PR-tested tree and base at the newest successful run attempt; missing evidence runs the ordinary checks and advisory scans stay fresh | `merge-group-result` owns checkout-bound attempt receipts and metadata lookup. `test_merge_group_result.py` rejects wrong identities, stale or duplicate receipts, incomplete indexes, newer failures/reruns and head races; `test_main_ci_plan.py` keeps reused groups out of fixture production and main-only maintenance, retains main's baseline guard, and executes root/fuzz advisory commands and their skipped/failure witness. Workflow self-tests reject non-required merge-group triggers. Tree equality alone cannot prove changed-path selection, a base comparison or a current advisory database | -| A closed-reader stdout test cannot inherit a pipe reader through another concurrently spawned test process | `a_closed_standard_output_exits_141_quietly` and `tiny_pcm_stdout_reports_a_closed_reader_during_final_flush` run their unchanged pipe assertions in single-test subprocesses and require exact success witnesses, so zero selected tests fail. Each subprocess owns pipe creation; the probes still require exit 141 and empty stderr on every platform. A descriptor's local lifetime cannot constrain inheritance by another thread's concurrent spawn | +| A closed-reader stdout test cannot inherit a pipe reader through another concurrently spawned test process | `tiny_pcm_stdout_reports_a_closed_reader_during_final_flush` and `a_closed_standard_output_exits_141_quietly` run their unchanged pipe assertions in single-test subprocesses and require the exact success witness, so zero selected tests fail. Each subprocess owns pipe creation; the probes still require exit 141 and empty stderr on every platform. A descriptor's local lifetime cannot constrain inheritance by another thread's concurrent spawn; separate test binaries need separate process isolation | | Required fixture branches consume one fresh run-bound set, H.264 smoke has exactly eight names and complete results, and full H.264 and transcode stay independent of queue reuse | `scripts/tests/test_core_ci_tiers.py` checks producer freshness on cache hits, artifact consumption without regeneration, moved command predicates, exact smoke membership and reports, full events, thread and safe-mode commands, and full-report failures. Workflow ownership is structural; types cannot govern YAML scheduling | | Full-suite reporting uses trusted main code and metadata, and final-head verification requires all three newest manual suites with complete matrix and proof-step success | `scripts/tests/test_ci_full.py` covers event and repository refusal, bot-owned issue deduplication, fixed public text, API and pagination failures, latest dispatch and attempt identity, skipped proofs, missing matrix legs, and unchanged PR heads. Live token permissions and first-merge dispatch registration remain operational checks, not test-proven claims | | VP9 routine original-file smoke and full-tier scheduling retain exact coverage | `scripts/vp9_smoke.py` owns the exact-eight selection, checksums, and completion checks. `scripts/tests/test_vp9_smoke.py` covers membership, original bytes, bounded retries, comparator execution, refusal or zero-frame failures, and closed stage/reason diagnostics without arbitrary exception arguments. `scripts/tests/test_vp9_ci_tiers.py` checks corpus events, PR and full fuzz budgets, proof-step wiring, and metadata-only reporting. Types cannot govern workflow scheduling | diff --git a/docs/agents/codecs.md b/docs/agents/codecs.md index 2031c23dd..694f9e1ce 100644 --- a/docs/agents/codecs.md +++ b/docs/agents/codecs.md @@ -4,12 +4,29 @@ H.264, AAC, Opus, VP9, AC-3, MP3 and AudioToolbox behavior; HEVC decoding and pl | Rule | Enforced by | |---|---| +| Zero future HEVC lag and row-epoch retirement release optional record capacity, not just record lengths | `Scheduler::replace_row_plan` and `retire_rows` own both the mode change and reclamation. `zero_lag_and_retirement_release_optional_record_capacities` checks pending capacity, job capacity and exact scheduler bytes at both transitions; replacing either drop with `clear` fails it. State ownership keeps callers from separating the transition from reclamation; runtime `Vec` capacities still need the executable assertion | +| Native HEVC lag qualification cannot depend on a compiler version or panic when a capacity probe fails | `CapacityProbe` checks concrete request sizes at admission; no build-script identity gate exists. `capacity_probe_mismatch_preserves_legacy_frames_and_summary` forces the production qualification branch to fail and checks legacy selection, frames and semantic counters. Pending, job and output-ring growth is removed from the premise by reserving and charging actual capacities before commitment; one-shot future-size prediction remains explicitly runtime-qualified rather than a std guarantee | +| A raster producer owns deblock ordering, unfiltered intra borders and pre-SAO borders before publishing immutable rows | `Producer` keeps its `Window` private and advances it through the shared entropy traversal. `private_filter_bands_match_whole_picture_samples_and_counts` covers CTU16/32/64, both depths and partial final rows. `raster_intra_top_border_precedes_vertical_deblocking` asserts that an eight-sample transform edge changes the last row before the next CTU intra-predicts, then compares complete samples and counters; removing the saved top border fails it. Reversing deblock order, omitting the saved SAO border, overwriting a published prefix or resetting QP also fails the producer regressions. Types cannot prove sample-dependent spatial ordering, so executable sample and entropy comparisons enforce it | +| Row readers use separate motion and component-sample frontiers, and every abandoned producer wakes its dependents | A non-clonable `RowPublisher` owns commits and fails its store on drop. Scoped `ReferenceRead` guards pin the checked prefix; waits release the progress lock before taking storage. Row-store regressions reject motion-only sample readiness, stale generations, prefix writes and missing failure notifications. `raster_reference_snapshots_reject_self_dependencies_before_any_wait` enforces descending coded identities. Condition-variable wakeups need runtime tests beyond the publisher type | +| Every current-picture cell read in a bounded HEVC row window uses the same translated owner, including WPP setup before the CTU walker starts | `SampleRows::cell_index` and `cell` serve both `start_ctu` and `Walker`. `private_row_wpp_above_right_uses_the_window_with_and_without_a_checkpoint` checks a 32-pixel-wide WPP row at nonzero origin with both checkpoint states, cursor/bin identity and restored contexts; replacing the setup lookup with global `Frame::cell` fails it. One translation owner prevents a second unchecked pre-walker path | +| A HEVC lag certificate covers every permitted SPS continuation, and a later activation gives up optional state and future lag before refusing | `rows::Admission` owns the checked envelope, including old-debt migration and preactivation entry bounds; the parser's existing active-SPS identity selects the gate after prior output. `Observer::slice` caps adaptive atomic work to that admitted envelope, counts the scheduler owner, reserves its admitted runtime ceiling and clears optional cache. Native cadence, zero-lag/debt-retirement and growth-refusal regressions preserve the prefix, accept an unused SPS and compare the same zero-debt continuation. `virtual_receive_offsets_follow_unreturned_old_geometry_debt` catches charging the physical front instead of the unreturned serial ordinal. Capacity qualification and producer allocation checks cover the standard-library premise; runtime geometry and ownership need these checks beyond types | +| A checked-wrapper limit cannot overtake an earlier deferred native HEVC reconstruction error | The opt-in hidden settlement capability runs before `Checked` tears down a limit failure; `checked_packet_refusal_settles_earlier_entropy_failure` uses `coreav_decode::open` with truncated entropy followed by an oversized packet at 1/2/4/8 requested threads. Backend-only tests bypass wrapper refusals, so the public-open regression is necessary | +| HEVC teardown documentation distinguishes an optional cooperative work deadline from a hard join timeout | `DecodeLimits::default` has no deadline and `Scheduler::drop` joins workers without interruption; deadline tests cover cooperative work checks only. Not enforced for prose: review must not claim that the work deadline interrupts a join or bounds default teardown | +| A shared HEVC reference publication identifies the same real picture as scalar reference access, never a generated same-index placeholder | `Picture::matches_reference` is the single predicate for immutable lookup and mutable publication. `sharing_real_cra_samples_skips_generated_same_index_references` retains a generated CRA follower beside the actual CRA, validates RADL access and exact sample/Arc identity, and rejects mismatched POC/index. Generated external identities need the behavioral regression in addition to the shared predicate | +| An output bump cannot expose a picture before an earlier coded picture's pending failure | Complete-reference delivery reconciles through `before_picture`; private row delivery captures the entire issued coded fence and fills only its confirmed prefix. The four-picture bump regression catches the former predictor removal. The dependent-row regression catches an own-picture-only fence leaking an output after a later coded CRC failure. External header order and deferred failure require these behavioral checks beyond ownership types | +| Informational HEVC SEI and ordinary decoded-picture hashes preserve eligible picture overlap without changing hash or error order | The caller parses SEI; a charged inline `OwnedSlice` hash slot retains sample-dependent payload type 132 at its slice position. `per_picture_sei_keeps_real_overlap_and_ordered_hash_failures` proves actual overlap with per-picture prefix/suffix metadata and hashes, and checks duplicate, malformed and earlier entropy failures. Complete conformance rejects WPP-only coverage. Types retain hash ownership, but external SEI ordering and runtime overlap need these checks | +| HEVC scheduler concurrency tests execute their requested worker count even on a one-CPU host, without weakening the public CPU clamp | The shared private `test_decoder` constructor sets the fixture's bounded worker count before initialization. `public_worker_boundaries_normalize_limits_and_existing_sessions` separately checks public construction and reconfiguration. A test fixture's requested concurrency and the deployment host's capacity are distinct inputs | +| HEVC picture jobs preserve the original physical admission allowance and ordinary polling does not discard WPP reuse or flush queued slices | `SerialAccounting` fixes the original typed summary layout; the DPB storage-layout and session-envelope tests reject bookkeeping growth. Independent cold memory/bin tests check scheduled admission, and `session_synchronization_and_polling_preserve_warm_wavefront_workers` checks both APIs with real cache hits. Runtime allocator capacity and host layout need tests in addition to ownership types | +| Scheduled HEVC input retains the serial first error even when a caller later fails framing, reading or output validation | `synchronize_pending` reconciles the earlier prefix without premature boundary finalization, and the backend routes terminal operations through `terminal_result`. Reversed worker faults, caller-validation, backend framing and example read-error regressions check error identity and output prefixes; removing the example reconciliation returns the later read error and fails its regression. Types cannot establish external I/O order | | A CRF numeric-scale claim is checked against fixed-QP bits and selected picture-QP distributions, including the IDR bit contribution | The combined adjustment has a -1 floor, so refresh and activity boosts cannot stack. `low_activity_crf_never_stacks_refresh_and_activity_quality_boosts` requires complete low-activity streams to equal CQP N-1; band tests and the independent reader check all QP boundaries. Reference measurements check content-dependent rates because decoder equality and Rust types cannot prove a scale claim. Source-pixel activity is not resolution invariant, so reference claims retain tested dimensions | | H.264 decision unwind and ordinary errors have the same bounded result at one or multiple workers, including narrow and single-row geometry | Both scheduler paths catch unwind; `one_worker_and_parallel_paths_report_the_same_compute_failures` injects errors and panic at the beginning, middle and end for every effective worker shape. Runtime unwind and geometry cannot be encoded in the candidate type | | H.264 thread-start refusal preserves results from started workers and uses the caller if none starts | A single shared row loop owns both fallback and worker execution. `zero_and_partial_worker_start_failures_preserve_results_and_errors` injects errors through the real spawn-result branch and checks complete tables and decision-error selection; OS resource refusal needs runtime coverage | | H.264 neighbour reads require published wavefront dependencies rather than only thread-count identity | `unavailable_wavefront_neighbours_are_refused_before_border_reads` removes each required neighbour; `a_delayed_top_right_is_published_before_its_dependent_read` delays publication under concurrent rows. The eight-worker textured P round trip checks complete reconstructions through the independent decoder. Lowering the progress lead or omitting top-right availability must fail these checks; a candidate type does not establish spatial identity or publication order | -| Parallel HEVC decoding reserves a complete entropy-work bound before dispatch rather than overspending a shared budget and detecting it afterward | `Cabac::max_bins` bounds each row, and `decode_wpp` owns the sum reservation and serial budget-tail fallback. `substream_bound_covers_the_longest_mps_runs_and_truncated_reads` checks the arithmetic bound; `wavefront_workers_preserve_samples_work_and_failure_order` requires low-budget calls to stay serial. A racing counter cannot preserve the serial error order | +| Parallel HEVC decoding reserves an entropy-work bound before dispatch rather than overspending a shared budget and detecting it afterward | `Cabac::max_bins` bounds each row, and `decode_wpp` owns the complete sum reservation and serial budget-tail fallback. Picture jobs cap their reservation at the remaining hard picture limit and normalize stream exhaustion at ordered reconciliation. `picture_budget_caps_entropy_reservations_without_losing_overlap_or_error_order` proves actual overlap and identical output/error prefixes at 1/2/4/8 threads around both limits. `substream_bound_covers_the_longest_mps_runs_and_truncated_reads` checks the arithmetic bound; `wavefront_workers_preserve_samples_work_and_failure_order` requires low-budget WPP calls to stay serial. A racing counter cannot preserve the serial error order | | HEVC worker reuse preserves cold admission, resets private state and accounts for idle allocations at each source allocation | Session ownership and scoped worker borrows prevent shared mutation and partial-state retention. `wavefront_warm_workers_match_cold_and_serial_across_picture_changes`, `wavefront_warm_memory_preserves_independent_cold_admission_boundaries` and `wavefront_warm_errors_panics_and_deadlines_release_every_private_frame` cover reset, cold limits and failure cleanup; omitted reset, release and transient-reservation mutations fail. The physical-envelope, empty-slot, output-capacity, RBSP, entry-storage and SAO allocation tests cover bookkeeping and release boundaries. Types cannot establish target padding, allocator capacity or a caller's runtime limit | +| HEVC worker reuse invalidates row-local metadata before prediction without copying or clearing full sample planes | `reset_row` clears the assigned CTU row and its upper and lower metadata halos before any dependency import. Intra references extend at most one CTU row below the assigned row; upper sample and syntax reads use the imported first or last four lines. `reused_metadata_is_invalidated_only_in_the_assigned_row_and_neighbour_halos` preserves poisoned samples and distant metadata, including a short final CTU row. The poisoned-worker reconstruction regression checks stale reads; the partial-slice-tail regression checks complete output, work and continuation at both depths. External geometry and scheduling require runtime checks | +| Public HEVC worker requests respect available CPUs without limiting private scheduler fixtures | `worker_count` owns the one-through-sixteen and checked CPU cap at both `PictureDecoder` configuration boundaries. The CPU-cap/fallback and existing-session regressions cover normalization, while unchanged private WPP tests still exercise eight workers. Host CPU availability is a runtime value | +| Skipped HEVC pictures retain mandatory RBSP and entry-storage charges independently of idle worker capacity | Parser reservations precede the reconstruction skip. `skipped_picture_scratch_preserves_cold_rbsp_and_entry_limits_with_idle_workers` checks generated RASL headers with zero and one entry, each one byte below and at its independent cold bound, plus eviction and retention of real cached frame capacity. Bitstream sizes and allocator capacities require runtime checks | | HEVC WPP prediction reads published above-right samples rather than passing only PCM or all-serial thread-count comparisons | `wavefront_above_right_prediction_waits_for_publication` uses non-PCM mode-34 CUs, distinct upper samples and delayed upper-right publication; reducing the dependency lead or omitting later imports must fail. `threading_coverage` rejects all-serial complete threaded conformance runs. Runtime sample identity and scheduling require these checks rather than a static worker type | | HEVC worker faults join all workers and distinguish decoder bugs from malformed data or caller limits | `wavefront_mid_row_errors_and_panics_join_every_worker` requires the serial error and bin count at 1/2/4/8 threads; `reconstruction_panics_are_backend_failures_not_media_limits` requires the existing backend failure kind. The standalone Apple adapter's exhaustive conversion and `decoder_internal_failure_is_not_a_metadata_or_resource_refusal` preserve that distinction outside the root workspace (`encoder-backends.yml`). `wavefront_deadline_expires_inside_a_parallel_dependency_wait` proves expiry after multiworker entry. The runtime fault location cannot be encoded in the worker type | | The H.264 encoder's SIMD exception covers AVX2 on x86-64 and NEON on AArch64 only; another instruction set needs a new policy record | Not mechanically enforced: review checks record 0044 point 2 in `docs/design/encoder-policy.md`. A capability check alone does not limit the policy's instruction-set scope, and Rust types cannot constrain policy prose | @@ -76,7 +93,7 @@ H.264, AAC, Opus, VP9, AC-3, MP3 and AudioToolbox behavior; HEVC decoding and pl | The active HEVC SPS ID and full content remain fixed throughout a coded video sequence | `HeaderParser` retains the active SPS identity and rejects a change before new-picture DPB processing; `Dpb::starts_cvs` alone owns the IRAP/NoRaslOutputFlag boundary used by both activation and POC. `first_slice_cannot_change_the_sps_identity_inside_a_cvs` rejects same-slot changes and new IDs for TRAIL and non-starting CRA, while `a_new_cvs_can_activate_a_different_sps_identity` permits IDR and CRA after EOS. A type cannot establish a temporal bitstream lifetime, so the single state owner and wire regressions enforce it | | HEVC parameter-set NALs between slice segments retain the current picture for identical active repeats and changes to unused IDs; a changed used set requires a new first slice | `HeaderParser` owns active PPS/SPS/VPS IDs and fixed-slot full-RBSP identities. The repeated-set, unused-ID replacement and changed-active-set wire regressions in `coreav-hevc/src/tests.rs` exercise all three set types. Ownership centralizes invalidation; types alone cannot establish external NAL content or ordering | | Serial tile, WPP and dependent-segment traversal and entropy state precede universal intra-picture hash gates, independently of parallel scheduling | Not enforced: review of milestone dependencies against H.265 clauses 9.3.1 and 9.3.2.1. The HEVC plan puts serial semantics in milestone 2 and requires coverage by G3; milestone 6 adds scheduling only. Types cannot establish the dependencies of a prose plan; the future conformance harness must enforce its feature coverage | -| A selected native HEVC session ends after any failed packet, EOF or receive operation, releases its queued frames and never opens a fallback, even when opened directly | `NativeDecoder` in `coreav-backend-hevc` owns the terminal state for every operation, independently of `coreav_decode::Checked`. `selected_native_errors_never_open_a_fallback`, `malformed_or_multi_picture_access_units_close_and_discard_queued_output` and `a_packet_after_eof_fails_and_drops_unreceived_frames` cover the wrapper and direct paths; the boundary fuzz target asserts that every operation after a failure fails. External packet content cannot be constrained by a type | +| A selected native HEVC failure releases live decode state, preserves validated packed output before the latched error, and never restarts or opens a fallback | The hidden `TerminalOutput` contains only packed-frame entries and an error; `Checked` takes it before dropping the native backend. `terminal_failures_preserve_validated_frames_through_public_open` covers direct and checked paths at 1/2/4/8 workers with framing, wrapper-packet, decoded-budget and post-EOF failures. `terminal_handoff_moves_only_packed_frames_after_decoder_teardown` checks queue and plane allocation identity after teardown. `wrapper_frame_refusal_discards_later_terminal_output` prevents later residual frames from escaping a wrapper frame refusal. The boundary fuzz target permits only packed-frame receive success after failure, followed by a terminal error. Types exclude parser/worker/reference owners from the residual; runtime stream order requires regressions | | Native HEVC thumbnails decode every packet from the selected start, drain the selected access unit and refuse a missing target, while every other route keeps its disposable-packet shortcut and early stop | `decode_through` gates the route on the HEVC codec and the selected `Native` backend kind. `only_native_hevc_decodes_every_packet_and_drains_the_selected_access_unit` records the packets, discard flags and EOF for native HEVC, fallback HEVC and native H.264, including a missing target and a late EOF error. Selection is a runtime result, so a type cannot choose the route | | A HEVC configuration record is admitted only with at most 64 distinct arrays, 4,096 initialization NALs, a one-, two- or four-byte length size and no trailing bytes | `Configuration::parse` owns the caps before any allocation; `invalid_configuration_and_other_codecs_are_never_admitted` covers each refusal and the admitted shapes. External bytes cannot be constrained by a type | | Stored HEVC configuration NALs follow the same terminal-zero padding rule as packet NALs before the shared decoder reads them | `Configuration::feed` passes every `hvcC` NAL through `coreav_hevc::packet_nal`, the owner the packet path already uses. `stored_terminal_zero_padding_follows_the_annex_b_rule_in_configuration_and_packets` fails with `rbsp_trailing_bytes` without it. External container bytes cannot be constrained by a type | diff --git a/docs/agents/docs.md b/docs/agents/docs.md index 73b1cd0c0..609396e62 100644 --- a/docs/agents/docs.md +++ b/docs/agents/docs.md @@ -4,9 +4,14 @@ Documentation-site prose, schema wording, and design-note updates. | Rule | Enforced by | |---|---| +| An implemented HEVC bridge is not described as a pending initial-stage alternative elsewhere in its design note | The current bridge section owns implementation status; initial-stage rationale and alternatives are labelled historical. Not enforced for prose: review against `Rows`, admission and the dependent-reader regression. Types cannot establish the tense or scope of a Markdown status claim | +| A lagged HEVC design names its later-activation refusal as a narrow exception to serial acceptance and specifies output draining after the failing call | Not enforced for prose: review against the selected admission rule. The existing terminal-prefix matrix receives after send or EOF failure; the future growth-refusal differential must preserve the exact prior-output prefix and all other serial outcomes. Types cannot constrain a Markdown equality claim | +| A HEVC row-pipeline design separates returned-frame equality from packet-origin and warning diagnostics | `Frame` exposes only timestamp, color and pixels; the design names separate diagnostic observations for origin-driven discard and warning identity. Not enforced for prose: review against that API, because Rust types cannot constrain a Markdown claim | | Feature documentation distinguishes an optional encoder from its shared transcode dependency | The native H.264 CLI workflow checks that default and release normal-dependency graphs exclude `coreav-h264-encoder`; the meaning of broader dependency claims remains review judgment. Rust types cannot establish prose meaning | | Documentation checks read Unicode measurement units with an explicit UTF-8 encoding rather than the host locale | The coefficient-design check selects UTF-8 at every file read. `test_document_reads_do_not_use_locale_defaults` runs its real checks with implicit-encoding warnings fatal, rejecting the original omitted encoding. Choosing the format at the I/O boundary removes the locale dependency without a new reader wrapper | | The coefficient-level design describes the all-zero-block shortcut, including skipped DC trials, and records the accepted throughput tradeoff rather than leaving an unmet target as its status | `test_h264_g1_design.py` binds the stronger policy's documented result to `AC_COORDINATES`, requires full source identities, all five Y/U/V and one/four-worker rows, and the shared-host qualification; omission and changed-budget controls reject the former missing-result state. The design owns the result block. Types cannot bind external timings to prose; numerical accuracy and the shortcut explanation remain review against `Decision::refine_levels` and the retained observations | +| A dependency-ceiling description distinguishes the unconstrained DAG work/span bound from worker-limited coded-order simulation | Not enforced for prose: review against the separately named analyzer fields. `test_independent_pictures_fill_available_workers` requires a bound of five beside a two-worker schedule of three steps; types cannot constrain a Markdown description | +| A deferred native-output design preserves discarded packets' no-output behavior before frame budgets and allocation | The proposed `PreparedOutput::Suppressed` variant distinguishes a record without packed storage; implementation remains pending. Existing `Output::push` checks discard first and `native_frames_preserve_depth_pts_and_discard_through_eof` covers output suppression; pending-budget equivalence needs the pipeline regression before implementation can be accepted | | An audio-option admission change updates both its explanatory prose and the option table | Not enforced: review judgment. The generated CLI tests establish effective codec and identity behavior, but types cannot establish whether separate prose statements agree | | Final HEVC SIMD guidance requires separate scalar and automatic evidence rather than treating a newest-dispatch verifier as a two-mode check | The owning HEVC tier note defines both modes; the central CI procedure links it and requires independently checked run attempts. `test_manual_c3_failure_blocks_verification_but_scalar_skip_has_no_c3_credit` demonstrates the verifier's scalar limit. Operator verification of both runs remains review judgment | | A weekly full-run drift claim distinguishes re-executing jobs from re-fetching cached pinned inputs | Not enforced: review judgment. Existing acquisition conditions can skip remote downloads on cache hits, so a schedule alone cannot establish fresh remote-input availability | diff --git a/docs/agents/measurement.md b/docs/agents/measurement.md index 6e2175b43..11dd75026 100644 --- a/docs/agents/measurement.md +++ b/docs/agents/measurement.md @@ -4,6 +4,11 @@ Speed rounds, bars, thresholds, capacity, and measurement hosts. | Rule | Enforced by | |---|---| +| Native HEVC row-overlap evidence cannot credit complete-reference jobs, zero-lag admission or different streams' admission and overlap | The test-support counter increments only for successful overlapping Raster jobs issued under positive lag; generated dependent-reader and complete-reference sibling controls distinguish them. `native_receipt` binds positive counts to a positive-lag admission, and complete conformance requires both in the same stream. Python regressions reject legacy-only overlap and zero-lag receipts; timed calls label zero row overlap explicitly. The packet driver owns collection, while a static worker setting cannot establish executed overlap | +| HEVC picture-overlap ceilings replay uncensored header dependencies, rather than counting only references still unavailable after output drains | The cfg-only `HeaderParser::diagnostic_dependencies` uses the normative DPB identities and the reconstruction owner's active-list derivation, unioned across slice segments. Generated zero-reorder chain/sibling and long-term/list-modification tests cover extraction. `hevc_dependency.analyze` validates coded identities and POCs and separates optimistic DAG width, coded-admission opportunities and an explicitly unit-duration model; graph tests check widths by brute force. Header-declared potential references are not entropy-observed block references, and none of these ceilings is observed overlap or measured speed | +| HEVC diagnostic output-barrier counts include drains while sealing a picture, not only predictions before the next picture starts | `Dpb::bump_constraints` owns the reorder/latency conditions used by resumption and diagnostics. `diagnostics_count_output_bumps_that_reconcile_at_picture_sealing` checks a zero-reorder three-picture stream and fails when the sealing hook is absent. Runtime scheduling attribution needs the executed regression in addition to a shared predicate | +| HEVC threading diagnostics describe a timed input only when their prepared-input hashes match; equal input IDs are insufficient when picture caps differ | Both drivers use `reference.inputs` and retain `input_sha256`. Diagnostics fix the cap at 32 while timed runs allow 64. Cross-run hash comparison remains a report-boundary check; the individual receipt validators cannot establish a relationship to a report they do not consume | +| A HEVC picture-scheduling reference retains wavefront decoding in the synchronous main baseline and distinguishes requested, host-bounded and observed workers | One shared constructor applies the thread request to both arms, and the baseline condition selects only its synchronous input method. `test_main_baseline_keeps_wavefront_threads_and_uses_only_the_sync_api` rejects removing that request; `test_timed_receipts_retain_requested_host_and_frame_work` checks the exported counts. External host capacity and runtime overlap require observed counters, not a thread-setting type | | A quantizer comparison cannot hide chroma regressions behind a luma-only result or an average of BD-rate percentages | `h264_quantizer_report.summarize` requires Y, U and V PSNR and calculates all three curves plus the 6:1:1 weighted-PSNR curve; `test_h264_quantizer_report.py` proves that a chroma regression stays visible and rejects incomplete or invalid cells. One report owner is the highest level available for external experiment results; Rust types cannot describe their JSON | | A HEVC reference result is accepted only for the expected full decoder commit, not another run sharing its workflow branch | `hevc_reference.py check-result` validates the expected SHA and complete receipt before emitting statistics; the direct and real-CLI tests in `test_reference_hevc.py` reject another candidate without printing numbers. Artifact names also include the candidate SHA. Workflow event routing alone cannot bind a decoder commit | | HEVC reference media children receive no acquisition credentials, oracle comparison forbids pixel conversion, and function profiles retain padded small percentages | The reference driver's one subprocess owner supplies an allowlisted environment and its oracle command uses a strict pixel format. `test_reference_hevc.py` checks credential isolation, strict format admission and a 1% Callgrind function. External tool formats and process environments require runtime checks rather than Rust types | diff --git a/docs/design/decode.md b/docs/design/decode.md index 3049fa4b9..e38d60f0f 100644 --- a/docs/design/decode.md +++ b/docs/design/decode.md @@ -217,6 +217,8 @@ other limits are the backend's to enforce. A `Limit` error from the decoder returned by `open` ends that session, whether the wrapper or the backend detected it. The wrapper drops the backend and its queued output, preserving the original error for the failing call. Later packet, EOF and frame calls return `InvalidData`; `info()` remains available. A caller must open a new decoder to continue. This is not a recoverable backpressure signal. The wrapper adds a state check and a limit-result check per call, and retains one metadata-string copy per opened session so failure cleanup needs no additional metadata allocation. +Opt-in native HEVC has a contained terminal-output exception through its documentation-hidden workspace feature. Before any limit, it reconciles earlier reconstruction so an earlier failure wins. On any terminal failure, it drops the live picture decoder and transfers only already validated packed frames, their byte and capacity accounting, and the latched error to the wrapper. Receive returns those frames in order, then the same error. If the wrapper itself refuses a received frame, it clears the residual because those frames follow the refused one. Packet and EOF calls cannot restart the session or consume its residual output. This owner retains no parser, worker or reference pin, and does not perform new decode work or reapply a deadline to validated output. Direct native HEVC follows the same failure-prefix rule at every worker count. Other backends and release builds keep the ordinary teardown above. + The defaults are: | Limit | Default | Why | diff --git a/docs/design/hevc-frame-threading.md b/docs/design/hevc-frame-threading.md new file mode 100644 index 000000000..0c5be493b --- /dev/null +++ b/docs/design/hevc-frame-threading.md @@ -0,0 +1,326 @@ +# HEVC picture scheduling + +## Scope + +This note describes bounded picture-level concurrency in the opt-in native HEVC decoder. The existing decoder thread option remains the only CLI control. Build defaults, syntax admission, decoder isolation, filter arithmetic and prediction arithmetic stay unchanged. + +The initial complete-reference stage overlapped independent intra pictures and pictures sharing older completed anchors, including streams without WPP or tiles. It could not overlap a chain in which each picture references its immediate predecessor. The current raster-row bridge below adds dependent-chain overlap. No input-coverage or speed claim follows from adding a queue. Verification must observe simultaneous picture reconstruction and report the actual eligible-picture share. + +## Current boundary + +`PictureDecoder::read_nal` validates headers and reconstructs each slice before returning. On the next first slice, `StreamCore` validates that slice header before finalizing the previous picture. `Dpb` permits only one pending attachment, publishes only complete filtered samples and owns POC, reference marking and presentation-order bumps. + +Queued work cannot retain the current `SliceInput`: it borrows the local RBSP, entry array, tile layout and active parameter-set slots. Nor can a worker borrow the mutable DPB while the caller admits another picture. These ownership boundaries must change before dispatch can overlap pictures. + +The synchronous `read_nal` contract is unchanged. The native backend uses `read_nal_scheduled` through its existing send/receive protocol. That path may defer reconstruction completion and failures until synchronization, but selects the same first failure and valid output prefix as serial execution. Header and packet validation, callback execution and external output reservations remain caller-owned. A caller reporting its own framing or read error first calls `synchronize_pending` to preserve an earlier reconstruction failure. Changing thread count does not silently make the synchronous API asynchronous. + +## Ownership and data flow + +The caller owns one parser and one normative DPB. A private scheduler in `decoder/scheduler.rs` owns bounded picture jobs, their input storage, started worker handles and ordered outcomes. Workers own a `Frame`, immutable input and complete reference snapshots. They never mutate the normative DPB, invoke callbacks or reserve external output memory. + +`Work` moves a `Frame`, owned slice operations, ready reference publications and reserved entropy work into one reconstruction task. `Job` retains its coded index, timestamp, origin, reservation and join handle or completed `Outcome`. `ReadyReferences` pins immutable publications identified by coded index and POC, not a recyclable DPB slot. + +`OwnedSlice` owns the bytes, entry lengths, activated parameters and layout required by the existing slice walker. Its charged inline hash slot can retain one decoded-picture-hash SEI at the exact position after that slice. Requested and actual capacities are checked before ownership transfers. Reference storage remains pinned through the last borrowing job even if normative reference marking removes that picture. A single reference-reader interface serves the serial DPB and ready snapshots; reconstruction arithmetic is shared rather than copied. + +The DPB still owns the current picture's pending header attachment. At a boundary, that attachment receives a validated immutable publication slot, while the scheduler tracks outstanding reconstructions by coded index. Reference marking, missing-reference validation and output selection remain in the existing DPB owner. A picture runs only after every selected reference has complete filtered samples and motion fields. Neither completion order nor DPB removal can release a reference still used by a job. + +The caller reconciles results at synchronization points: required reference acquisition, output bumps, memory or entropy reservation pressure, explicit drain, end-of-sequence transitions and failure. It waits for the required serial prefix before committing counters, sample attachments and output callbacks. To preserve the output prefix before a later failure, impending output bumps currently synchronize outstanding work conservatively. Ordinary reorder pressure can therefore limit overlap as well as reference dependencies. Suffix hashes stay associated with their original picture and are checked before that picture can become an output or reference. + +All SEI syntax is parsed on the caller. Informational prefix and suffix SEI, including picture timing, does not synchronize reconstruction. The implemented sample-dependent SEI is decoded-picture-hash (payload type 132, including reserved hash types): a picture job verifies it in slice order after reconstructing the associated samples, not at parser arrival. A second hash at the same slice position reconciles the first before applying the existing duplicate-placement refusal. Malformed SEI also reconciles earlier work before reporting its parser error. No new SEI interpretation or syntax admission is introduced. + +## Ordering and limits + +Operation order is finer than picture order. A later first-slice header is validated before the previous picture's boundary finalization, while an earlier slice's reconstruction failure precedes that later header. Queued slices are reconciled without premature boundary finalization when header or caller validation fails. Earlier jobs are joined before a later result becomes authoritative, and the first error is latched. A later worker's error never cancels earlier undecided work. Worker panics use the existing internal-error mapping, and every started worker is joined before teardown. + +Dropping a session joins running whole-picture jobs and can block until they finish. Workers check an optional deadline cooperatively; the default has no deadline, and a configured deadline does not interrupt a join or provide a hard teardown timeout. There is no separate cancellation flag. + +The execution limit is `min(requested_threads, 16, available_parallelism)`, with a one-worker fallback when host capacity is unavailable. The caller counts against this limit whenever it executes reconstruction. Frame and WPP work share this one limit; the first implementation selects one scheduling level for a group instead of nesting pools. An explicit request above host capacity must not create oversubscription. + +Cold serial admission remains authoritative. Scheduler records, owned input, retained reference owners, working pictures, bounded stacks and finalization scratch are reserved without double-counting shared samples. Before optional work could make a required allocation fail, the caller reconciles it and releases optional storage. A decode admitted without scheduling remains admitted with scheduling at the same caller limit. The original typed accounting layout fixes the cold allowance even when optional counters are added; physical-envelope tests check the actual storage against it. Independent cold decodes test admission boundaries. There is no pressure-sensitive worker-cache policy. + +Reserve the smaller of the conservative complete entropy-work bound and the picture's remaining hard bin limit before speculative dispatch. Both successful and failing jobs stop within that reservation; a large byte-derived bound alone cannot exclude an otherwise eligible picture. Keep independent stream and picture limits. If the reservation cannot fit the remaining stream allowance after outstanding reservations, reconcile prior consumption and continue through the existing serial budget path. Reconcile actual bin counts and map exhausted stream limits in operation order rather than allowing a racing shared counter to select a failure. Allocation and deadline failures remain terminal, with no fallback to another decoder. + +## Reference-only overlap ceiling + +Before implementing the deferred-output amendment, replay headers on all thirty existing reference inputs at the same thirty-two-picture cap and retain their prepared-input hashes. `thread_dependencies_only` selects this Actions-only path without sample reconstruction, timing, conformance or sanitizer work. Its acquisition limits and sealed metadata handling are shared with diagnostics. + +Record real coded-picture identities and POCs, every used current-RPS member, and the union of active reference lists across independent slice segments. I slices add no active reference; dependent segments inherit their independent header. Use the existing DPB and active-list owners, not a second POC or list-derivation implementation. Skipped reconstruction pictures remain visible in the coded roster but do not become work vertices. A malformed, missing, forward or mismatched dependency fails the record. + +Report RPS and active-list graphs separately. Maximum DAG antichain width is an optimistic bound that permits arbitrary header lookahead. Coded-order admission is narrower: a picture can remain outstanding only until its first later dependent blocks the caller. Count duration-independent coexistence opportunities within that interval and distinguish non-WPP candidate pairs. The unit-cost work/span bound uses the unconstrained DAG critical path, independently of worker count and coded-order admission. Separately simulate coded-order dispatch at 1/2/4/8 workers with explicitly equal, unit-duration pictures, reporting overlap, steps and peak workers. Neither the DAG bound nor the dispatch simulation is measured throughput or a duration-independent speedup. Ignore output, memory, SEI and presentation barriers in this analysis; it does not authorize early reference publication. + +The ceiling must be read before deciding whether an output-journal redesign is useful. A serial reference chain cannot benefit from removing output drains; overlapping it would require a separate row-level reference-publication design. + +The second replay adds normative output events and assumes one coded-picture access unit per send, followed by receive-until-`None`. Admit the current picture's job, then join every earlier submitted picture covered by that call's output events before returning their frames. EOF joins the remaining prefix. Report modeled steps and peak workers at four threads separately from the no-output-barrier model. Picture-ready admission remains optimistic: the existing scheduler seals a picture at the next boundary, so this model does not prove that its proposed submission timing is already implemented or that arbitrary packet grouping has the same ceiling. + +Report existing seal timing separately: call `i` first submits picture `i - 1`, then requires the current picture's references before accepting its input, and finally applies the same output-receive fence. The EOF call submits the last picture and completes every job. The used-RPS graph matches the existing `Scheduler::required_references` gate; substituting active references is a separate hypothetical model. These equal-duration replays still omit memory, SEI and output-processing costs and are not full implementation simulations. + +A separate four-worker study permits at most three normative output events to remain unconsumed. Its abstract producer/consumer rendezvous consumes the oldest causally completed record before admitting a fourth; ordinary receive steps consume already-ready records without waiting, and EOF drains all records. Count the maximum held output records and report current-picture-ready and delayed-sealing schedules separately. This is an optimistic frame-count/backpressure model, not an implementation of the existing send/receive API or a proof of a byte-budget rule. Publishing a record without consumption does not free its slot. + +The row-reference variant must use decoded motion-vector footprints and CTU-row dependencies if those are available. The retained header graph contains neither, so its explicit fallback keeps worst-case full-picture reference dependencies and produces the same schedule as the whole-reference variant. Equality under that fallback does not establish that actual row publication has no benefit. No row-level sample publication is implemented or authorized by a model. + +An additional diagnostic-only decode captures those missing reads on the twenty-four non-WPP inputs, once each at the existing thirty-two-picture cap. It runs synchronous reconstruction and records the actual clipped interpolation support from the kernel, including chroma scaling and fractional-filter margins, plus separate collocated motion-field row reads. Stable coded index and POC identify each reference; the collector retains each current CTU row's maximum row per reference. It is bounded to thirty-two pictures, 2,048 rows per picture and sixteen references per row, and it is compiled out with the existing diagnostic cfg. Generated probes verify the kernel's last read row with poisoned samples, preserve decoded output and reject a missing recording hook. Ordinary binaries reject the row-inspection mode. + +Capture refuses a multi-worker decoder before reconstruction, and extraction rejects an observed worker count other than one. Diagnostic identity checks are fallible and can stop a trace where an ordinary run would continue or fail differently; this mode never judges correctness. Ordinary Linux pull-request CI runs the cfg-gated extraction and kernel tests with exact success witnesses. + +Replay four-worker row tasks with equal row costs including amortized filtering work. Rows within a picture stay sequential. In the hypothetical incremental-filter model, a sample dependency on reference row `q` waits for reconstructed row `min(q + 1, last)` to cover horizontal deblocking and SAO's one-pixel neighbour margin; motion-field reads wait only for their reconstructed row. Report a separately labelled two-row lookahead sensitivity and a complete-reference control. Actual filtering remains whole-picture: this is not a claim that incremental filtering or its necessary unfiltered-border storage already exists. Tiled pictures conservatively retain atomic whole-picture readiness and publication rather than assuming a raster-row entropy order. + +Report row-unit work, span and scheduled steps, normalized picture steps, peak workers and the lag-three pending-record bound for both submission timings. Row variants hypothetically replace the current full-reference input gate with row readiness; the control retains that gate. Distinguish the modeled serial-work/steps ratio from the four-worker DAG ceiling `min(4, work / span)`. Both are conditional model quantities, not measured speedups or a byte-budget proof. Keep the original source and prepared-input hashes and the request cap; no pipeline or reference-publication change follows from collecting the trace. + +## Raster-row amendment: design rationale + +This amendment specifies the filtered raster-row bridge described under “Current bridge implementation”. It builds on the actual-read model rather than the rejected serial-exact output journal below. Existing `PictureDecoder` methods, their callback behavior, typed cold admission, build defaults and the supported API remain unchanged. Only the contained native-backend bridge may select this path. + +### Output contract and caller boundary + +At more than one effective worker, permit bounded output latency. Let `V(c)` be the ordered frames returned by the real one-worker native decoder through caller operation `c`, for the same send/receive/EOF sequence. Let `R(c)` be the frames actually returned by the native row path. `R(c)` must be an exact prefix of `V(c)`, including samples, dimensions, pixel format, timestamps and color. These are the fields exposed by `Frame`. Packet origin and decoder warnings are separate diagnostic observations, not returned-frame fields. Check origin-driven discard selection and warning identity through those observations without adding fields to the supported API. Its frame-count debt is at most `L = threads - 1`; the host-bounded effective worker count may impose a smaller bound. Once EOF/drain finishes, debt is zero. One effective worker has no output lag. At every worker count, native HEVC preserves the validated pre-failure frames before the same terminal error; this corrects the old one-worker queue-clear behavior rather than treating it as the oracle. + +Lag changes availability, not content or failure selection. Every frame whose normative output event precedes the first serial failure remains receivable in order, followed by the same terminal result. No frame after that failure becomes observable. Suppressed/discarded outputs neither create frame debt nor consume pending/decoded output bytes; their causal ordering still matters. A later job completing first does not commit an output, free a pending slot or move the serial failure boundary. + +A failing send or EOF reports its error immediately. The caller can then receive the validated prefix until receive reports that same latched error. There is no intervening `None` and no promise that the caller has received all those frames before the failing send returns. The later-activation memory refusal below is the sole exception to terminal-result and admission equality with one-worker decoding: the row path may refuse while direct decoding continues. Up to that activation, frame-prefix and debt rules still apply. At the refusal, preserve every valid prior-picture output, select any earlier serial failure first, and emit nothing from the refused current picture. No other serial-result difference is permitted. + +Preserve the shared pre-EOF nonblocking `receive_frame` contract. Receive can return `None` within the permitted debt, but cannot wait for reconstruction to make a frame ready. The caller-owned send/synchronization path must prepare a causally validated ready prefix before returning whenever additional `None` responses would exceed the debt allowance. EOF joins all work and makes every valid output drainable; receive after EOF may wait as already documented. Do not revive the rejected pre-EOF blocking-receive exception. + +The implementation needs an ordinal ledger derived from the existing normative output owner, not a second DPB/output-order implementation. It records output events, virtual serial receive opportunities, actually returned frames and their byte charges. A `Some` transfers only the oldest validated frame; a `None` cannot discard, consume or pretend to return one. Tests run a real synchronous decoder as the oracle for `V(c)` rather than trusting the ledger's own counters. + +### Row ownership: selected conservative shape + +The first producer stage converted complete-reference jobs when its extra buffers and row-store record fit within the job's existing full-component SAO allowance. Conversion moves the full planes and cells into the shared backing, retains one CTU metadata vector and reuses the original scratch. If conversion cannot fit, it returns the unchanged frame for the whole-picture path. The producer owns filter ordering, saved borders and row commits. The current native bridge also uses its row publications for dependent-picture overlap, with bounded output lag and continuation admission. + +Considered immutable separately allocated row bands versus one contiguous picture store. Separate immutable bands would replace the current plane owner and require a gather/window interface in interpolation and output packing. Select a shared contiguous store with read locks for consumers and short write holds only for band commits/publication. The producer reconstructs a bounded band privately without holding the store lock, including while waiting for another reference, then copies that band into the backing store. This is a charged transient row-band copy, not a retained second full picture. Readers must not stall behind an entire row's reconstruction or reference wait. The row model does not measure these copy/lock costs. Do not add unsafe aliases to make it look like the model. + +The private row store uses coded index, POC and generation to identify its backing. Its progress owner separates reconstructed motion from filtered samples. Scoped reference guards expose only the requested capability. `decoder::rows::Rows` owns the bounded pending window and ordinal ledger. The ordinary `Dpb` and `StreamCore` layouts stay unchanged. Instead of a second identity table, the existing heap publication optionally owns a row store on the threaded path. Complete stores move their vectors into the ordinary retained-picture owner without copying sample planes. Outstanding guards and owners keep that generation alive through the transition. + +The ownership sketch is not code to compile. Frontiers are exclusive luma-row endpoints and advance only after a complete CTU row, including a partial final row. Zero publishes no row: + +```rust +struct Progress { motion: usize, filtered: usize, outcome: Outcome } +enum Outcome { Building, Finishing, Complete, Failed } +struct RowStore { + identity: ReferenceIdentity, + generation: u64, + storage: RwLock>, + completed: OnceLock, + progress: Mutex, + changed: Condvar, +} +enum ReadPicture<'a> { + Complete(&'a RetainedPicture), + Building(RwLockReadGuard<'a, Option>), +} +struct ReferenceRead<'a> { picture: ReadPicture<'a>, support: ReadSupport } +``` + +Only the producer's non-clonable `RowPublisher` can commit a building band or advance its frontiers. Each commit includes every dirty halo row in the band above, but never writes its published prefix. Its private entropy cursor, band and unfiltered borders survive between short store accesses. The ordinary `Frame` layout remains unchanged. `RowStore::read` waits for the requested motion or component-sample capability and returns a scoped guard. Commits and failure notify all waiters, including when the publisher is dropped. The progress owner records completion or failure. A complete view and a row view use the same allocation owner, not separately charged copies. The guard exposes no mutable data or unrestricted future-row slice to its caller. + +Keep next-boundary job sealing initially: a worker receives the existing owned slice operations and parameter snapshots, not syntax commands for future packets. It owns its CABAC cursor on its stack and resumes the existing traversal at a validated CTU boundary. Releasing the picture lock between completed rows must not restart CABAC, copy contexts, reset QP or require a self-referential input object. A scoped row step borrows the owned slice and picture only while working. Independent/dependent slice segments can divide a row; readiness advances only after every required CTU is reconstructed and the contiguous prefix is valid, never merely at NAL end. Missing CTUs and malformed segments retain their original error positions. + +Motion-field readiness and sample readiness are different capabilities. A collocated-cell probe waits for the reconstructed motion frontier even if motion selection ultimately finds no usable vector. Motion compensation waits for the complete clipped support of each component/list, including signed vertical reach, fractional luma/chroma support, chroma-to-luma scaling and edge replication. Factor the existing interpolation-support calculation into one kernel owner used by both validation and readiness; do not derive a fixed row lead or maintain a second footprint formula in the scheduler. The scoped reference guard pins the identity and dimensions used by that calculation. + +Wait on the progress mutex/condition variable without holding any sample lock, then recheck generation, progress and failure while acquiring the read guard. Hold at most one reference-store guard during a kernel call and release it before waiting for another. Reference edges strictly descend in coded identity. A producer finishes its short storage write and releases that lock before advancing the progress frontier and notifying waiters. No guard or pointer outlives its lock. A failed producer wakes every dependent; dependents report a private dependency-failed outcome while the caller selects the producer's original error in serial order. Poisoned-lock, partial-start, deadline and drop paths must also wake and join every started worker, not leave a row waiter asleep forever. + +There are no more row-worker threads than effective workers, and parked reference waits create no replacement worker. Start jobs in coded order. Run a job on the caller only after every earlier unfinished job is already running or complete; never occupy the caller with a later consumer while its producer remains unstarted. After partial spawn failure, the caller starts or completes the earliest remaining producer first and joins already started work. Thus the earliest unfinished producer has no unstarted dependency and must progress or report failure. Reference waits use deadline-bounded condition-variable waits, and terminal/drop notification wakes every waiter. Reconstruction/filter work on the caller consumes the same CPU allowance. WPP retains its existing path and worker limit, never a nested pool. Tiled pictures use the existing atomic whole-picture path and complete-reference publication. A raster consumer of a tiled producer waits for full completion; mixed groups cannot create another worker allowance. + +### Filtered publication and current-picture prediction + +The current finalizer does all vertical deblocking, all horizontal deblocking and then SAO. The row path must change only the traversal/ownership of those operations, preserving each scalar/kernel result and every slice/tile/picture boundary rule. Intra prediction still reads unfiltered current-picture samples: do not filter a band while any later permitted intra reference can read its original values. + +For a raster band, reconstruct the required following CTU row before filtering/publishing its upper band. Complete vertical deblocking for every row touched by the upper band's horizontal edges, then all horizontal edges that can modify or supply SAO neighbours for that band, then SAO from deblocked, pre-SAO values. Retain the necessary unfiltered intra borders and pre-SAO neighbour rows in separately charged storage. A previous band's SAO output must never become the next band's SAO input. Process each edge once with the same protection and boundary flags; no future filter operation may write an already published band. + +The diagnostic model's one-row lookahead covers the current bounded deblock and one-pixel SAO spatial neighbourhood at admitted CTU sizes; the two-row model is a sensitivity, not an implementation requirement or proof. The implementation must derive exact dirty/read/write extents from the filter owner and expose `FilteredThrough` only when that entire prefix is immutable. Flush the last band at picture completion. Motion cells may publish earlier, but an output or ordinary complete reference still waits for every row, inline hash check and coded-order causal fence. Consumers may speculatively use earlier rows of a producer that later fails; none of their dependent output may escape that failed fence. + +### Lag, pending bytes and the admission gate + +The caller's pending set is serial-equivalent rather than the physical output queue. Derive its ledger from the existing normative output owner, ignoring reconstruction status: enqueue unsuppressed output events in serial order, and count every actual receive call made while that virtual queue is nonempty as a virtual serial receive, regardless of `Some` or `None`. Virtual pending bytes are output-event bytes minus those virtual receives. Output debt is virtual receives minus real receives, bounded by `L`. A real receive during catch-up may return an older withheld frame after the virtual queue empties. Publication itself consumes neither queue. + +`max_pending_bytes` checks that virtual pending set at the same output event as serial decoding, so caller-induced retention is identical. At the event, retain the existing frame check and charge cumulative `max_decoded_bytes`; neither is delayed until publication or receive. Reconcile earlier causal work before any limit refusal, because tentative later events may follow a deferred failure that serial would have reported first. The checked owner's frame recheck at real receive must be incapable of a new refusal after the backend's identical event-time check. The differential must establish all four conditions: one normative event owner, virtual receives, reconciliation before limits, and event-time frame/decoded checks. + +**The two-frame example must succeed.** With equal frame bytes `b` and pending limit `2*b`, direct receive returns E1, then sends producing E2 and E3 fit without another receive. If row receive returns `None`, that call still consumes E1 from the virtual queue. E2/E3 occupy `2*b` of virtual pending and E1 is pipeline debt, not a third caller-pending frame. Its real storage stays charged to memory until actually received. The exact call schedule must succeed and ultimately return E1/E2/E3 in order; no frame is invisibly discarded. + +The debt's packed capacities, source pins, row state, synchronization records, filter borders, input scratch, transient copies and worker stacks are physical pipeline memory under `max_memory_bytes`, with shared storage charged once. Reclassifying at most `L` outputs does not waive those charges. Joining workers can reclaim optional job/scratch state but cannot release a withheld output the caller has not received. Shrinking lag later only prevents new debt: it cannot repair an already withheld frame before a send-only burst or larger in-band SPS fills serial's memory budget. + +The selected design chooses `L` at admission from the first active SPS's literal coded and visible geometry, depth and highest-sublayer DPB capacity, capped at effective workers minus one. Choose the largest nonnegative `L` satisfying `L * F + S <= max_memory_bytes` before creating debt. `F` is the SPS's packed output bound. `S` covers every permitted continuation under that SPS, not just the current picture or observed occupancy. It includes the highest-sublayer DPB capacity, the full caller-pending byte limit, the largest scratch and capacities any permitted PPS or slice can require, external reservations and bounded stacks. Include row workers, synchronization records, private bands, filter borders and retained source owners in their charged reservations. The packed-frame term alone does not cover an extra unfiltered working picture. Count each shared allocation owner once, using checked capacity arithmetic. If no positive lag fits at first admission, retain the ordinary direct path rather than rejecting a stream admitted by its existing cold rules. Neither inactive parameter-set arrival nor an arbitrary packet boundary selects the envelope. + +The whole-future limit-maxima certificate gives zero useful lag under the default memory limits and is not the selected admission policy. The first-SPS certificate is conditional because a later activated SPS can increase packed output, working or reference storage, or DPB capacity while old debt remains pinned. At that activation, settle earlier work and reclaim optional pipeline storage. Then choose the largest future lag that fits the new continuation-wide envelope together with outstanding debt and its pins. Future lag may decrease to zero, which selects the direct path from then on without erasing old debt. Do not reserve the old future-lag allowance or retain optional state when deciding whether to refuse. Count shared allocation owners once, including old-geometry pins. A new coded video sequence does not erase those owners. + +`Decoder::send_packet` returns only `Result<(), Error>`. The error kinds provide no retry or send-side backpressure result, and the native error path is terminal. This design does not invent a receive-first status, reinterpret a terminal error as retryable, or accept a packet that will never be decoded. Refuse a later activation only if its new envelope plus outstanding debt and unavoidable pins cannot fit even at zero future lag after optional storage is reclaimed. Place that decision after the new IRAP's slice-header processing outputs prior pictures, but before allocating the current picture's storage or starting its reconstruction. Those prior outputs remain receivable through the terminal-output handoff. The refusal must not ship before that handoff. A merely stored, unused SPS is not refused. The original one-worker/direct path keeps its ordinary geometry admission; the new condition belongs only to the selected lagged path. + +This is a new potential refusal for a stream serial could accept under the original limits, not a proof of unchanged acceptance. Count its impact over the complete conformance roster and the registered commands that actually select native HEVC. A zero count on those sets does not prove that arbitrary in-band growth cannot reach it. Generated tests must exercise a reachable growth refusal, preserve all pre-failure output and the serial first-error precedence, and show that a fitting activation and an unused larger SPS are not refused. Include an activation that fits only after future lag becomes zero. Vary receive cadence so the same activation is tested with and without outstanding debt, including the IRAP's prior-picture outputs. The current implementation and remaining verification requirements are described below. + +The in-call-only journal settles a send's output events before returning, so it has zero caller-visible lag and no cross-call output debt. Its lower ceiling remains separate from the measured lag-three model. It remains a design comparison rather than the selected implementation. Ordinary cold layouts and an allocation-free direct selection remain authoritative. Optional allocation failure falls back before visible state changes; never retry a side-effecting sink callback or expose a speculative allocation refusal as a serial resource failure. + +### Terminal prefix and containment + +Keep the accepted documentation-hidden, workspace-feature bridge with `publish = false`, no stability guarantee and no supported application API. It owns row slots, output ordinals, sink preparation and the budget decision; existing direct `PictureDecoder` methods and callback-free external-memory methods are not repurposed. The normal feature-graph check still excludes this bridge and `native-hevc` from release-default and release-full. No thread-count sentinel selects hidden semantics. + +On first failure, stop admission, join workers, resolve earlier coded operations, finish only the valid output prefix and release later speculative state. Keep valid pre-failure frames receivable before the latched terminal result. The native backend now drops its live picture decoder but retains validated packed output on failure. The hidden `TerminalOutput` handoff moves that queue, its per-frame byte and capacity accounting, and the latched error to `Checked` before it drops the backend. It performs no queue or sample-plane copy. The residual owner retains no parser, worker, row store or reference pins. Receive drains it without new decode work or deadline checks, then returns the same error. Packet and EOF calls return that error without consuming output. Other backends' limit teardown and isolation stay unchanged. + +The one-worker oracle preserves the same required pre-failure prefix as 2/4/8 workers; its old queue-clear error behavior is not retained. The hidden `native-hevc-private` checked-owner settlement capability reconciles a deferred reconstruction failure before a wrapper limit drops the backend. `checked_packet_refusal_settles_earlier_entropy_failure` tests truncated entropy followed by an oversized packet through `coreav_decode::open`, not only `NativeHevc::open`, at 1/2/4/8 requested workers. Settlement defaults to the original refusal for other backends and is absent without the private feature. Separate terminal-prefix regressions cover direct and checked opens, both packet framings, eight- and ten-bit samples, early and deferred receives, framing and entropy failures, decoded-output and wrapper-packet limits, and input after EOF. The queue-move and checked-owner destruction tests reject sample copies or a retained live backend. This handoff does not implement the row pipeline. + +```mermaid +sequenceDiagram + participant C as Send/receive caller + participant B as Contained native bridge + participant W as Bounded row workers + participant O as Ordered output owner + C->>B: Send packet + B->>B: Validate metadata and admission certificate + B->>W: Seal owned earlier picture + W->>W: Reconstruct, deblock and SAO safe bands + W-->>B: Filtered rows or ordered failure + B->>O: Charge normative output and causal fence + B->>B: Prepare ready prefix required by lag bound + B-->>C: Return send result + C->>B: Receive frame + B->>O: Pop validated front, never consume on None + O-->>C: Frame or bounded-lag None + C->>B: EOF + B->>W: Join and flush final bands + B->>O: Drain valid prefix, then terminal result +``` + +### Current bridge implementation + +The native backend enables the workspace-only `native-hevc-rows-private` feature. Existing closure-based decoder entry points keep their synchronous or complete-reference contracts. The private sink prepares the real packed output entry at its normative event and fills it only after its entire coded fence succeeds. An optional heap owner in the scheduler holds emitted, validated, serial-received and returned ordinal frontiers. Its unresolved window holds at most `L + 1` entries during an event and has preallocated capacity of at least `W`. It pins publications, not duplicate picture storage. A successful receive consumes a virtual serial entry before an optional physical pop, never joins or waits, and releases external capacity only after a pop. Existing terminal-error cleanup, including a receive-time deadline failure, still settles the valid prefix and joins before releasing the core. EOF retains the ordinal owner until the real queue drains. + +Admission qualifies the one-shot capacity premise at runtime, without a compiler identity gate or toolchain pin. Empty-`Vec` probes reserve the active geometry's plane, cell, CTU, row-band and scratch sizes, the observed source and entry lengths and their configured maxima, reference and operation-record bounds, and packed plane descriptors. Each probe checks that reported capacity equals its request and then frees the buffer. Probes run sequentially within the remaining live-memory allowance, and their peak is accounted. A mismatch at initial admission retains legacy output without panic; at a later activation it disables positive future lag before the existing old-debt decision. The forced-mismatch regression compares real legacy frames and semantic summaries with synchronous decoding. + +This is runtime qualification of the selected request sizes on the current compiler and allocator, not a standard-library guarantee or a proof for every unseen future length. Every real allocation still charges its observed capacity defensively. The continuation envelope's one-shot size predictions are conditional on this qualified premise; they do not derive exact capacity from `try_reserve_exact`'s name. + +Growth metadata does not use that premise. Admission reserves the pending window, job slots and packed-output ring, reads their actual capacities and charges those capacities before committing a plan. The output ring is reserved as a `Vec` and converted without reallocation to the existing `VecDeque`; a rejected candidate leaves the original queue untouched. Replacing an old ring charges both buffers until the old buffer is released. Rows mode never grows these buffers, and empty job slots persist until the row epoch retires. Each picture's transferred operation vector reserves its entire bounded record count once instead of growing with each slice. The cold queue path and no-copy terminal handoff remain unchanged. + +For a candidate window of `W` jobs, sample and motion-cell owners are bounded by `(D - 1) + (W + 1) + W`: the pre-window DPB, live jobs plus current picture, and the event-time pin ring. Snapshots cannot recover evicted owners. Conversion moves backing without copying, jobs retire in coded order, idle cache is cleared, and atomic branches and changed active SPS identities settle first. + +The base envelope includes the full caller-pending allowance, old queue capacities and transient descriptor growth, all packed plane records, global preactivation entry storage, maximum current-SPS substream ranges, packet-bounded owned sources, all slice and parameter snapshots, retained owners, row windows and borders, synchronization records and explicit stacks. A queued raster picture owns at most the smaller of CTU count and packet NAL count in operations; reaching that bound falls back through serial validation before another owned operation is allocated. Tiled and WPP pictures are atomic. Their optional worker setup is capped by the admitted total, with a separate admitted runtime reserve, and their cache is cleared before returning to row scheduling. + +Old debt can migrate into larger frames even when future lag is zero. For old packed debt sizes `b[0..d]` and new packed size `F`, the migration peak is `max_k(sum(b[k..d]) + k * F)`, including both endpoints. The selected certificate is `L * F + S`, where `S = S_base + max(0, migration_peak - L * F)`. Thus the total covers both future lag and every old-debt replacement order. A changed active SPS settles and fills prior events before recomputing this bound. A zero-lag owner retires once old debt and unresolved events are gone; an initial zero-lag selection never creates it. + +Generated tests force a dependent reader to wait before its predecessor's first publication and verify two executing raster jobs with serial-identical pixels and semantic counters. Reinstating the complete-reference barrier fails that test. A delayed output followed by an earlier coded CRC failure catches an own-picture-only fence, which otherwise leaks a frame. Native tests enumerate all short receive cadences at 1/2/4/8 requested workers, exercise a fitting growth that reduces future lag to zero, and exercise a reachable larger-envelope refusal against a successful one-worker continuation while preserving exactly the old output prefix. These fixtures do not establish corpus overlap or a speedup. + +### Required implementation evidence, after design clearance + +- A lag-aware per-call differential runs identical arbitrary send, receive and EOF sequences against direct decoding. It compares actual frames/metadata and the terminal result, checks prefix debt at every call, exact one-worker behavior, and zero debt at drain. The sole terminal-result exception is the specified later-activation memory refusal, tested separately against a successful one-worker continuation and its exact prior-output prefix. After a failing send or EOF, continue receiving until the latched error. Cover empty polls, partial drains, burst sends, multi-output packets, reorder, suppression, geometry changes, headers/framing/sink failures and wrapper-level limits. The two-frame E1/E2/E3 schedule must succeed. +- Mutations must fail when a virtual pending event or receive is omitted, frame/decoded checks move to publication, a finite memory-headroom guard falsely admits lag, publication is treated as consumption, the lag bound is exceeded, EOF loses debt, a causal fence covers only its output picture, or limit teardown erases the valid prefix. Keep the original reserve-failure stream-error and latency-only resumption mutations. +- Poison future rows, delay producers, use motion vectors/chroma support that cross row boundaries, and fail after partial row publication. Mutations publishing motion as filtered samples, omitting interpolation support, using SAO-mutated borders, filtering before intra reads finish, reusing a generation early or notifying no waiter on failure must fail. Tiled/WPP/mixed controls must preserve their original path and CPU limit. +- Find independent cold boundaries and observe actual allocations on all workers using the counting allocator, plus explicit stack reservations. Include hidden outputs, pins, filter borders, transient capacities, external reservations and changing input geometry. Return values and valid output prefixes must match serial acceptance at each boundary except the specified later-activation memory refusal. Test that exception without relaxing any other cold boundary; omitting any charge must fail. A final heap counter alone is insufficient. +- Before completion, observe actual reconstruction overlap and automatic/4T versus 1T wall/CPU timings on the same twenty-four prepared inputs on Actions, with hashes and work identity. Require complete 1/2/4/8 conformance identity, sanitizer stress, the lag-aware differential/mutations, green CI and independent review at the final head. Models, queued jobs and diagnostic counters cannot substitute for ordinary measured overlap. Every pending run retains a timer backstop of at most thirty minutes. + +## Deferred-output amendment + +This serial-exact alternative is not selected. Its required receive wait conflicts with the shared nonblocking contract, and a journal alone does not change next-boundary picture submission. This section, including its reservation and proof subsections, is retained history, not the raster-row implementation contract above. In particular its exact per-call multi-worker visibility requirement is not the bounded-lag contract. Its unsolved failure-handoff and physical-accounting obligations remain relevant review questions; they are not implemented by retaining this sketch. + +The complete-reference implementation joined picture work at output bumps. This historical amendment specified a separate native-backend pipeline that could advance normative DPB metadata without immediately exposing the corresponding samples. Its serial-exact receive wait is not implemented; the selected bounded-lag bridge above preserves nonblocking receive instead. + +### Boundary and workspace interface + +Keep the existing direct decoder and its callback, polling, memory-reservation and error contracts unchanged. Only the native backend selects the new typed pipeline. There is no new CLI control, default feature, admission rule or fallback path. Complete-reference picture jobs, the shared CPU limit, WPP exclusivity and all reconstruction arithmetic remain unchanged. + +A queue solely in `coreav-backend-hevc` cannot lift this barrier: the decoder joins work inside `Observer::output` before the backend callback gets control. Changing the existing scheduled method to defer arbitrary callbacks is also insufficient: `set_external_memory(bytes)` has no callback with which to publish and release deferred output under pressure. Neither approach preserves the existing contract. + +Keep the journal, fences and DPB access private to `coreav-hevc`. The backend owns output buffers through a two-phase sink rather than an arbitrary delayed callback. The intended workspace-only interface is: + +```rust +struct NativePipeline; +struct OutputDescriptor; +enum PreparedOutput { Emit(T), Suppressed } +struct CompletionFence; + +trait NativeOutput { + type Slot; + fn prepare(&mut self, picture: OutputDescriptor, memory: &mut OutputMemory<'_>) -> Result>; + fn publish(&mut self, slot: Self::Slot, picture: DecodedPicture<'_>) -> Result<()>; + fn abort(&mut self, slot: Self::Slot); +} +``` + +`NativePipeline` owns the decoder and journal. Its operations are NAL input, serial-equivalent output receive, callback-aware external-memory reservation, explicit synchronization and EOF. Reservation and synchronization have access to the sink; neither silently substitutes the existing callback-free memory method. Reconfiguration joins work without publishing unresolved results. Drop joins workers and aborts unpublished slots without invoking output callbacks. + +The two owners are separate Rust crates, so `pub(crate)` cannot connect this interface. The bridge is one documentation-hidden module behind a workspace-only Cargo feature enabled by `coreav-backend-hevc`; it is technically Rust-public, has no stability guarantee and exports no DPB, scheduler or publication internals. Its owning package uses `publish = false`. Cargo feature unification prevents a literal guarantee that another workspace consumer cannot enable it. A CI `cargo tree` check must instead prove that neither `release-default` nor `release-full` enables `native-hevc` or this bridge. It adds no supported application API. Do not encode activation in a thread-count sentinel or duplicate decoder source in the backend. + +### Completion fences and output order + +Assign bounded, monotonic submission ordinals to owned reconstruction work. A caller-owned committed prefix advances only after successful coded-order reconciliation, including inline slice hashes and finalization. Physical worker completion does not advance it. A failed job latches the first error and cannot satisfy a later fence. + +At each normative output event, capture its output descriptor, immutable sample publication, prepared sink slot and the ordinal of **all preceding submitted reconstruction**, not merely the picture being output. The journal is ordered by normative output events, including multiple bumps and sequence drains. A record may publish only when its fence is satisfied and its complete sample publication exists. The sink receives records strictly from the journal front. No worker calls the sink. + +This wider fence is essential when an earlier output picture is ready but a later coded picture must fail before the bump is allowed to become visible. Merely waiting for the output picture reproduces the existing early-output regression. Header and framing failures reconcile the preceding work, publish only the journal prefix whose fences succeeded, and then choose the first decoder, sink or caller error in serial operation order. An earlier sink failure wins over a later worker failure even if that worker finished first. Records beyond the first failed fence are aborted, not emitted. + +Keep decoded-picture hashes at their existing slice positions before committing a job. Informational SEI does not become a barrier. Random-access discard removes the same normative outputs as the direct path; it never cancels a record already valid before that discard. Geometry and sequence changes cannot relabel an older descriptor or reuse its sink slot generation. + +### Received output and terminal states + +Compare the concatenated frames actually returned by `receive`, including samples and metadata, followed by the terminal result. Equal final decoder counters or equal callback invocations do not prove this contract. Run the direct and threaded native-pipeline arms through the same failure-delivery boundary at every thread count; existing `PictureDecoder` callback methods remain unchanged. Up to the first serial error, every returned send, EOF and receive call must have delivered the same frames and hold the same pending-frame count and bytes, with the same cumulative decoded-byte charge, as direct decoding driven by that call sequence. + +Reconstruction may be deferred; visibility may not. Receive retires suppressed records in causal order without returning a frame. If an emitting record remains, it joins that front record's entire causal fence, publishes it and returns the frame. It returns `None` when no emitting record remains, exactly as direct decoding would. A pending emitting fence is never a reason to return premature `None`. A reconstruction error may surface on a later call than in direct decoding, but in that interval only frames whose output events precede the first serial failure may be returned. + +This proposed native wait conflicts with the shared `Decoder::receive_frame` documentation, which currently promises no waiting before EOF. That public boundary contract needs explicit treatment before implementation; keeping the method signature unchanged alone does not settle it. The output-barrier ceiling is measured first, without silently introducing an exception in code. + +The intended terminal transition is `Active -> FailedDraining -> Failed`. On failure, stop input, join workers, reconcile earlier causal outcomes, publish only the satisfied journal prefix and abort the rest. Keep that validated output prefix receivable, then report the latched terminal error after its last frame. No later packet may restart decoding or select another backend. Do not use the current `NativeDecoder::fail`, which clears the ready queue, for this transition. Header, framing, sink and limit failures must all follow the same ordering. + +The common `Checked::call` boundary also drops its decoder on `ErrorKind::Limit`, including wrapper packet refusals before the backend runs. A backend-only failed-draining state cannot survive that drop. The bridge must transfer the validated terminal output into an owner that survives decoder teardown before exposing such an error. That owner must retain no decoder, parser or worker, and the ordinary limit-failure and isolation behavior of other backends must remain unchanged. The transfer and error-delivery protocol still require design review; retaining a live decoder after a limit is not an accepted shortcut. + +### Reservations before speculation + +Preparing a native output slot reserves its complete packed-buffer capacity and queue metadata at the normative event, before admitting later picture work. Preparation must not expose a frame or perform an externally visible callback. Publication fills that already reserved slot without a new frame allocation; only then can `receive` return it. Both prepared and ready slots count toward backend retained bytes and queue limits. + +Preserve the direct path's discard check before geometry, output-budget and packing checks. A descriptor whose packet requests discard prepares a no-output slot with zero frame capacity and no pending or cumulative decoded-byte charge. Keep its causal fence and charge its journal metadata, but allocate no packed frame and expose no output. A discarded picture must still succeed with a pending allowance smaller than one frame whenever direct decoding does. + +Pre-reservation is necessary for cold admission: allowing an arbitrary callback to request memory after later pictures have occupied it is not equivalent to direct decoding. Before optional retention could make a required slot allocation fail, reconcile the causal prefix, reclaim optional work and recompute the cold allowance. A genuine allocator failure remains terminal, but an earlier reconstruction failure still precedes it. Do not retry a failed allocation or a callback with side effects. The sink's reserve/abort operations must be transactional; no partially prepared frame reaches the output queue. + +One accounting owner counts journal capacity, descriptors, slot metadata, worker stacks, owned input, private scratch, working pictures and pinned publications. Shared samples are charged once across the DPB, jobs, journal and a record temporarily removed for publication. Charge actual allocation capacities and transient growth, including source and destination storage while packing. Removing a picture from the normative DPB does not release its journal pin. Moving a record onto the caller stack does not make its samples disappear from accounting. + +At each normative output event, prepare checks and charges the complete output against cumulative `max_decoded_bytes` in direct operation order, before admitting later work. Publication must not defer or repeat that charge. Aborting a later record does not refund cumulative bytes for successfully prepared earlier outputs. Source validation, output suppression and pending-budget checks keep their direct ordering; a failure halfway through a journal must select the same error position. + +Pending admission checks and charges each slot's complete packed size at its normative output event on the same caller call as direct `Output::push`. This is the only pending and cumulative decoded-byte charging point; publication fills an already charged slot. Serial-equivalent receive releases the same frame on the same call, so ready and prepared slots together have exactly the direct pending charge at every successful call boundary. No extra-frame guard is needed. Allocation capacity, queue metadata and sample pins also count independently against `max_memory_bytes`. + +**Why a headroom guard is insufficient.** Take equal frame charges of `b`, a pending limit of `2 * b` and no other pending reservation. If direct receive returns E1 while threaded receive returns `None`, two later sends producing E2 and E3 leave direct pending storage at `2 * b` but threaded storage at `3 * b`. Switching to direct at E2 cannot remove E1, because publishing does not release it. The receive invariant above prevents this divergence at its first call; merely reserving room for one future frame does not. This is a scalar counterexample, not a measured codec fixture. + +Bound the journal by the existing maximum worker window plus the maximum DPB output burst. Every refusal, including pending, decoded or memory limits and journal-capacity pressure, first reconciles its causal prefix so an earlier reconstruction failure wins. Optional journal or memory pressure degrades to direct execution rather than refusing a cold-admitted stream. Reconcile and reclaim optional pins, scratch, metadata and stacks before that transition; publishing prepared records alone is not reclamation. Existing typed cold accounting remains authoritative, with physical-layout checks extended to the pipeline owner. + +The cold proof must include consumer behavior, not only final memory totals: compare identical send/receive sequences, empty receives, pending prepared slots, caller pops, new external reservations, larger pictures and parameter changes. A receive must not leave extra retained frames that make a later cold-admitted call fail. Where optional storage cannot preserve that guarantee, choose the direct path before optional work is queued; do not repair it by weakening the limit or retrying a callback with side effects. This is a required design and implementation check, not an assumed property of joining workers. + +Use the existing test-only `coreav-counting-alloc` allocator with `measure_process` in a dedicated integration-test binary containing one measured test. Count allocations on every worker, retain caller-owned fixture storage outside the measured region until it closes, and include transient source/destination and reallocation peaks. Assert `observed peak live heap <= accounted heap peak`; combine that bound with separately charged fixed state and explicit worker-stack reservations, then require the complete accounted peak to fit `max_memory_bytes`. A heap allocator alone does not observe operating-system thread stacks. Return-value equality cannot substitute for these assertions. Mutations omitting a pin or transient capacity, reusing a slot too early, or allocating during publication must each fail an allocator-observed assertion; allocation count and poisoned contents must also expose reuse that leaves the peak unchanged. + +### Proof before reference measurements + +- Extend the four-picture output-bump regression through the native pipeline. Delay the causal earlier job, make it fail, and require the exact direct output prefix. Mutating the fence to the output picture alone must fail. Reverse completion and error order, including a sink failure before a later worker failure. +- Prove actual overlap for generated zero-reorder siblings sharing a completed anchor, while a strict predecessor chain still waits. Compare samples, output order, timestamps, origins, warnings and semantic work at 1/2/4/8 threads. Queue admission and prepared slots do not count as overlap. +- Test prepared-slot cancellation, random-access discard, multiple bumps, geometry changes, malformed caller input, duplicate and malformed hashes, EOF, partial worker start, deadlines, drop and configuration changes. Every started worker joins and every unpublished slot is released. +- Compare received frames plus the terminal result with drain-after-every-send, partial-drain and burst-send consumers. Leave a satisfied record unpublished before header, framing and sink failures, and exercise both backend and common-wrapper limit errors. Require its valid prefix to remain receivable after decoder teardown. Include one- and two-frame pending limits, the E1/E2/E3 counterexample, and cumulative decoded bytes ending mid-journal. +- Enumerate short call schedules over send, receive and EOF for generated counterexample and reorder streams, crossing pending and decoded allowances at every frame boundary and one byte below and above. Compare each call's `Some` identity, `None`, error kind and retained bytes. Only a later-reported reconstruction error is permitted to differ, with no extra frame in that interval. Mutations returning `None` at a pending front fence, charging a slot at publication rather than its event, or narrowing a fence to the output picture alone must each fail. +- Find independent cold boundaries for memory, input scratch, picture/stream bins and output slots. Cross them with pending jobs and prepared outputs, and compare both return values and the valid output prefix. Mutation tests must catch omitted pins, uncharged transient capacity, early slot reuse, publication-time allocation and receive-poll admission regressions. +- Run the same ordinary executable through the native boundary and complete conformance at 1/2/4/8 requested threads, with sanitizer stress. Retain complete normalized records and actual picture-job coverage at each parallel count. +- Completion requires real picture overlap on the reference inputs. Report every input's ordinary overlap count and automatic/4T versus 1T wall and CPU times, with input hashes, work identity and observed worker counts. Untimed diagnostics explain remaining barriers but never replace those ordinary measurements. Every active run retains a timer backstop of at most thirty minutes. + +## Existing worker follow-ups + +- Resolve host capacity once through the shared worker-limit owner. Test requests below, equal to and above available CPUs, including sixteen requested on four CPUs. +- Do not copy a working picture into each picture job. Move its buffer owner. For reused WPP frames, replace whole-frame `reset_private` copies with a proved row-local initialization and publication rule. Stale samples, cells or filter records must not become available to prediction. Full-frame zero fills are not the intended optimization. +- Preserve scratch admission for skipped pictures. RBSP and entry-array storage is allocated before temporal extraction or RASL suppression is known. Compare its success and refusal boundaries with independent cold decoding, both with and without retained worker storage. Do not remove a charge because no sample frame exists. + +## Initial alternatives + +Incremental reference rows offer overlap in dependent picture chains. They require separate publication of motion fields and filtered samples, waiting for actual motion-compensation footprints including fractional-filter halos and edge replication. A fixed row lead is insufficient. The initial complete-reference stage retained whole-picture finalization. The raster-row amendment above implements the filtering control flow and storage needed for early publication. + +A syntax-ahead pipeline could overlap entropy decoding with the previous picture's reconstruction while retaining complete-reference publication. It requires a bounded command representation and syntax-only neighbour state. The current traversal immediately resolves motion, predicts and reconstructs residuals, so this is a different front-end and reconstruction split, not a scheduler-only shortcut. + +The initial stage chose complete-reference jobs to retain existing filtering and CTU semantics behind a small scheduled-input boundary. The current native bridge extends that boundary with dependent row publication. Tile scheduling remains separate unless the same ready-job and publication machinery applies without another ownership change. + +## Verification before timing + +The `coreav-backend-hevc` `reference` example exercises the public native packet boundary. Build it with `native-hevc,test-support`. Its `check` mode hashes returned pixels and emits per-frame timestamps, colour, packing and byte counts; `decode` consumes those same frames without hashing. Both modes emit semantic work, requested and effective worker counts, observed picture jobs, accounted memory and runtime admission records. The caller-thread collector keeps at most 1,024 distinct admission records and fails on overflow; its storage belongs to the evidence driver, outside the decoder's memory allowance. + +The `hevc-native-rows` workflow compares complete native output with the qualified scalar C1 receipts at 1/2/4/8/automatic workers and stresses the actual row scheduler under ThreadSanitizer. Native sessions use the backend's ordinary memory allowance, recorded separately from the scalar reference allowance. Its reference mode checks every worker count against the pinned oracle before timing same-head native 1/4/automatic and FFmpeg one-thread/default calls. One warmup and five rotating measured rounds include startup, access-unit framing, packing and teardown. Per-call CPU and wall times, host load, input and binary hashes, semantic work and actual overlap remain in encrypted reports. A separate test-support counter counts successful overlapping Raster jobs issued under positive lag; complete-reference jobs never increment it. Conformance requires a positive-lag admission and row overlap in the same stream. Every timed native call labels whether row overlap occurred, including atomic or legacy controls with none. These shared-host calls are reference figures, not speed bars. + +Before this workflow exists on the default branch, dispatch `hevc-reference` from the candidate branch with `baseline=native`. A selected `only` input runs the small reference preflight; an empty selection with `diagnose_only=true` runs complete checks without timing. An empty selection with `diagnose_only=false` runs both phases. Ordinary complete-reference and inverse-comparison selections remain separate. + +First generate a non-WPP anchor with two pictures that depend on that anchor but not on each other. Barriers must prove both reconstruction jobs run before either finishes. A strict reference-chain control must prove that a dependent job cannot read an unfinished reference. A queue-only or all-serial test does not establish concurrency. + +Compare complete planes, coded hashes, presentation order, timestamps, origins, warnings and work counters with the synchronous decoder. Exercise independent and dependent segments, suffix hashes, output suppression, POC wrap, reference removal, sequence changes, geometry changes and partial worker start. Reverse worker completion and failure order. Inject errors, panics, callback failure and deadlines after real parallel entry, then prove all workers and pinned allocations are released. + +Locate independent cold memory and bin-budget boundaries with large parser scratch, skipped pictures, queued outputs and retained references. Require identical admission and first-error selection across thread counts. Poison reusable storage to detect omitted initialization, and delay reference completion to detect premature reads. Run complete conformance at 1, 2, 4 and 8 requested threads, recording host-bounded actual workers, plus sanitizer stress where supported. Each parallel conformance arm must report nonzero actual picture-job overlap, not just WPP work, and retain the picture and stream counts. + +Only then run paired reference measurements on existing inputs, using exact candidate and baseline commits and the qualified pinned FFmpeg oracle. Separate useful frame overlap, WPP execution and serial calls. Report output identity, coverage, wall time, CPU time and host capacity. Shared-runner figures are references, not acceptance bars. Every pending remote run has a timer backstop of at most thirty minutes. + +When overlap is low, an untimed `hevc_thread_diagnostics` build records fixed-size caller-thread counters on all thirty reference inputs with a fixed cap of thirty-two pictures. Timed runs allow up to sixty-four pictures, so compare the retained prepared-input hashes before attributing diagnostic counters to a timed input; different hashes describe different bytes. Barrier counters count each affected picture once per reason and can overlap across reasons; they describe scheduling decisions, not blocking duration or an additive partition of pictures. Event counters retain slice types, parameter-set NALs and SEI payload types separately. This build is diagnostic only: its overlap count is not substituted for the ordinary executable's timed-call counts, and its counters and public accessor are compiled out of normal builds. + +Sources remain ITU-T H.265 and conformance data only. The first implementation step is the generated overlap and dependency proof together with its bounded ownership path, before a wider scheduling rollout. diff --git a/docs/design/hevc-threading.md b/docs/design/hevc-threading.md index 8f7b6ff79..dd6f21255 100644 --- a/docs/design/hevc-threading.md +++ b/docs/design/hevc-threading.md @@ -20,7 +20,7 @@ Rows are assigned in increasing order to a bounded group of scoped workers. Depe The decoder session retains private worker allocations only after every row has reconciled successfully. Reuse requires the same worker count, geometry, bit depth, CTU shape, reference-storage shape, buffer lengths and total retained capacity. Every frame field and scratch buffer is reset from the new working picture before dispatch, with exhaustive destructuring requiring new fields to be handled. Workers borrow these private frames only within the thread scope. Failure drops them rather than returning partial state to the session. -The codec exposes `PictureDecoder::with_threads`. The backend follows the existing native decoder option shape, `NativeOptions { threads: Option }`, with `None` selecting available host parallelism and explicit counts bounded to 1 through 16. Memory admission can reduce the effective count to one. Threadless targets retain serial decoding. The existing command-line decoder thread option supplies this backend option without adding a new CLI flag. +The codec exposes `PictureDecoder::with_threads`. The backend follows the existing native decoder option shape, `NativeOptions { threads: Option }`, with `None` selecting available host parallelism and explicit counts bounded to 1 through 16 and available CPUs. Memory admission can reduce the effective count to one. Threadless targets retain serial decoding. The existing command-line decoder thread option supplies this backend option without adding a new CLI flag. ## Failure and resource bounds @@ -47,3 +47,5 @@ Warm-worker tests compare serial, cold and reused output across changed samples, Measure the existing reference inputs on GitHub Actions with full candidate and main SHAs, separate executables, warm-up and alternating repeated calls at automatic, 1, 4, 8 and 16 threads. The reference helper takes its memory budget from the backend policy defaults and records both that budget and the resolved automatic worker limit. First verify pixels against the qualified pinned oracle. Export enhancement metadata rather than applying it so both arms reconstruct the HEVC base raster, without resizing or pixel conversion. Report main time divided by candidate time, candidate one-thread time divided by candidate multithread time, CPU usage, host core count and WPP coverage. Separate WPP from non-WPP inputs when checking the automatic-worker cost. These are shared-runner reference figures, not speed acceptance bars. Retain the result privately and delete the disposable runs afterward. No change in this work modifies `inverse_residual`, filter arithmetic or control flow, the default build, decoder isolation, or the admitted coding-tool set. Sources are ITU-T H.265 and conformance data only. + +The next picture-level scheduling work is planned in [HEVC picture scheduling](hevc-frame-threading.md). It keeps complete-reference publication and does not make dependent picture chains parallel. diff --git a/scripts/hevc_boundary.py b/scripts/hevc_boundary.py index 9235f8252..a3caa73e1 100644 --- a/scripts/hevc_boundary.py +++ b/scripts/hevc_boundary.py @@ -28,6 +28,21 @@ def counter(value, maximum=None): return harness.integer(value) and (maximum is None or value <= maximum) +def valid_frame(value): + if not isinstance(value, dict) or value.get('type') != 'picture': + return False + clock = value.get('timestamp') + return ((clock is None or type(clock) is int and -(1 << 63) <= clock < 1 << 63) + and all(harness.integer(value.get(key), 1) for key in ('width', 'height', 'bit_depth', 'output_bytes')) + and value['width'] % 2 == value['height'] % 2 == 0 and value['bit_depth'] in (8, 10) + and value.get('packing') == ('yuv420p' if value['bit_depth'] == 8 else 'yuv420p10le') + and value['output_bytes'] == value['width'] * value['height'] * 3 // 2 + * (2 if value['bit_depth'] == 10 else 1) + and isinstance(value.get('output_md5'), str) and len(value['output_md5']) == 32 + and all(byte in '0123456789abcdef' for byte in value['output_md5']) + and valid_color(value.get('color'))) + + def annex_b_c5(value): require(isinstance(value, dict) and value.get('complete') is True and isinstance(value.get('results'), list) and counter(value.get('starts')) and value['starts'] == len(value['results']) >= 1 @@ -66,7 +81,7 @@ def thumbnail_c5(value): 'duplicate_pts_checked', 'dishonest_sync_checked')} -def compare(path, pictures, summary, *, coded_clock=False): +def compare_frames(path, pictures, summary, *, coded_clock=False): require(0 < path.stat().st_size < harness.CAPTURE_CAP, 'boundary_capture_size') rows = [json.loads(line, object_pairs_hook=harness.unique_object) for line in path.read_text().splitlines()] @@ -86,10 +101,9 @@ def compare(path, pictures, summary, *, coded_clock=False): require(valid_color(inherited), 'boundary_container_colour') for actual, expected in zip(frames, pictures): timestamp = expected['picture_index'] if coded_clock else expected['timestamp'] - clock = actual.get('timestamp') - require(clock is None or type(clock) is int and -(1 << 63) <= clock < 1 << 63, 'boundary_clock_type') - require(all(harness.integer(actual.get(key), 1) for key in ('width', 'height', 'bit_depth', 'output_bytes')) - and actual.get('timestamp') == timestamp + require(actual.get('timestamp') is None or type(actual.get('timestamp')) is int + and -(1 << 63) <= actual['timestamp'] < 1 << 63, 'boundary_clock_type') + require(valid_frame(actual) and actual.get('timestamp') == timestamp and all(actual.get(key) == expected[key] for key in FRAME_FIELDS), 'boundary_frame_mismatch') declared = expected.get('color_declarations') require(valid_color(declared), 'boundary_colour_declarations') @@ -98,6 +112,11 @@ def compare(path, pictures, summary, *, coded_clock=False): require(result.get('output_bytes') == sum(row['output_bytes'] for row in frames) == summary.get('output_bytes') and result.get('output_md5') == summary.get('output_md5'), 'boundary_whole_output_mismatch') + return frames, result + + +def compare(path, pictures, summary, *, coded_clock=False): + frames, result = compare_frames(path, pictures, summary, coded_clock=coded_clock) c5 = annex_b_c5(result.get('c5')) if coded_clock else thumbnail_c5(result.get('c5')) return dict(complete=True, outputs=len(frames), output_bytes=result['output_bytes'], input_bytes=result['input_bytes'], sent_packets=result['sent_packets'], diff --git a/scripts/hevc_decode.py b/scripts/hevc_decode.py index 3637bd824..d9249bd0d 100644 --- a/scripts/hevc_decode.py +++ b/scripts/hevc_decode.py @@ -265,20 +265,27 @@ def controls(args, stream, directory, expected): def threading_coverage(entries, threads): parallel_pictures = 0 + frame_parallel_pictures = 0 max_workers = 0 for entry in entries: if not entry['complete']: continue summary = entry['summary'] pictures, workers = summary.get('parallel_pictures'), summary.get('max_workers') + frames = summary.get('frame_parallel_pictures', 0) require(harness.integer(pictures) and pictures <= summary['decoded_pictures'] and harness.integer(workers) and workers <= min(threads, 16), 'native_thread_counters') - require(pictures == 0 or workers > 1 and summary['stats']['wpp_restarts'] > 0, + require(harness.integer(frames) and frames <= pictures and frames != 1, + 'native_frame_thread_counters') + require(pictures == 0 or workers > 1, 'native_thread_counters') + require(pictures == frames or summary['stats']['wpp_restarts'] > 0, 'native_parallel_wpp_work') parallel_pictures += pictures + frame_parallel_pictures += frames max_workers = max(max_workers, workers) require(threads == 1 or parallel_pictures > 0, 'native_parallel_coverage_missing') - return dict(parallel_pictures=parallel_pictures, max_workers=max_workers) + return dict(parallel_pictures=parallel_pictures, frame_parallel_pictures=frame_parallel_pictures, + max_workers=max_workers) def run(args): diff --git a/scripts/hevc_dependency.py b/scripts/hevc_dependency.py new file mode 100644 index 000000000..322e9460d --- /dev/null +++ b/scripts/hevc_dependency.py @@ -0,0 +1,325 @@ +"""Header-declared HEVC potential dependencies, not entropy-observed block references.""" + +import copy +import itertools + + +def _validate(data): + if not isinstance(data, dict): + raise ValueError('expected header dependency object') + if (data.get('mode') != 'header_dependencies' + or data.get('header_only') is not True or data.get('complete') is not True): + raise ValueError('expected complete header_dependencies mode') + for field in ('coded_pictures', 'reconstruction_required', 'output_pictures', 'nals'): + if type(data.get(field)) is not int: + raise ValueError(f'{field} must be an integer') + count = data['coded_pictures'] + if (not 1 <= count <= 32 or not 0 <= data['reconstruction_required'] <= count + or not 0 <= data['output_pictures'] <= count or data['nals'] < count): + raise ValueError('invalid header dependency counts') + pictures = data.get('pictures') + if not isinstance(pictures, list) or len(pictures) != count: + raise ValueError('pictures must cover every coded picture') + for index, picture in enumerate(pictures): + if not isinstance(picture, dict): + raise ValueError('picture must be an object') + if type(picture.get('index')) is not int or picture['index'] != index: + raise ValueError('pictures must be in coded index order') + if type(picture.get('poc')) is not int or not -(1 << 31) <= picture['poc'] < (1 << 31): + raise ValueError('picture POC must be an integer') + for field in ('reconstruction_required', 'wpp'): + if type(picture.get(field)) is not bool: + raise ValueError(f'picture {field} must be a boolean') + if 'envelope' in picture: + envelope = picture['envelope'] + if not isinstance(envelope, dict): + raise ValueError('envelope must be an object') + for field in ('coded_width', 'coded_height', 'width', 'height', 'bit_depth', + 'ctb_log2', 'dpb_capacity', 'highest_sps_layer'): + if type(envelope.get(field)) is not int: + raise ValueError('envelope fields must be integers') + if (not 0 < envelope['width'] <= envelope['coded_width'] <= (1 << 32) - 1 + or not 0 < envelope['height'] <= envelope['coded_height'] <= (1 << 32) - 1 + or envelope['bit_depth'] not in (8, 10) or envelope['ctb_log2'] not in (4, 5, 6) + or not 1 <= envelope['dpb_capacity'] <= 16 + or not 0 <= envelope['highest_sps_layer'] <= 6): + raise ValueError('invalid active SPS envelope') + types = picture.get('slice_types') + if (not isinstance(types, list) or not types + or any(type(value) is not int or value not in (0, 1, 2) for value in types) + or len(set(types)) != len(types)): + raise ValueError('slice_types must contain unique integers 0, 1 or 2') + for field in ('rps', 'active'): + refs = picture.get(field) + if not isinstance(refs, list) or len(refs) > 16: + raise ValueError(f'{field} must contain at most 16 references') + seen = set() + for ref in refs: + if (not isinstance(ref, list) or len(ref) != 3 + or type(ref[0]) is not int or type(ref[1]) is not int + or type(ref[2]) is not bool): + raise ValueError('reference must be [coded_index, poc, long_term_bool]') + target, poc, _ = ref + if not 0 <= target < index or target in seen: + raise ValueError('reference must be a unique earlier coded picture') + earlier = pictures[target] + if not earlier['reconstruction_required'] or earlier['poc'] != poc: + raise ValueError('reference must match a reconstructed picture POC') + seen.add(target) + if not picture['reconstruction_required'] and refs: + raise ValueError('skipped picture must have no references') + if any(ref not in picture['rps'] for ref in picture['active']): + raise ValueError('active reference must be in rps with the same identity') + if picture['reconstruction_required']: + if any(kind < 2 for kind in types) != bool(picture['active']): + raise ValueError('active references must match inter slice presence') + if sum(picture['reconstruction_required'] for picture in pictures) != data['reconstruction_required']: + raise ValueError('reconstruction_required count does not match pictures') + if 'output_events' in data: + events = data['output_events'] + if not isinstance(events, list) or len(events) != data['output_pictures']: + raise ValueError('output_events must cover every output picture') + emitted = set() + previous_boundary = 0 + for event in events: + if not isinstance(event, dict) or any(type(event.get(field)) is not int + for field in ('picture_index', 'poc', 'before_picture')): + raise ValueError('output event must contain integer picture_index, poc and before_picture') + index, boundary = event['picture_index'], event['before_picture'] + if not 0 <= index < boundary <= count or index in emitted or boundary < previous_boundary: + raise ValueError('output events must have unique earlier pictures and ordered boundaries') + picture = pictures[index] + if not picture['reconstruction_required'] or picture['poc'] != event['poc']: + raise ValueError('output event must match a reconstructed picture POC') + emitted.add(index) + previous_boundary = boundary + + +def _metrics(pictures, field): + vertices = [picture['index'] for picture in pictures if picture['reconstruction_required']] + refs = {index: {ref[0] for ref in pictures[index][field]} for index in vertices} + ancestors = {} + depths = {} + first_dependent = {index: len(pictures) for index in vertices} + for index in vertices: + ancestors[index] = set(refs[index]) + for ref in refs[index]: + ancestors[index].update(ancestors[ref]) + first_dependent[ref] = min(first_dependent[ref], index) + depths[index] = 1 + max((depths[ref] for ref in refs[index]), default=0) + + matched = {} + + def augment(index, seen): + for ancestor in sorted(ancestors[index]): + if ancestor in seen: + continue + seen.add(ancestor) + if ancestor not in matched or augment(matched[ancestor], seen): + matched[ancestor] = index + return True + return False + + matching = sum(augment(index, set()) for index in vertices) + participants = set() + non_wpp_participants = set() + for offset, a in enumerate(vertices): + for b in vertices[offset + 1:]: + if b < first_dependent[a]: + participants.update((a, b)) + if not pictures[a]['wpp'] and not pictures[b]['wpp']: + non_wpp_participants.update((a, b)) + admission_width = max((sum(a <= b < first_dependent[a] for a in vertices) + for b in vertices), default=0) + schedules = {} + for workers in (1, 2, 4, 8): + completed = set() + cursor = steps = maximum = 0 + overlap = set() + non_wpp_overlap = set() + while cursor < len(vertices): + jobs = [] + while cursor < len(vertices) and len(jobs) < workers: + index = vertices[cursor] + if not refs[index] <= completed: + break + jobs.append(index) + cursor += 1 + steps += 1 + maximum = max(maximum, len(jobs)) + if len(jobs) > 1: + overlap.update(jobs) + non_wpp = [index for index in jobs if not pictures[index]['wpp']] + if len(non_wpp) > 1: + non_wpp_overlap.update(non_wpp) + completed.update(jobs) + schedules[str(workers)] = { + 'steps': steps, + 'max_workers': maximum, + 'overlap_pictures': len(overlap), + 'non_wpp_overlap_pictures': len(non_wpp_overlap), + } + span = max(depths.values(), default=0) + return { + 'vertices': len(vertices), + 'edges': sum(len(value) for value in refs.values()), + 'immediate_predecessor_edges': sum(index - 1 in refs[index] for index in vertices), + 'long_term_edges': sum(ref[2] for index in vertices for ref in pictures[index][field]), + 'critical_path_unit_pictures': span, + 'maximum_antichain_width': len(vertices) - matching, + 'unit_cost_work_span_bound': len(vertices) / span if span else 0.0, + 'coded_admission_width': admission_width, + 'potential_overlap_pictures': len(participants), + 'non_wpp_potential_overlap_pictures': len(non_wpp_participants), + 'unit_duration_coded_order': schedules, + } + + +def _receive_barrier(pictures, field, events, *, delayed_sealing=False, output_lag3=False): + boundaries = {event['before_picture'] for event in events} + schedules = {} + for workers in ((4,) if output_lag3 else (1, 2, 4, 8)): + time = 0 + starts = {} + finishes = {} + pending = [] + max_pending = 0 + + def fence_finish(boundary): + return max((finish for earlier, finish in finishes.items() if earlier < boundary), default=0) + + for boundary in range(len(pictures) + 1): + index = boundary - 1 if delayed_sealing else boundary + if 0 <= index < len(pictures) and pictures[index]['reconstruction_required']: + picture = pictures[index] + time = max(time, max((finishes[ref[0]] for ref in picture[field]), default=0)) + active = [finish for finish in finishes.values() if finish > time] + if len(active) >= workers: + time = min(active) + starts[index] = time + finishes[index] = time + 1 + if delayed_sealing and boundary < len(pictures): + time = max(time, max((finishes[ref[0]] for ref in pictures[boundary][field]), default=0)) + if output_lag3: + for event in events: + if event['before_picture'] != boundary: + continue + if len(pending) == 3: + time = max(time, fence_finish(pending[0])) + pending.pop(0) + pending.append(boundary) + max_pending = max(max_pending, len(pending)) + while pending and fence_finish(pending[0]) <= time: + pending.pop(0) + elif boundary in boundaries: + time = max(time, fence_finish(boundary)) + if output_lag3: + time = max(time, max(finishes.values(), default=0)) + while pending: + time = max(time, fence_finish(pending[0])) + pending.pop(0) + overlap = set() + non_wpp_overlap = set() + for a, b in itertools.combinations(starts, 2): + if max(starts[a], starts[b]) < min(finishes[a], finishes[b]): + overlap.update((a, b)) + if not pictures[a]['wpp'] and not pictures[b]['wpp']: + non_wpp_overlap.update((a, b)) + maximum = max((sum(start <= time < finishes[index] for index, start in starts.items()) + for time in starts.values()), default=0) + schedules[str(workers)] = { + 'steps': max(finishes.values(), default=0), + 'max_workers': maximum, + 'overlap_pictures': len(overlap), + 'non_wpp_overlap_pictures': len(non_wpp_overlap), + } + if output_lag3: + schedules[str(workers)]['max_pending_outputs'] = max_pending + return schedules + + +def analyze(data): + """Validate a decoded JSON object; return replayable input and two graph models. + + RPS contains every used_by_curr_pic member. Active is the Rust active-list + owner's union across independently signalled segments: I segments contribute + no references and dependent segments inherit. Neither graph observes blocks. + """ + _validate(data) + result = { + 'graph': copy.deepcopy(data), + 'semantics': { + 'references': 'header-declared potential references, not entropy-observed block references', + 'rps': 'every used_by_curr_pic member', + 'active': 'Rust active-list owner union across independently signalled segments; ' + 'I segments have no references; dependent segments inherit', + 'vertices': 'reconstruction_required pictures only; indices retain coded order', + 'immediate_predecessor_edges': 'references to coded index minus one', + 'dag': 'optimistic dependency availability, not coded-order admission', + 'coded_admission': 'intervals [coded index, first later direct dependent); ' + 'no dependent ends at coded_pictures', + 'non_wpp': 'participants of pairs with both pictures non-WPP; optimistic frame-job ' + 'eligibility, not observed overlap', + 'unit_cost_work_span_bound': 'unit-picture work divided by unit-picture critical path; ' + 'equal-cost DAG bound, not measured throughput', + 'unit_duration_coded_order': 'equal-cost unit-duration list scheduling at 1/2/4/8 workers; ' + 'admit next only with all references completed and a free slot, ' + 'then complete active jobs in one step; not measured throughput ' + 'or a general duration-free speedup', + 'excluded': ['output', 'reorder', 'capacity', 'SEI', 'memory barriers'], + }, + 'rps': _metrics(data['pictures'], 'rps'), + 'active': _metrics(data['pictures'], 'active'), + } + if 'output_events' in data: + for field in ('rps', 'active'): + result[field]['unit_duration_receive_barrier'] = _receive_barrier( + data['pictures'], field, data['output_events']) + result[field]['unit_duration_delayed_sealing_receive_barrier'] = _receive_barrier( + data['pictures'], field, data['output_events'], delayed_sealing=True) + lagged = {'workers': 4, 'output_lag_limit': 3} + for mode in ('picture_ready', 'delayed_sealing'): + metrics = _receive_barrier(data['pictures'], field, data['output_events'], + delayed_sealing=mode == 'delayed_sealing', output_lag3=True)['4'] + lagged[mode] = {'full_picture_references': metrics, + 'row_references_worst_case': copy.deepcopy(metrics)} + result[field]['unit_duration_output_lag3'] = lagged + result['semantics']['unit_duration_receive_barrier'] = ( + 'optimistic picture-ready access-unit equal-cost model at 1/2/4/8 workers; ' + 'one coded picture per send and receive-until-None after every send, including skipped pictures; ' + 'admit current required picture only with completed references and a free worker; ' + 'after admitting i, an output event before_picture=i waits for every job with coded index> picture['ctb_log2'] + rows = picture.get('rows') + if not 1 <= count <= 2048 or not isinstance(rows, list) or len(rows) != count: + raise ValueError('rows must cover the CTU geometry within the 2048-row cap') + allowed = {ref[0]: ref[1] for ref in identity['rps']} + for ordinal, row in enumerate(rows): + if not isinstance(row, dict) or type(row.get('row')) is not int or row['row'] != ordinal: + raise ValueError('rows must be in complete CTU row order') + for field in ('sample_calls', 'motion_calls'): + if type(row.get(field)) is not int or row[field] < 0: + raise ValueError('row call counts must be nonnegative integers') + refs = row.get('references') + if not isinstance(refs, list) or len(refs) > 16: + raise ValueError('row references must contain at most 16 identities') + seen = set() + reads = [0, 0] + for ref in refs: + if (not isinstance(ref, list) or len(ref) != 4 + or type(ref[0]) is not int or type(ref[1]) is not int): + raise ValueError('row reference must contain index, POC, sample row and motion row') + index, poc, sample, motion = ref + if (index in seen or index not in earlier or index not in allowed + or earlier[index]['poc'] != poc or allowed[index] != poc): + raise ValueError('row reference must match a unique earlier real RPS picture') + if sample is None and motion is None: + raise ValueError('row reference must record an actual sample or motion read') + for kind, maximum in enumerate((sample, motion)): + if maximum is not None: + if type(maximum) is not int or not 0 <= maximum < len(earlier[index]['rows']): + raise ValueError('read maximum must lie inside reference CTU rows') + reads[kind] += 1 + seen.add(index) + for distinct_reads, field in zip(reads, ('sample_calls', 'motion_calls')): + if row[field] < distinct_reads or bool(row[field]) != bool(distinct_reads): + raise ValueError('row call counts must agree with actual referenced reads') + if identity['slice_types'] == [2] and refs: + raise ValueError('I-only picture cannot contain prediction reads') + earlier[picture['index']] = picture + + +def _tasks(data, filter_lag): + pictures = {picture['index']: picture for picture in data['pictures']} + nodes = {index: [(index, row) for row in range(1 if picture['tiles'] else len(picture['rows']))] + for index, picture in pictures.items()} + costs = {} + dependencies = {} + depths = {} + for index, picture in pictures.items(): + header = data['header_dependencies']['pictures'][index] + for row, node in enumerate(nodes[index]): + costs[node] = len(picture['rows']) if picture['tiles'] else 1 + refs = {nodes[index][row - 1]} if row else set() + if filter_lag is None or picture['tiles']: + if not row: + refs.update(nodes[ref[0]][-1] for ref in header['rps']) + else: + for target, _, sample, motion in picture['rows'][row]['references']: + if pictures[target]['tiles']: + refs.add(nodes[target][-1]) + continue + if sample is not None: + refs.add(nodes[target][min(sample + filter_lag, len(nodes[target]) - 1)]) + if motion is not None: + refs.add(nodes[target][motion]) + dependencies[node] = refs + depths[node] = costs[node] + max((depths[ref] for ref in refs), default=0) + return nodes, costs, dependencies, max(depths.values(), default=0) + + +def _simulate(data, filter_lag, delayed_sealing): + nodes, costs, dependencies, span = _tasks(data, filter_lag) + header = data['header_dependencies'] + progress = {index: 0 for index in nodes} + remaining = dict(costs) + admitted = set() + completed = set() + done = set() + pending = [] + overlap = set() + time = maximum_workers = maximum_admitted = maximum_pending = 0 + + def tick(): + nonlocal time, maximum_workers + ready = [index for index in sorted(admitted) + if dependencies[nodes[index][progress[index]]] <= done] + if not ready: + raise ValueError('row model has no runnable admitted work') + maximum_workers = max(maximum_workers, len(ready)) + if len(ready) > 1: + overlap.update(ready) + for index in ready: + node = nodes[index][progress[index]] + remaining[node] -= 1 + if not remaining[node]: + done.add(node) + progress[index] += 1 + if progress[index] == len(nodes[index]): + completed.add(index) + admitted.remove(index) + time += 1 + + def prefix_complete(boundary): + return all(index in completed for index in nodes if index < boundary) + + for call in range(header['coded_pictures'] + 1): + index = call - int(delayed_sealing) + if index in nodes: + required = {ref[0] for ref in header['pictures'][index]['rps']} if filter_lag is None else set() + while len(admitted) == 4 or not required <= completed: + tick() + admitted.add(index) + maximum_admitted = max(maximum_admitted, len(admitted)) + if delayed_sealing and filter_lag is None and call < header['coded_pictures']: + required = {ref[0] for ref in header['pictures'][call]['rps']} + while not required <= completed: + tick() + for event in header['output_events']: + if event['before_picture'] != call: + continue + if len(pending) == 3: + while not prefix_complete(pending[0]): + tick() + pending.pop(0) + pending.append(call) + maximum_pending = max(maximum_pending, len(pending)) + while pending and prefix_complete(pending[0]): + pending.pop(0) + while admitted: + tick() + pending.clear() + work = sum(costs.values()) + mean_rows = work / len(nodes) if nodes else 0 + return { + 'steps': time, + 'max_workers': maximum_workers, + 'serial_row_work': work, + 'critical_path_row_units': span, + 'normalized_picture_work': len(nodes), + 'normalized_picture_span': span / mean_rows if mean_rows else 0.0, + 'normalized_picture_steps': time / mean_rows if mean_rows else 0.0, + 'serial_work_over_steps': work / time if time else 0.0, + 'unit_row_work_span_bound': work / span if span else 0.0, + 'four_worker_work_span_bound': min(4.0, work / span) if span else 0.0, + 'max_in_flight_pictures': maximum_admitted, + 'max_pending_outputs': maximum_pending, + 'overlap_pictures': len(overlap), + } + + +def analyze(data): + """Validate actual single-worker read traces and replay hypothetical four-worker models.""" + _validate(data) + models = {} + for mode in ('picture_ready', 'delayed_sealing'): + models[mode] = {name: _simulate(data, lag, mode == 'delayed_sealing') + for name, lag in (('filter_lag1', 1), ('filter_lag2', 2), + ('whole_reference_control', None))} + return { + 'graph': copy.deepcopy(data), + 'tiled_picture_indices': [picture['index'] for picture in data['pictures'] if picture['tiles']], + 'models': models, + 'semantics': { + 'capture': 'actual clipped interpolation-read maxima across all components and separate ' + 'TMVP motion-field read maxima, not header-declared potential reads', + 'cost': 'hypothetical equal CTU-row unit costs including amortized filter work; ' + 'serial rows within each picture, at most four unfinished admitted pictures and four ' + 'CPU workers; waiting pictures consume no CPU worker; unequal row counts retain unequal work', + 'filter_lag1': 'hypothetical incremental filtering, not current whole-picture filtering; ' + 'sample row q waits for reconstruction row min(q+1,last) to finish for ' + 'horizontal deblocking and the one-pixel SAO margin; motion reads wait only for their row; ' + 'the last row publishes at picture completion', + 'filter_lag2': 'separately labeled conservative sample lookahead min(q+2,last); ' + 'motion-field readiness is unchanged', + 'tiles': 'whole-picture readiness and publication fallback: a tiled picture is an atomic job ' + 'costing its row count, starts after complete header RPS references and publishes only ' + 'on completion; its row labels imply no raster-row entropy order; consumers wait for ' + 'complete tiled references', + 'admission': 'picture_ready admits the current picture; delayed_sealing submits the previous ' + 'picture at the next boundary and seals the final picture at EOF; row variants ' + 'hypothetically replace whole-reference input gates with per-row waits; ' + 'whole_reference_control retains complete used-RPS submission and current-input gates', + 'outputs': 'FIFO lag-three logical pending emitted frames; before appending a fourth record ' + 'join and consume the oldest all-rows-of-all-earlier-pictures causal fence; ' + 'receive-until-None consumes only ready front records; EOF joins all jobs and drains ' + 'records; publication is not reclamation and never removes reference availability', + 'normalization': 'divide row-unit span and modeled steps by mean rows per required picture; ' + 'normalized_picture_work is the required picture count; work/span ignores ' + 'admission and output fences; four_worker_work_span_bound is min(4,work/span), ' + 'a DAG speedup upper bound conditional on equal-row costs and hypothetical filter ' + 'publication, while serial_work_over_steps uses the modeled schedule', + 'limits': 'optimistic abstract producer/consumer rendezvous inside admission, not implementability ' + 'under the current Decoder send/receive API, an allocated-byte/capacity proof, ' + 'measured throughput or a duration-free speedup; no pipeline implementation, ' + 'memory, SEI, packet-grouping or CPU timing model', + }, + } diff --git a/scripts/hevc_threading.py b/scripts/hevc_threading.py index ebc533434..0b178f531 100644 --- a/scripts/hevc_threading.py +++ b/scripts/hevc_threading.py @@ -2,6 +2,7 @@ """Paired HEVC threading reference with complete-output identity checks.""" import argparse +import hashlib import json import os import pathlib @@ -15,11 +16,195 @@ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) import hevc_reference as reference import hevc_conformance as conformance +import hevc_decode +import hevc_boundary +import hevc_dependency +import hevc_row_dependency import oracle_pin import round_files import test_oracle THREADS = (0, 1, 4, 8, 16) +INPUT_IDS = tuple([f'd{index:02}' for index in range(26)] + [f'c{index:02}' for index in range(4)]) +ROW_INPUT_IDS = tuple(identity for identity in INPUT_IDS + if identity not in {'d03', 'd06', 'd07', 'd10', 'd14', 'd22'}) +DIAGNOSTIC_KEYS = frozenset(('barrier_' + name) for name in ( + 'workers_disabled', 'wpp', 'output_random_access', 'output_capacity', 'output_reorder', + 'output_latency', 'reference_unavailable', 'memory_input', 'memory_scratch', 'memory_frame', + 'queue_budget', 'queue_memory', 'snapshot_allocation', 'window_full', 'duplicate_hash', 'worker_spawn' +)) | frozenset(('event_' + name) for name in ( + 'picture_i', 'picture_p', 'picture_b', 'prefix_sei', 'suffix_sei_informational', 'suffix_sei_hash', + 'parameter_set', 'queue_admitted', 'spawned_job', 'caller_job', 'forced_flush', 'reconcile', 'sequence_end', + 'sei_type_0', 'sei_type_1', 'sei_type_5', 'sei_type_129', 'sei_type_132', 'sei_type_other' +)) + + +def diagnostic_record(data): + if not isinstance(data, dict): + raise ValueError('diagnostic_record_missing') + fields = ('pictures', 'decoded_pictures', 'bins', 'requested_threads', 'host_threads', + 'thread_limit', 'max_memory_bytes', 'parallel_pictures', + 'frame_parallel_pictures', 'max_workers') + if any(type(data.get(key)) is not int or not 0 <= data[key] <= (1 << 64) - 1 for key in fields): + raise ValueError('diagnostic_counts_missing') + if not (0 < data['pictures'] <= data['decoded_pictures'] <= 32 and data['bins'] > 0 + and data['max_memory_bytes'] > 0 and data['requested_threads'] == 4 + and data['host_threads'] > 0 and data['thread_limit'] == min(4, data['host_threads']) + and 1 <= data['max_workers'] <= data['thread_limit'] + and 0 <= data['frame_parallel_pictures'] <= data['parallel_pictures'] <= data['decoded_pictures'] + and data['frame_parallel_pictures'] != 1 + and (data['parallel_pictures'] == 0 or data['max_workers'] > 1)): + raise ValueError('diagnostic_counts_invalid') + counters = data.get('thread_diagnostics') + if not isinstance(counters, list) or not counters: + raise ValueError('diagnostic_counters_missing') + histogram = {} + events = {} + for pair in counters: + if not isinstance(pair, list) or len(pair) != 2: + raise ValueError('diagnostic_counter_invalid') + key, value = pair + if not isinstance(key, str) or re.fullmatch(r'(barrier|event)_[a-z0-9_]+', key) is None \ + or type(value) is not int or not 0 <= value <= (1 << 64) - 1 \ + or key in histogram or key in events: + raise ValueError('diagnostic_counter_invalid') + if key.startswith('barrier_'): + if value > data['decoded_pictures']: + raise ValueError('diagnostic_barrier_denominator') + histogram[key] = value + else: + events[key] = value + if set(histogram) | set(events) != DIAGNOSTIC_KEYS: + raise ValueError('diagnostic_counters_missing') + return dict(raw=data, barrier_histogram=histogram, events=events) + + +def diagnostic_summary(files, selected): + ids = [row['id'] for row in files] + successful = [row for row in files if row['status'] == 'decoded'] + histogram = {} + for row in successful: + for key, value in row['barrier_histogram'].items(): + histogram[key] = histogram.get(key, 0) + value + return dict(selected_inputs=len(selected), attempted_inputs=len(files), decoded_inputs=len(successful), + errors=len(files) - len(successful), + missing_inputs=len(set(selected) - set(ids)), + complete=bool(selected) and len(ids) == len(set(ids)) and set(ids) == set(selected), + pictures=sum(row['raw']['pictures'] for row in successful), + decoded_pictures=sum(row['raw']['decoded_pictures'] for row in successful), + frame_parallel_pictures=sum(row['raw']['frame_parallel_pictures'] for row in successful), + inputs_with_overlap=sum(row['raw']['frame_parallel_pictures'] > 0 for row in successful), + barrier_histogram=histogram) + + +def diagnostics(argv): + parser = argparse.ArgumentParser(description='Internal Actions-only picture-job diagnostics') + for name in ('native', 'oracle-bin', 'out'): + parser.add_argument('--' + name, required=True, type=pathlib.Path) + parser.add_argument('--candidate', required=True) + selection = parser.add_mutually_exclusive_group() + selection.add_argument('--dependencies-only', action='store_true') + selection.add_argument('--rows-only', action='store_true') + args = parser.parse_args(argv) + if sys.platform != 'linux' or os.environ.get('GITHUB_ACTIONS') != 'true': + parser.error('diagnostics_require_actions_linux') + if re.fullmatch('[0-9a-f]{40}', args.candidate) is None: + parser.error('full_candidate_sha_required') + selected_ids = ROW_INPUT_IDS if args.rows_only else INPUT_IDS + args.only = set(selected_ids) + args.frames = 32 + oracle = test_oracle.require(args.oracle_bin / 'ffmpeg', args.oracle_bin / 'ffprobe') + args.out.mkdir(parents=True, exist_ok=True) + report = dict(candidate=args.candidate, candidate_sha256=conformance.digest(args.native), + oracle=oracle_pin.load()['BUILD'], selected_ids=list(selected_ids), frames_cap=32, + requested_threads=1 if args.rows_only else 0 if args.dependencies_only else 4, + mode='row_dependencies' if args.rows_only else 'header_dependencies' if args.dependencies_only else 'decode', + header_only=args.dependencies_only, + scope='actual decoded inter-reference reads by CTU row; synchronous reconstruction; ' + 'diagnostic models only, no baseline, timing or profiling' + if args.rows_only else + 'header-only RPS and active-list dependency replay; no reconstruction, timing or profiling' + if args.dependencies_only else + 'picture-job diagnostics only; no pixel comparison, baseline, timing or profiling; ' + 'barriers count affected pictures per reason and may overlap across reasons; ' + 'events are not picture counts', + host=dict(cpu=next(line.split(':', 1)[1].strip() for line in pathlib.Path('/proc/cpuinfo').read_text().splitlines() if line.startswith('model name')), + cores=os.cpu_count(), affinity=sorted(os.sched_getaffinity(0)), kernel=platform.release()), + complete=False, passed=False, files=[]) + pool = round_files.pool() + pool.limits.max_requests = 4000 + pool.limits.max_bytes = 32 << 30 + try: + with tempfile.TemporaryDirectory(prefix='hevc-thread-diagnostics-') as temp: + seen = set() + for identity, source, original_sha in reference.inputs(args, oracle, pathlib.Path(temp), pool): + if identity not in args.only or identity in seen: + raise ValueError('diagnostic_input_identity') + seen.add(identity) + row = dict(id=identity, source_sha256=original_sha, status='failed') + report['files'].append(row) + try: + row['input_sha256'] = conformance.digest(source) + mode = ('rows', '1') if args.rows_only else ('dependencies', '1') if args.dependencies_only else ('decode', '4') + child = subprocess.run([str(args.native), str(source), *mode], + capture_output=True, check=True, timeout=600, env=reference.CHILD_ENV) + data = json.loads(child.stdout) + row['raw'] = data + if args.rows_only: + row.update(row_dependency_ceiling=hevc_row_dependency.analyze(data), status='decoded') + elif args.dependencies_only: + if 'output_events' not in data: + raise ValueError('dependency_output_events_missing') + row.update(dependency_ceiling=hevc_dependency.analyze(data), status='parsed') + else: + row.update(diagnostic_record(data), status='decoded') + except (ValueError, OSError, subprocess.SubprocessError) as error: + row['failure'] = type(error).__name__ + if args.rows_only and isinstance(error, ValueError): + row['failure_reason'] = 'row_dependency_record_invalid' + if args.dependencies_only and isinstance(error, ValueError): + row['failure_reason'] = 'dependency_record_invalid' + if isinstance(error, ValueError) and str(error) in { + 'diagnostic_record_missing', 'diagnostic_counts_missing', 'diagnostic_counts_invalid', + 'diagnostic_counters_missing', 'diagnostic_counter_invalid', 'diagnostic_barrier_denominator'}: + row['failure_reason'] = str(error) + if isinstance(error, subprocess.CalledProcessError): + row['exit_code'] = error.returncode + (args.out / 'results.json').write_text(json.dumps(report, indent=2) + '\n') + except (ValueError, OSError, KeyError, TypeError, subprocess.SubprocessError) as error: + report['failure'] = type(error).__name__ + finally: + pool.close() + report['r2_requests'] = pool.limits.requests + if args.rows_only: + rows = report['files'] + successful = [row for row in rows if row['status'] == 'decoded'] + ids = [row['id'] for row in rows] + report['summary'] = dict(selected_inputs=len(args.only), attempted_inputs=len(rows), + decoded_inputs=len(successful), errors=len(rows) - len(successful), + missing_inputs=len(args.only - set(ids)), + complete=len(ids) == len(set(ids)) and set(ids) == args.only, + decoded_pictures=sum(row['raw']['decoded_pictures'] for row in successful)) + elif args.dependencies_only: + rows = report['files'] + successful = [row for row in rows if row['status'] == 'parsed'] + ids = [row['id'] for row in rows] + report['summary'] = dict(selected_inputs=len(args.only), attempted_inputs=len(rows), + parsed_inputs=len(successful), errors=len(rows) - len(successful), + missing_inputs=len(args.only - set(ids)), + complete=len(ids) == len(set(ids)) and set(ids) == args.only, + coded_pictures=sum(row['raw']['coded_pictures'] for row in successful), + reconstruction_required=sum(row['raw']['reconstruction_required'] for row in successful)) + else: + report['summary'] = diagnostic_summary(report['files'], args.only) + report['summary']['preparation_errors'] = int('failure' in report) + report['summary']['errors'] += report['summary']['preparation_errors'] + report['complete'] = report['summary']['complete'] + report['passed'] = report['complete'] and report['summary']['errors'] == 0 and 'failure' not in report + (args.out / 'results.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(dict(selected=len(args.only), attempted=report['summary']['attempted_inputs'], + errors=report['summary']['errors'], complete=report['complete'], passed=report['passed']))) + return 0 if report['passed'] else 1 def syntax_record(item, source, directory, oracle, parser): @@ -72,12 +257,291 @@ def inspect(item, source, directory, oracle, **unused): def normalized(value): if isinstance(value, dict): - return {key: normalized(item) for key, item in value.items() if key not in {'memory_bytes', 'parallel_pictures', 'max_workers'}} + return {key: normalized(item) for key, item in value.items() if key not in {'memory_bytes', 'parallel_pictures', 'frame_parallel_pictures', 'max_workers'}} if isinstance(value, list): return [normalized(item) for item in value] return value +def compare_conformance(root): + reports = [conformance.read_json(root / f'threads-{threads}' / 'result.json') + for threads in (1, 2, 4, 8)] + for report in reports: + conformance.require(report.get('complete') is True and report.get('passed') is True + and report.get('scope') == 'complete_c1' and report.get('results'), + 'thread_conformance_incomplete') + expected = normalized(reports[0]) + conformance.require(all(normalized(report) == expected for report in reports[1:]), + 'thread_conformance_mismatch') + coverage = {str(threads): hevc_decode.threading_coverage(report['results'], threads) + for threads, report in zip((1, 2, 4, 8), reports)} + for threads, report in zip((1, 2, 4, 8), reports): + coverage[str(threads)]['picture_job_streams'] = sum( + row['summary'].get('frame_parallel_pictures', 0) > 0 for row in report['results']) + conformance.require(all(coverage[str(threads)]['frame_parallel_pictures'] > 0 + for threads in (2, 4, 8)), 'native_picture_job_coverage_missing') + canonical = json.dumps(expected, sort_keys=True, separators=(',', ':')).encode() + return dict(passed=True, streams=len(expected['results']), threading=coverage, + pictures=sum(row['summary']['decoded_pictures'] for row in expected['results']), + normalized_sha256=hashlib.sha256(canonical).hexdigest()) + + +SEMANTIC_FIELDS = ('headers', 'decoded_pictures', 'skipped_pictures', 'skipped_picture_hashes', + 'skipped_reserved_hashes', 'stats', 'inter') + + +def native_receipt(stdout, threads, *, check): + def counters(value): + if isinstance(value, dict): + return all(isinstance(key, str) and counters(item) for key, item in value.items()) + if isinstance(value, list): + return all(counters(item) for item in value) + return type(value) is int and 0 <= value <= (1 << 64) - 1 + rows = [json.loads(line, object_pairs_hook=conformance.unique_object) for line in stdout.splitlines()] + conformance.require(rows and all(isinstance(row, dict) for row in rows), 'native_rows_records') + record = rows[-1] + conformance.require(record.get('type') == 'summary' and record.get('complete') is True + and record.get('boundary') is True and record.get('backend') == 'native', + 'native_rows_completion') + fields = ('pictures', 'decoded_pictures', 'bins', 'requested_threads', 'host_threads', 'thread_limit', + 'max_workers', 'parallel_pictures', 'frame_parallel_pictures', 'max_memory_bytes', + 'memory_bytes', 'input_bytes', 'sent_packets') + fields += ('row_parallel_pictures',) + conformance.require(all(type(record.get(key)) is int and 0 <= record[key] <= (1 << 64) - 1 for key in fields), + 'native_rows_counts') + conformance.require(record['requested_threads'] == threads and record['host_threads'] > 0 + and record['thread_limit'] == min(threads or 16, record['host_threads'], 16) + and 1 <= record['max_workers'] <= record['thread_limit'] + and 0 < record['pictures'] <= record['decoded_pictures'] and record['bins'] > 0 + and 0 <= record['frame_parallel_pictures'] <= record['parallel_pictures'] + <= record['decoded_pictures'] and record['frame_parallel_pictures'] != 1 + and (record['parallel_pictures'] == 0 or record['max_workers'] > 1) + and record['row_parallel_pictures'] <= record['frame_parallel_pictures'] + and record['max_memory_bytes'] > 0 and record['input_bytes'] > 0 + and 0 < record['memory_bytes'] <= record['max_memory_bytes'] + and record['sent_packets'] > 0, 'native_rows_count_identity') + semantic = record.get('semantic') + conformance.require(isinstance(semantic, dict) and set(semantic) == set(SEMANTIC_FIELDS) + and isinstance(semantic.get('headers'), dict) and isinstance(semantic.get('stats'), dict) + and isinstance(semantic.get('inter'), dict) + and semantic['decoded_pictures'] == record['decoded_pictures'] + and semantic['headers'].get('pictures') == semantic['decoded_pictures'] + semantic['skipped_pictures'] + and semantic['headers'].get('nals', 0) >= semantic['headers']['pictures'] + and semantic['stats'].get('ctus', 0) >= semantic['decoded_pictures'] + and semantic['headers'].get('output_pictures') == record['pictures'] + and semantic['stats'].get('bins') == record['bins'], 'native_rows_semantic_identity') + warnings = semantic['headers'].get('warnings') + conformance.require(isinstance(warnings, dict) and set(warnings) == { + 'profile_bit_depth', 'invalid_sample_aspect_ratio', 'temporal_nesting_nal_type'} + and all(type(value) is bool for value in warnings.values()) + and counters({**semantic, 'headers': {key: value for key, value in semantic['headers'].items() + if key != 'warnings'}}), 'native_rows_semantic_types') + admissions = record.get('admissions') + conformance.require(isinstance(admissions, list) and (record['thread_limit'] != 1 or not admissions) + and (record['thread_limit'] != 1 or record['parallel_pictures'] == 0), 'native_rows_admissions') + for admission in admissions: + conformance.require(isinstance(admission, dict) + and all(type(admission.get(key)) is int and admission[key] >= 0 for key in + ('ceiling', 'workers', 'lag', 'frame_bytes', 'reserved_bytes', 'limit_bytes', 'sps_id')) + and 1 <= admission['workers'] <= admission['ceiling'] == record['thread_limit'] + and admission['lag'] < admission['workers'] and admission['frame_bytes'] > 0 + and admission['limit_bytes'] == record['max_memory_bytes'] + and admission['reserved_bytes'] + admission['lag'] * admission['frame_bytes'] + <= admission['limit_bytes'] and admission['sps_id'] <= 15 + and isinstance(admission.get('sps_sha256'), list) and len(admission['sps_sha256']) == 32 + and all(type(byte) is int and 0 <= byte <= 255 for byte in admission['sps_sha256']), + 'native_rows_admission_bound') + conformance.require(record['row_parallel_pictures'] == 0 or any(item['lag'] > 0 for item in admissions), + 'native_rows_overlap_without_positive_lag') + conformance.require((check and len(rows) == record['pictures'] + 1 + and all(row.get('type') == 'picture' for row in rows[:-1])) + or (not check and len(rows) == 1), 'native_rows_output_denominator') + if check: + conformance.require(isinstance(record.get('output_md5'), str) + and re.fullmatch('[0-9a-f]{32}', record['output_md5']) is not None + and record.get('output_bytes') == sum(frame.get('output_bytes', 0) for frame in rows[:-1]) + and all(hevc_boundary.valid_frame(frame) for frame in rows[:-1]), + 'native_rows_frame_receipt') + else: + conformance.require(record.get('output_md5') is None and record.get('output_bytes') is None, + 'native_rows_decode_did_not_disable_hashing') + return rows[:-1], record + + +def native_conformance(argv): + parser = argparse.ArgumentParser(description='Compare real native row output with qualified complete receipts') + for name in ('registration', 'census', 'data', 'out', 'parser', 'native', 'scalar-result'): + parser.add_argument('--' + name, required=True, type=pathlib.Path) + args = parser.parse_args(argv) + conformance.require(sys.platform == 'linux', 'linux_actions_required') + raw = conformance.read_json(args.scalar_result) + conformance.require(raw.get('complete') is True and raw.get('passed') is True + and raw.get('scope') == 'complete_c1', 'complete_scalar_c1_required') + conformance.require(all(raw['identities'][name] == conformance.digest(path) for name, path in ( + ('registration', args.registration), ('census', args.census), ('parser', args.parser), + ('acquisition', args.data / 'acquisition.json'))), 'native_rows_reference_identity') + selected = hevc_decode.streams(args, conformance.read_json(args.census)) + identities = [dict(test=item['case']['test'], sha256=item['member']['sha256']) for item in selected] + conformance.require(identities == raw['denominator'] and len(selected) == len(raw['results']), + 'native_rows_membership') + args.out.mkdir(parents=True, exist_ok=False) + report = dict(complete=False, passed=False, scope='native_rows_complete_c1', + native_sha256=conformance.digest(args.native), reference_sha256=conformance.digest(args.scalar_result), + streams=len(selected), results=[]) + try: + for ordinal, (stream, expected) in enumerate(zip(selected, raw['results'])): + source = stream['source'] + conformance.require(expected.get('complete') is True and conformance.digest(source) == expected['sha256'] + == stream['member']['sha256'] and expected['test'] == stream['case']['test'], + 'native_rows_source_identity') + row = dict(sha256=expected['sha256'], threads={}) + report['results'].append(row) + for threads in (1, 2, 4, 8, 0): + code, output, _ = conformance.child([args.native, source, 'check', str(threads)], + args.out / str(ordinal), f'native-{threads}', timeout=600) + conformance.require(code == 0, 'native_rows_decode') + frames, summary = native_receipt(output.read_bytes(), threads, check=True) + hevc_boundary.compare_frames(output, expected['pictures'], expected['summary'], coded_clock=True) + conformance.require(summary['input_bytes'] == source.stat().st_size + and normalized(summary['semantic']) == normalized({key: expected['summary'][key] + for key in SEMANTIC_FIELDS}), + 'native_rows_semantic_mismatch') + row['threads'][str(threads)] = dict(outputs=len(frames), semantic=summary['semantic'], + admissions=summary['admissions'], max_workers=summary['max_workers'], + frame_parallel_pictures=summary['frame_parallel_pictures'], row_parallel_pictures=summary['row_parallel_pictures'], + memory_bytes=summary['memory_bytes'], + max_memory_bytes=summary['max_memory_bytes']) + conformance.require(conformance.digest(source) == expected['sha256'], 'native_rows_source_changed') + conformance.write_json(args.out / 'result.json', report) + conformance.require(conformance.digest(args.native) == report['native_sha256'] + and conformance.digest(args.scalar_result) == report['reference_sha256'], + 'native_rows_run_identity_changed') + report['coverage'] = {str(threads): dict( + admitted_streams=sum(any(item['lag'] > 0 for item in row['threads'][str(threads)]['admissions']) + for row in report['results']), + overlap_streams=sum(row['threads'][str(threads)]['frame_parallel_pictures'] > 0 for row in report['results']), + row_overlap_streams=sum(row['threads'][str(threads)]['row_parallel_pictures'] > 0 + and any(item['lag'] > 0 for item in row['threads'][str(threads)]['admissions']) + for row in report['results']), + row_parallel_pictures=sum(row['threads'][str(threads)]['row_parallel_pictures'] for row in report['results']), + frame_parallel_pictures=sum(row['threads'][str(threads)]['frame_parallel_pictures'] for row in report['results'])) + for threads in (1, 2, 4, 8, 0)} + conformance.require(all(report['coverage'][str(threads)]['row_overlap_streams'] > 0 for threads in (2, 4, 8)), + 'native_rows_real_coverage_missing') + report.update(complete=True, passed=True) + finally: + conformance.write_json(args.out / 'result.json', report) + return 0 + + +def measure_native(args, oracle, source, identity): + row = dict(id=identity, input_sha256=conformance.digest(source), status='checking', + load_average_before=list(os.getloadavg())) + row.update(reference.verify(args.native, oracle, source, timeout=600)) + expected = None + for threads in (1, 2, 4, 8, 0): + stdout, _ = reference.command([args.native, source, 'check', threads], timeout=600) + frames, summary = native_receipt(stdout, threads, check=True) + signature = dict(frames=frames, semantic=summary['semantic'], output_md5=summary['output_md5']) + if expected is None: + expected = signature + row['work'] = {key: summary[key] for key in ('pictures', 'decoded_pictures', 'bins')} + conformance.require(summary['pictures'] == row['frames'] and summary['output_md5'] == row['output_md5'], + 'native_rows_oracle_identity') + conformance.require(signature == expected and summary['input_bytes'] == source.stat().st_size, + 'native_rows_thread_output_mismatch') + commands = {f'candidate_{threads}': [args.native, source, 'decode', threads] for threads in (1, 4, 0)} + commands['ffmpeg_one'] = reference.ffmpeg_command(oracle.ffmpeg, source, True, + ['-progress', 'pipe:1', '-f', 'null', '-']) + commands['ffmpeg_default'] = reference.ffmpeg_command(oracle.ffmpeg, source, False, + ['-progress', 'pipe:1', '-f', 'null', '-']) + names = tuple(commands) + calls = [] + for repeat in range(-1, args.repeats): + offset = max(0, repeat) % len(names) + order = names[offset:] + names[:offset] + if repeat % 2: + order = tuple(reversed(order)) + for name in order: + stdout, receipt = reference.command(commands[name], timeout=600) + reference.check_work('coreav' if name.startswith('candidate_') else name, stdout, row['frames']) + if name.startswith('candidate_'): + _, summary = native_receipt(stdout, int(name.split('_')[1]), check=False) + conformance.require({key: summary[key] for key in row['work']} == row['work'] + and summary['semantic'] == expected['semantic'], 'native_rows_timed_work_mismatch') + receipt.update({key: summary[key] for key in ('requested_threads', 'thread_limit', 'host_threads', + 'max_memory_bytes', 'memory_bytes', 'max_workers', 'parallel_pictures', 'frame_parallel_pictures', + 'row_parallel_pictures', 'admissions')}) + receipt['row_overlap'] = summary['row_parallel_pictures'] > 0 + calls.append(dict(arm=name, repeat=repeat, **receipt)) + conformance.require(conformance.digest(source) == row['input_sha256'], 'native_rows_input_changed') + row.update(status='timed', calls=[call for call in calls if call['repeat'] >= 0], + warmups=[call for call in calls if call['repeat'] < 0]) + return row + + +def native_reference(argv): + parser = argparse.ArgumentParser(description='Measure the real native row pipeline against its one-worker arm') + for name in ('native', 'oracle-bin', 'out'): + parser.add_argument('--' + name, required=True, type=pathlib.Path) + parser.add_argument('--candidate', required=True) + parser.add_argument('--shard-index', type=int, default=0) + parser.add_argument('--shard-count', type=int, default=1) + parser.add_argument('--only', default='') + parser.add_argument('--frames', type=int, choices=range(1, 33), default=32) + parser.add_argument('--repeats', type=int, choices=range(3, 10), default=5) + args = parser.parse_args(argv) + if re.fullmatch('[0-9a-f]{40}', args.candidate) is None or not 0 <= args.shard_index < args.shard_count <= 24: + parser.error('invalid_native_rows_identity_or_shard') + selected = set(args.only.split(',')) if args.only else set(ROW_INPUT_IDS) + if not selected <= set(ROW_INPUT_IDS): + parser.error('native_rows_requires_non_wavefront_inputs') + args.only = {identity for index, identity in enumerate(ROW_INPUT_IDS) + if index % args.shard_count == args.shard_index and identity in selected} + if not args.only: + parser.error('empty_shard') + conformance.require(sys.platform == 'linux', 'linux_actions_required') + oracle = test_oracle.require(args.oracle_bin / 'ffmpeg', args.oracle_bin / 'ffprobe') + args.out.mkdir(parents=True, exist_ok=False) + report = dict(candidate=args.candidate, native_sha256=conformance.digest(args.native), + oracle=oracle_pin.load()['BUILD'], selected_ids=sorted(args.only), frames_cap=args.frames, + repeats=args.repeats, + scope='real native packet boundary; same-head one-worker baseline; automatic/four-worker calls; ' + 'startup, framing, packing and teardown included; shared-host reference figures, not speed bars', + host=dict(cpu=next(line.split(':', 1)[1].strip() for line in pathlib.Path('/proc/cpuinfo').read_text().splitlines() + if line.startswith('model name')), cores=os.cpu_count(), + affinity=sorted(os.sched_getaffinity(0)), kernel=platform.release(), load_average=list(os.getloadavg())), + complete=False, passed=True, files=[]) + pool = round_files.pool() + pool.limits.max_requests = 4000 // args.shard_count + pool.limits.max_bytes = (32 << 30) // args.shard_count + try: + with tempfile.TemporaryDirectory(prefix='hevc-native-rows-') as temp: + for identity, source, original_sha in reference.inputs(args, oracle, pathlib.Path(temp), pool): + try: + row = measure_native(args, oracle, source, identity) + except (ValueError, KeyError, TypeError, subprocess.SubprocessError) as error: + row = dict(id=identity, status='failed', failure=str(error) if isinstance(error, ValueError) + else type(error).__name__) + report['passed'] = False + row['source_sha256'] = original_sha + report['files'].append(row) + conformance.write_json(args.out / 'results.json', report) + print(json.dumps(dict(id=identity, status=row['status'], failure=row.get('failure'))), flush=True) + if not report['passed']: + break + report['complete'] = {row['id'] for row in report['files']} == args.only + conformance.require(conformance.digest(args.native) == report['native_sha256'], 'native_rows_binary_changed') + except BaseException: + report['passed'] = False + raise + finally: + pool.close() + report['r2_requests'] = pool.limits.requests + conformance.write_json(args.out / 'results.json', report) + return 0 if report['complete'] and report['passed'] else 1 + + def signature(binary, source, threads): stdout, _ = reference.command([binary, source, 'check', threads], timeout=600) rows = [json.loads(line) for line in stdout.splitlines()] @@ -147,17 +611,33 @@ def measure(args, oracle, source, identity): if work is not None and work != observed: raise ValueError('thread_work_mismatch') work = observed - receipt.update(thread_limit=data['thread_limit'], max_memory_bytes=data['max_memory_bytes']) + receipt.update(thread_limit=data['thread_limit'], requested_threads=data['requested_threads'], + host_threads=data['host_threads'], max_memory_bytes=data['max_memory_bytes']) if name.startswith('candidate_'): - receipt.update(parallel_pictures=data['parallel_pictures'], max_workers=data['max_workers']) + receipt.update(parallel_pictures=data['parallel_pictures'], + frame_parallel_pictures=data['frame_parallel_pictures'], + max_workers=data['max_workers']) calls.append(dict(arm=name, repeat=repeat, **receipt)) row.update(status='timed', calls=[call for call in calls if call['repeat'] >= 0], warmups=[call for call in calls if call['repeat'] < 0], work=work) return row def main(): + if sys.argv[1:2] == ['native-conformance']: + return native_conformance(sys.argv[2:]) + if sys.argv[1:2] == ['native-reference']: + return native_reference(sys.argv[2:]) + if sys.argv[1:2] == ['diagnostics']: + return diagnostics(sys.argv[2:]) if sys.argv[1:2] == ['census']: return census(sys.argv[2:]) + if sys.argv[1:2] == ['compare-conformance']: + parser = argparse.ArgumentParser(description='Compare complete threaded conformance records') + parser.add_argument('--root', required=True, type=pathlib.Path) + parser.add_argument('--out', required=True, type=pathlib.Path) + args = parser.parse_args(sys.argv[2:]) + args.out.write_text(json.dumps(compare_conformance(args.root), indent=2) + '\n') + return 0 parser = argparse.ArgumentParser(description=__doc__) for name in ('native', 'main', 'oracle-bin', 'out'): parser.add_argument('--' + name, required=True, type=pathlib.Path) @@ -187,7 +667,7 @@ def main(): baseline_sha256=conformance.digest(args.main), oracle=oracle_pin.load()['BUILD'], host=dict(cpu=next(line.split(':', 1)[1].strip() for line in pathlib.Path('/proc/cpuinfo').read_text().splitlines() if line.startswith('model name')), cores=os.cpu_count(), affinity=sorted(os.sched_getaffinity(0)), kernel=platform.release()), - scope='whole-process Annex B decode-to-null; startup, framing and teardown included; count 0 selects automatic workers with the backend memory budget; main is serial at every requested count', + scope='whole-process Annex B decode-to-null; startup, framing and teardown included; count 0 selects automatic workers with the backend memory budget; workers are bounded by available CPUs; baseline uses the synchronous decode entry point', selected_ids=sorted(args.only), frames_cap=args.frames, repeats=args.repeats, complete=False, passed=True, files=[]) pool = round_files.pool() pool.limits.max_requests = 4000 diff --git a/scripts/tests/test_hevc_dependency.py b/scripts/tests/test_hevc_dependency.py new file mode 100644 index 000000000..77f24f502 --- /dev/null +++ b/scripts/tests/test_hevc_dependency.py @@ -0,0 +1,701 @@ +import copy +import itertools +import pathlib +import random +import sys +import unittest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) +from hevc_dependency import analyze + + +def graph(refs, *, active=None, pocs=None, skipped=(), wpp=(), long_term=()): + count = len(refs) + pocs = pocs if pocs is not None else list(range(count)) + pictures = [] + for index, targets in enumerate(refs): + pictures.append({ + 'index': index, 'poc': pocs[index], + 'reconstruction_required': index not in skipped, + 'wpp': index in wpp, 'slice_types': [1] if (targets if active is None else active[index]) else [2], + 'rps': [[target, pocs[target], (index, target) in long_term] for target in targets], + 'active': [[target, pocs[target], (index, target) in long_term] + for target in (targets if active is None else active[index])], + }) + return {'mode': 'header_dependencies', 'header_only': True, 'complete': True, + 'coded_pictures': count, 'reconstruction_required': count - len(skipped), + 'output_pictures': count, 'nals': count + 3, 'pictures': pictures} + + +def brute_force(data, field): + rows = data['pictures'] + vertices = [row['index'] for row in rows if row['reconstruction_required']] + + def reaches(a, b): + return any(ref[0] == a or reaches(a, ref[0]) for ref in rows[b][field]) + + def can_coexist(a, b): + return not any(any(ref[0] == a for ref in rows[later][field]) + for later in vertices if a < later <= b) + + dag_width = admission_width = 0 + for size in range(len(vertices) + 1): + for subset in itertools.combinations(vertices, size): + pairs = list(itertools.combinations(subset, 2)) + if all(not reaches(a, b) for a, b in pairs): + dag_width = size + if all(can_coexist(a, b) for a, b in pairs): + admission_width = size + participants = set() + non_wpp = set() + for a, b in itertools.combinations(vertices, 2): + if can_coexist(a, b): + participants.update((a, b)) + if not rows[a]['wpp'] and not rows[b]['wpp']: + non_wpp.update((a, b)) + return dag_width, admission_width, len(participants), len(non_wpp) + + +def emissions(data, pairs): + data = copy.deepcopy(data) + data['output_events'] = [{'picture_index': index, 'poc': data['pictures'][index]['poc'], + 'before_picture': boundary} for index, boundary in pairs] + data['output_pictures'] = len(pairs) + return data + + +def barrier_interval_oracle(data, field, workers, *, delayed_sealing=False): + rows = data['pictures'] + starts = {} + for index, row in enumerate(rows): + if not row['reconstruction_required']: + continue + fences = [event['before_picture'] for event in data['output_events'] + if event['before_picture'] < index + int(delayed_sealing)] + for time in range(len(rows) + 1): + if any(start > time for start in starts.values()): + continue + if any(starts[ref[0]] + 1 > time for ref in row[field]): + continue + if any(start + 1 > time for earlier, start in starts.items() + if any(earlier < boundary for boundary in fences)): + continue + if sum(start <= time < start + 1 for start in starts.values()) >= workers: + continue + starts[index] = time + break + else: + raise AssertionError('no feasible unit start time') + slots = [[index for index, start in starts.items() if start == time] + for time in range(len(rows))] + overlap = {index for slot in slots if len(slot) > 1 for index in slot} + non_wpp_slots = [[index for index in slot if not rows[index]['wpp']] for slot in slots] + non_wpp = {index for slot in non_wpp_slots if len(slot) > 1 for index in slot} + return {'steps': max(starts.values(), default=-1) + 1, + 'max_workers': max(map(len, slots), default=0), + 'overlap_pictures': len(overlap), 'non_wpp_overlap_pictures': len(non_wpp)} + + +def lag3_event_oracle(data, field, delayed_sealing): + rows = data['pictures'] + completed = set() + running = set() + slots = {} + pending = [] + maximum_pending = time = 0 + + def tick(): + nonlocal time + time += 1 + completed.update(running) + running.clear() + + for call in range(len(rows) + 1): + index = call - int(delayed_sealing) + if 0 <= index < len(rows) and rows[index]['reconstruction_required']: + required = {ref[0] for ref in rows[index][field]} + while not required <= completed or len(running) == 4: + tick() + running.add(index) + slots.setdefault(time, []).append(index) + if delayed_sealing and call < len(rows): + required = {ref[0] for ref in rows[call][field]} + while not required <= completed: + tick() + for event in data['output_events']: + if event['before_picture'] != call: + continue + if len(pending) == 3: + while not pending[0] <= completed: + tick() + pending.pop(0) + assert len(pending) < 3 + pending.append({index for index in range(call) if rows[index]['reconstruction_required']}) + maximum_pending = max(maximum_pending, len(pending)) + while pending and pending[0] <= completed: + pending.pop(0) + if running: + tick() + assert all(fence <= completed for fence in pending) + pending.clear() + overlap = {index for slot in slots.values() if len(slot) > 1 for index in slot} + non_wpp_slots = [[index for index in slot if not rows[index]['wpp']] for slot in slots.values()] + non_wpp = {index for slot in non_wpp_slots if len(slot) > 1 for index in slot} + return {'steps': time, 'max_workers': max(map(len, slots.values()), default=0), + 'overlap_pictures': len(overlap), 'non_wpp_overlap_pictures': len(non_wpp), + 'max_pending_outputs': maximum_pending} + + +class HeaderDependencies(unittest.TestCase): + def test_literal_active_sps_envelope_is_retained_and_validated(self): + original = graph([[]]) + envelope = dict(coded_width=64, coded_height=64, width=60, height=62, + bit_depth=10, ctb_log2=4, dpb_capacity=4, highest_sps_layer=0) + original['pictures'][0]['envelope'] = envelope + self.assertEqual(analyze(original)['graph']['pictures'][0]['envelope'], envelope) + for field, value in [('coded_width', 0), ('height', 65), ('bit_depth', 12), + ('ctb_log2', 3), ('dpb_capacity', 0), ('dpb_capacity', 17), + ('highest_sps_layer', 7), ('width', True)]: + changed = copy.deepcopy(original) + changed['pictures'][0]['envelope'][field] = value + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + analyze(changed) + + def test_strict_chain_has_no_overlap(self): + result = analyze(graph([[], [0], [1], [2]])) + for field in ('rps', 'active'): + metrics = result[field] + self.assertEqual((metrics['vertices'], metrics['edges']), (4, 3)) + self.assertEqual(metrics['immediate_predecessor_edges'], 3) + self.assertEqual(metrics['long_term_edges'], 0) + self.assertEqual(metrics['critical_path_unit_pictures'], 4) + self.assertEqual(metrics['maximum_antichain_width'], 1) + self.assertEqual(metrics['unit_cost_work_span_bound'], 1.0) + self.assertEqual(metrics['coded_admission_width'], 1) + self.assertEqual(metrics['potential_overlap_pictures'], 0) + for schedule in metrics['unit_duration_coded_order'].values(): + self.assertEqual(schedule, {'steps': 4, 'max_workers': 1, + 'overlap_pictures': 0, 'non_wpp_overlap_pictures': 0}) + + def test_shared_anchor_siblings_count_only_two_participants(self): + metrics = analyze(graph([[], [0], [0]]))['active'] + self.assertEqual(metrics['maximum_antichain_width'], 2) + self.assertEqual(metrics['coded_admission_width'], 2) + self.assertEqual(metrics['potential_overlap_pictures'], 2) + self.assertEqual(metrics['non_wpp_potential_overlap_pictures'], 2) + self.assertEqual(metrics['unit_duration_coded_order']['2'], + {'steps': 2, 'max_workers': 2, 'overlap_pictures': 2, + 'non_wpp_overlap_pictures': 2}) + + def test_independent_pictures_fill_available_workers(self): + metrics = analyze(graph([[], [], [], [], []]))['active'] + self.assertEqual(metrics['maximum_antichain_width'], 5) + self.assertEqual(metrics['coded_admission_width'], 5) + self.assertEqual(metrics['unit_cost_work_span_bound'], 5.0) + self.assertEqual(metrics['potential_overlap_pictures'], 5) + for workers, expected in [('1', (5, 1, 0)), ('2', (3, 2, 4)), + ('4', (2, 4, 4)), ('8', (1, 5, 5))]: + schedule = metrics['unit_duration_coded_order'][workers] + self.assertEqual((schedule['steps'], schedule['max_workers'], + schedule['overlap_pictures']), expected) + + def test_hierarchical_b_graph_uses_coded_order_not_poc_order(self): + data = graph([[], [0], [0, 1], [0, 2], [1, 2], [0, 3], [2, 3], [2, 4], [1, 4]], + pocs=[-8, 8, 0, -4, 4, -6, -2, 2, 6]) + metrics = analyze(data)['rps'] + self.assertEqual(metrics['edges'], 15) + self.assertEqual(metrics['critical_path_unit_pictures'], 5) + self.assertEqual(metrics['maximum_antichain_width'], 4) + self.assertEqual(metrics['coded_admission_width'], 4) + self.assertEqual(metrics['potential_overlap_pictures'], 6) + self.assertEqual(metrics['unit_duration_coded_order']['4'], + {'steps': 5, 'max_workers': 4, 'overlap_pictures': 6, + 'non_wpp_overlap_pictures': 6}) + + def test_periodic_independent_cvs_roots_do_not_reset_graph_indices(self): + data = graph([[], [0], [], [2], [], [4]], pocs=[0, 1, 0, 1, 0, 1]) + metrics = analyze(data)['rps'] + self.assertEqual(metrics['maximum_antichain_width'], 3) + self.assertEqual(metrics['critical_path_unit_pictures'], 2) + self.assertEqual(metrics['coded_admission_width'], 3) + self.assertEqual(metrics['potential_overlap_pictures'], 5) + self.assertEqual(metrics['unit_duration_coded_order']['8']['steps'], 4) + self.assertEqual(metrics['unit_duration_coded_order']['8']['max_workers'], 2) + self.assertEqual(metrics['unit_duration_coded_order']['8']['overlap_pictures'], 4) + + def test_fork_unequal_depth_distinguishes_dag_from_coded_admission(self): + metrics = analyze(graph([[], [0], [0], [1, 2], [0], [4], [5]]))['rps'] + self.assertEqual(metrics['maximum_antichain_width'], 3) + self.assertEqual(metrics['coded_admission_width'], 2) + self.assertEqual(metrics['critical_path_unit_pictures'], 4) + self.assertEqual(metrics['potential_overlap_pictures'], 6) + self.assertEqual(metrics['unit_duration_coded_order']['8']['max_workers'], 2) + self.assertEqual(metrics['unit_duration_coded_order']['8']['steps'], 5) + + def test_long_term_identity_and_active_subset_are_separate_graphs(self): + data = graph([[], [0], [0, 1], [0]], active=[[], [0], [0], []], + pocs=[-10, 4, -2, 8], long_term=[(2, 0), (3, 0)]) + result = analyze(data) + self.assertEqual(result['rps']['long_term_edges'], 2) + self.assertEqual(result['active']['long_term_edges'], 1) + self.assertEqual(result['rps']['edges'], 4) + self.assertEqual(result['active']['edges'], 2) + self.assertEqual(result['rps']['critical_path_unit_pictures'], 3) + self.assertEqual(result['active']['critical_path_unit_pictures'], 2) + + def test_union_across_slice_lists_is_consumed_without_dropping_members(self): + data = graph([[], [], [0, 1]], active=[[], [], [0, 1]]) + data['pictures'][2]['slice_types'] = [2, 1, 0] + result = analyze(data) + self.assertEqual(result['active']['edges'], 2) + self.assertEqual(result['graph']['pictures'][2]['active'], [[0, 0, False], [1, 1, False]]) + self.assertIn('independently signalled segments', result['semantics']['active']) + self.assertIn('not entropy-observed', result['semantics']['references']) + + def test_skipped_rows_are_not_vertices_or_scheduled_jobs(self): + data = graph([[], [], [0], []], skipped=[1, 3]) + metrics = analyze(data)['active'] + self.assertEqual(metrics['vertices'], 2) + self.assertEqual(metrics['edges'], 1) + self.assertEqual(metrics['immediate_predecessor_edges'], 0) + self.assertEqual(metrics['maximum_antichain_width'], 1) + self.assertEqual(metrics['unit_duration_coded_order']['8']['steps'], 2) + skipped = analyze(graph([[], []], skipped=[0, 1]))['active'] + self.assertEqual(skipped['vertices'], 0) + self.assertEqual(skipped['critical_path_unit_pictures'], 0) + self.assertEqual(skipped['maximum_antichain_width'], 0) + self.assertEqual(skipped['coded_admission_width'], 0) + self.assertEqual(skipped['unit_cost_work_span_bound'], 0.0) + self.assertEqual(skipped['unit_duration_coded_order']['1']['steps'], 0) + + def test_single_picture_has_no_overlap(self): + metrics = analyze(graph([[]]))['active'] + self.assertEqual(metrics['critical_path_unit_pictures'], 1) + self.assertEqual(metrics['maximum_antichain_width'], 1) + self.assertEqual(metrics['coded_admission_width'], 1) + self.assertEqual(metrics['potential_overlap_pictures'], 0) + + def test_wpp_excludes_only_pairs_with_a_wpp_member(self): + for wpp, expected in [([0, 1, 2], 0), ([0, 1], 0), ([0], 2), ([], 3)]: + metrics = analyze(graph([[], [], []], wpp=wpp))['active'] + self.assertEqual(metrics['potential_overlap_pictures'], 3) + self.assertEqual(metrics['non_wpp_potential_overlap_pictures'], expected) + self.assertEqual(metrics['unit_duration_coded_order']['4']['overlap_pictures'], 3) + self.assertEqual(metrics['unit_duration_coded_order']['4']['non_wpp_overlap_pictures'], expected) + + def test_raw_graph_is_preserved_and_detached_for_replay(self): + data = graph([[], [0], [0]]) + original = copy.deepcopy(data) + result = analyze(data) + self.assertEqual(data, original) + self.assertEqual(result['graph'], original) + self.assertEqual(analyze(result['graph']), result) + result['graph']['pictures'][1]['rps'].clear() + self.assertEqual(data, original) + + def test_widths_match_brute_force_subsets_and_interval_predicate(self): + generator = random.Random(516) + cases = [graph([[], [0], [0], [1, 2], [0], [4], [5]])] + for count in range(1, 6): + pairs = list(itertools.combinations(range(count), 2)) + for mask in range(1 << len(pairs)): + refs = [[] for _ in range(count)] + for bit, (a, b) in enumerate(pairs): + if mask & (1 << bit): + refs[b].append(a) + cases.append(graph(refs)) + for count in (6, 7): + for _ in range(30): + refs = [[a for a in range(b) if generator.random() < 0.35] for b in range(count)] + cases.append(graph(refs, wpp=[a for a in range(count) if a % 2])) + for data in cases: + result = analyze(data) + for field in ('rps', 'active'): + metrics = result[field] + with self.subTest(refs=[row[field] for row in data['pictures']]): + self.assertEqual((metrics['maximum_antichain_width'], metrics['coded_admission_width'], + metrics['potential_overlap_pictures'], + metrics['non_wpp_potential_overlap_pictures']), brute_force(data, field)) + + def test_missing_root_and_row_fields_are_rejected(self): + original = graph([[], [0]]) + for field in original: + changed = copy.deepcopy(original) + del changed[field] + with self.subTest(root=field), self.assertRaises(ValueError): + analyze(changed) + for field in original['pictures'][1]: + changed = copy.deepcopy(original) + del changed['pictures'][1][field] + with self.subTest(row=field), self.assertRaises(ValueError): + analyze(changed) + + def test_root_count_and_completeness_mutations_are_rejected(self): + original = graph([[], [0]]) + mutations = {'mode': [None, 'decode', True], 'header_only': [False, 1], + 'complete': [False, 1], 'coded_pictures': [True, 0, 1, 3, 33, 2.0], + 'reconstruction_required': [True, -1, 1, 3, 2.0], + 'output_pictures': [True, -1, 3, 2.0], 'nals': [True, 1, -1, 4.0], + 'pictures': [None, {}, [], original['pictures'][:1], + original['pictures'] * 2, [None, original['pictures'][1]]]} + for field, values in mutations.items(): + for value in values: + changed = copy.deepcopy(original) + changed[field] = value + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + analyze(changed) + for value in (None, [], 1, True): + with self.subTest(value=value), self.assertRaises(ValueError): + analyze(value) + accepted = graph([[], [0]]) + accepted['output_pictures'] = 0 + accepted['nals'] = 2 + self.assertEqual(analyze(accepted)['rps']['vertices'], 2) + self.assertEqual(analyze(graph([[] for _ in range(32)]))['rps']['vertices'], 32) + + def test_row_and_reference_mutations_are_rejected(self): + original = graph([[], [0]]) + mutations = {'index': [True, 0, 2, 1.0], 'poc': [True, 1.0, '1', None, -(1 << 31) - 1, 1 << 31], + 'reconstruction_required': [0, 1, None, False], 'wpp': [0, 1, None], + 'slice_types': [None, {}, [], [True], [1.0], [-1], [3], [1, 1], [[1]]], + 'rps': [None, {}, [None], [[0]], [[0, 0]], [[0, 0, False, 1]], + [[True, 0, False]], [[0, True, False]], [[0, 0, 0]], + [[0.0, 0, False]], [[0, 0.0, False]], [[0, 9, False]], + [[-1, 0, False]], [[1, 1, False]], [[2, 2, False]], + [[0, 0, False], [0, 0, False]], + [[0, 0, False], [0, 0, True]], [[0, 0, False]] * 17], + 'active': [None, {}, [], [None], [[0, 0, True]], [[1, 1, False]], + [[0, 0, False], [0, 0, False]], [[0, 0, False]] * 17]} + for field, values in mutations.items(): + for value in values: + changed = copy.deepcopy(original) + changed['pictures'][1][field] = value + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + analyze(changed) + changed = graph([[], [], [1]], skipped=[1]) + with self.assertRaises(ValueError): + analyze(changed) + changed = graph([[], [], [1]], active=[[], [], [0]]) + with self.assertRaises(ValueError): + analyze(changed) + changed = graph([[], [0], [0]]) + changed['pictures'][0]['rps'] = [[2, 2, False]] + with self.assertRaises(ValueError): + analyze(changed) + changed = graph([[], [0]]) + changed['pictures'][1]['slice_types'] = [2] + with self.assertRaises(ValueError): + analyze(changed) + + def test_sixteen_unique_references_are_allowed_but_seventeen_are_not(self): + data = graph([[] for _ in range(17)] + [list(range(16))]) + self.assertEqual(analyze(data)['rps']['edges'], 16) + data['pictures'][-1]['rps'].append([16, 16, False]) + with self.assertRaises(ValueError): + analyze(data) + + +class ReceiveBarriers(unittest.TestCase): + def test_absent_events_preserve_legacy_results_and_empty_events_match_old_schedule(self): + for refs in ([[]], [[], [], [], [], []], [[], [0], [0], [1, 2], [0], [4], [5]]): + data = graph(refs) + legacy = analyze(data) + present = analyze(emissions(data, [])) + self.assertNotIn('unit_duration_receive_barrier', legacy['semantics']) + for field in ('rps', 'active'): + metrics = present[field].copy() + self.assertNotIn('unit_duration_receive_barrier', legacy[field]) + self.assertEqual(metrics.pop('unit_duration_receive_barrier'), + legacy[field]['unit_duration_coded_order']) + self.assertEqual(metrics.pop('unit_duration_delayed_sealing_receive_barrier'), + legacy[field]['unit_duration_coded_order']) + lagged = metrics.pop('unit_duration_output_lag3') + for mode in ('picture_ready', 'delayed_sealing'): + expected = {**legacy[field]['unit_duration_coded_order']['4'], 'max_pending_outputs': 0} + self.assertEqual(lagged[mode]['full_picture_references'], expected) + self.assertEqual(lagged[mode]['row_references_worst_case'], expected) + self.assertEqual(metrics, legacy[field]) + + def test_independent_five_with_every_output_boundary_peak_at_two(self): + data = emissions(graph([[], [], [], [], []]), [(i, i + 1) for i in range(5)]) + result = analyze(data) + for field in ('rps', 'active'): + self.assertEqual(result[field]['unit_duration_receive_barrier']['4'], + {'steps': 3, 'max_workers': 2, 'overlap_pictures': 4, + 'non_wpp_overlap_pictures': 4}) + self.assertEqual(result[field]['unit_duration_coded_order']['4']['steps'], 2) + self.assertEqual(result[field]['unit_duration_coded_order']['4']['max_workers'], 4) + self.assertEqual(result['graph']['output_events'], data['output_events']) + self.assertEqual(analyze(result['graph']), result) + self.assertIn('one-picture-late job sealing', result['semantics']['unit_duration_receive_barrier']) + self.assertIn('not measured throughput', result['semantics']['unit_duration_receive_barrier']) + + def test_anchor_with_four_children_has_three_steps_instead_of_two(self): + data = emissions(graph([[], [0], [0], [0], [0]]), [(i, i + 1) for i in range(5)]) + result = analyze(data)['active'] + self.assertEqual(result['unit_duration_receive_barrier']['4'], + {'steps': 3, 'max_workers': 2, 'overlap_pictures': 4, + 'non_wpp_overlap_pictures': 4}) + self.assertEqual(result['unit_duration_coded_order']['4'], + {'steps': 2, 'max_workers': 4, 'overlap_pictures': 4, + 'non_wpp_overlap_pictures': 4}) + + def test_strict_chains_remain_serial_with_every_boundary(self): + data = emissions(graph([[], [0], [1], [2]]), [(i, i + 1) for i in range(4)]) + for field in ('rps', 'active'): + metrics = analyze(data)[field] + self.assertEqual(metrics['unit_duration_receive_barrier'], metrics['unit_duration_coded_order']) + self.assertEqual(metrics['unit_duration_receive_barrier']['8'], + {'steps': 4, 'max_workers': 1, 'overlap_pictures': 0, + 'non_wpp_overlap_pictures': 0}) + + def test_fence_waits_for_unrelated_earlier_jobs_even_if_output_picture_finished(self): + data = emissions(graph([[], [0], [], [], []]), [(0, 3)]) + metrics = analyze(data)['active'] + self.assertEqual(metrics['unit_duration_receive_barrier']['4'], + {'steps': 3, 'max_workers': 3, 'overlap_pictures': 3, + 'non_wpp_overlap_pictures': 3}) + self.assertEqual(metrics['unit_duration_coded_order']['4']['steps'], 2) + self.assertEqual(metrics['unit_duration_coded_order']['4']['overlap_pictures'], 4) + + def test_skipped_sends_still_apply_barriers_and_eof_finishes_jobs(self): + data = emissions(graph([[], [], [], []], skipped=[1]), [(0, 1), (2, 4), (3, 4)]) + metrics = analyze(data)['active'] + self.assertEqual(metrics['unit_duration_receive_barrier']['4'], + {'steps': 2, 'max_workers': 2, 'overlap_pictures': 2, + 'non_wpp_overlap_pictures': 2}) + self.assertEqual(metrics['unit_duration_coded_order']['4']['max_workers'], 3) + empty = emissions(graph([[], []], skipped=[0, 1]), []) + self.assertEqual(analyze(empty)['active']['unit_duration_receive_barrier']['4'], + {'steps': 0, 'max_workers': 0, 'overlap_pictures': 0, + 'non_wpp_overlap_pictures': 0}) + eof_only = emissions(graph([[], [], []]), [(2, 3), (0, 3), (1, 3)]) + metrics = analyze(eof_only)['active'] + self.assertEqual(metrics['unit_duration_receive_barrier'], metrics['unit_duration_coded_order']) + + def test_output_bursts_at_one_boundary_are_one_prefix_fence(self): + data = emissions(graph([[], [], [], []], pocs=[2, -1, 0, 2]), + [(1, 2), (0, 2), (3, 4), (2, 4)]) + self.assertEqual(analyze(data)['active']['unit_duration_receive_barrier']['4'], + {'steps': 2, 'max_workers': 3, 'overlap_pictures': 3, + 'non_wpp_overlap_pictures': 3}) + + def test_non_wpp_counts_use_real_interval_pairs_not_queued_or_completed_jobs(self): + for wpp, expected in [([], 4), ([0], 2), ([0, 2], 0), (list(range(5)), 0)]: + data = emissions(graph([[], [], [], [], []], wpp=wpp), [(i, i + 1) for i in range(5)]) + metrics = analyze(data)['active']['unit_duration_receive_barrier']['4'] + self.assertEqual(metrics['overlap_pictures'], 4) + self.assertEqual(metrics['non_wpp_overlap_pictures'], expected) + + def test_event_mutations_reject_missing_identity_order_count_and_boolean_numbers(self): + original = emissions(graph([[], [], []]), [(0, 1), (1, 2), (2, 3)]) + for value in (None, {}, 1, [], original['output_events'][:-1], original['output_events'] * 2): + changed = copy.deepcopy(original) + changed['output_events'] = value + with self.subTest(events=value), self.assertRaises(ValueError): + analyze(changed) + for field, values in {'picture_index': [True, False, 0.0, '0', -1, 3, 1], + 'poc': [True, False, 0.0, '0', -1, 3], + 'before_picture': [True, False, 1.0, '1', 0, -1, 4]}.items(): + for value in values: + changed = copy.deepcopy(original) + changed['output_events'][0][field] = value + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + analyze(changed) + for field in original['output_events'][0]: + changed = copy.deepcopy(original) + del changed['output_events'][0][field] + with self.subTest(missing=field), self.assertRaises(ValueError): + analyze(changed) + for value in (None, [], [0, 0, 1]): + changed = copy.deepcopy(original) + changed['output_events'][0] = value + with self.subTest(event=value), self.assertRaises(ValueError): + analyze(changed) + duplicate = copy.deepcopy(original) + duplicate['output_events'][1] = duplicate['output_events'][0].copy() + with self.assertRaises(ValueError): + analyze(duplicate) + out_of_order = emissions(graph([[], [], []]), [(0, 3), (1, 2)]) + with self.assertRaises(ValueError): + analyze(out_of_order) + skipped = emissions(graph([[], [], []], skipped=[1]), [(1, 2)]) + with self.assertRaises(ValueError): + analyze(skipped) + + def test_small_dags_and_barrier_subsets_match_independent_interval_oracle(self): + pairs = list(itertools.combinations(range(4), 2)) + for edge_mask in range(1 << len(pairs)): + refs = [[] for _ in range(4)] + for bit, (a, b) in enumerate(pairs): + if edge_mask & (1 << bit): + refs[b].append(a) + active = [targets[::2] for targets in refs] + for boundary_mask in range(1 << 4): + data = emissions(graph(refs, active=active, wpp=[1]), + [(b - 1, b) for b in range(1, 5) if boundary_mask & (1 << (b - 1))]) + result = analyze(data) + for field in ('rps', 'active'): + for workers in (1, 2, 4, 8): + with self.subTest(edges=edge_mask, barriers=boundary_mask, + field=field, workers=workers): + self.assertEqual(result[field]['unit_duration_receive_barrier'][str(workers)], + barrier_interval_oracle(data, field, workers)) + self.assertEqual( + result[field]['unit_duration_delayed_sealing_receive_barrier'][str(workers)], + barrier_interval_oracle(data, field, workers, delayed_sealing=True)) + + +class DelayedSealing(unittest.TestCase): + def test_absent_events_do_not_invent_delayed_sealing_barriers(self): + result = analyze(graph([[], [], []])) + field = 'unit_duration_delayed_sealing_receive_barrier' + self.assertNotIn(field, result['semantics']) + for kind in ('rps', 'active'): + self.assertNotIn(field, result[kind]) + + def test_no_events_controls_preserve_reference_and_worker_limits(self): + for refs in ([[]], [[], [], [], [], []], [[], [0], [0], [0], [0]], + [[], [0], [1], [2]], [[], [0], [0], [1, 2], [0], [4], [5]]): + result = analyze(emissions(graph(refs, wpp=[0]), [])) + for field in ('rps', 'active'): + self.assertEqual(result[field]['unit_duration_delayed_sealing_receive_barrier'], + result[field]['unit_duration_coded_order']) + + def test_anchor_children_are_serial_when_each_receive_fences_the_sealed_prefix(self): + data = emissions(graph([[], [0], [0], [0], [0]]), [(i, i + 1) for i in range(5)]) + result = analyze(data) + for field in ('rps', 'active'): + self.assertEqual(result[field]['unit_duration_receive_barrier']['4'], + {'steps': 3, 'max_workers': 2, 'overlap_pictures': 4, + 'non_wpp_overlap_pictures': 4}) + for schedule in result[field]['unit_duration_delayed_sealing_receive_barrier'].values(): + self.assertEqual(schedule, {'steps': 5, 'max_workers': 1, 'overlap_pictures': 0, + 'non_wpp_overlap_pictures': 0}) + label = result['semantics']['unit_duration_delayed_sealing_receive_barrier'] + self.assertIn('before accepting its input', label) + self.assertIn('actual Scheduler.required_references used-RPS gate', label) + self.assertIn('not measured throughput', label) + + def test_chain_stays_serial_with_delayed_sealing_and_barriers(self): + data = emissions(graph([[], [0], [1], [2]]), [(i, i + 1) for i in range(4)]) + for field in ('rps', 'active'): + result = analyze(data)[field] + self.assertEqual(result['unit_duration_delayed_sealing_receive_barrier'], + result['unit_duration_coded_order']) + + def test_skipped_calls_seal_previous_job_and_still_apply_fences(self): + data = emissions(graph([[], [], [], []], skipped=[1]), [(0, 1), (2, 4), (3, 4)]) + for field in ('rps', 'active'): + schedules = analyze(data)[field]['unit_duration_delayed_sealing_receive_barrier'] + self.assertEqual(schedules['4'], {'steps': 2, 'max_workers': 2, 'overlap_pictures': 2, + 'non_wpp_overlap_pictures': 2}) + trailing_skips = emissions(graph([[], [], []], skipped=[1, 2]), [(0, 3)]) + self.assertEqual(analyze(trailing_skips)['rps']['unit_duration_delayed_sealing_receive_barrier']['8'], + {'steps': 1, 'max_workers': 1, 'overlap_pictures': 0, + 'non_wpp_overlap_pictures': 0}) + all_skipped = emissions(graph([[], []], skipped=[0, 1]), []) + self.assertEqual(analyze(all_skipped)['rps']['unit_duration_delayed_sealing_receive_barrier']['8'], + {'steps': 0, 'max_workers': 0, 'overlap_pictures': 0, + 'non_wpp_overlap_pictures': 0}) + + def test_eof_submits_final_job_but_output_burst_fences_all_submitted_jobs(self): + data = emissions(graph([[], [], [], []]), [(1, 2), (0, 2), (3, 4), (2, 4)]) + metrics = analyze(data)['active'] + self.assertEqual(metrics['unit_duration_delayed_sealing_receive_barrier']['4'], + {'steps': 2, 'max_workers': 2, 'overlap_pictures': 4, + 'non_wpp_overlap_pictures': 4}) + eof_only = emissions(graph([[], [], []], wpp=[0, 1, 2]), [(2, 3), (0, 3), (1, 3)]) + self.assertEqual(analyze(eof_only)['rps']['unit_duration_delayed_sealing_receive_barrier']['4'], + {'steps': 1, 'max_workers': 3, 'overlap_pictures': 3, + 'non_wpp_overlap_pictures': 0}) + + +class OutputLagThree(unittest.TestCase): + def test_legacy_records_do_not_invent_output_lag_results(self): + result = analyze(graph([[], [], []])) + self.assertNotIn('unit_duration_output_lag3', result['semantics']) + for field in ('rps', 'active'): + self.assertNotIn('unit_duration_output_lag3', result[field]) + + def check_models(self, data): + result = analyze(data) + for field in ('rps', 'active'): + lagged = result[field]['unit_duration_output_lag3'] + self.assertEqual(lagged['workers'], 4) + self.assertEqual(lagged['output_lag_limit'], 3) + for mode in ('picture_ready', 'delayed_sealing'): + metrics = lagged[mode]['full_picture_references'] + self.assertEqual(metrics, lag3_event_oracle(data, field, mode == 'delayed_sealing')) + self.assertEqual(lagged[mode]['row_references_worst_case'], metrics) + self.assertLessEqual(metrics['max_pending_outputs'], 3) + return result + + def test_independent_and_anchor_controls_fill_four_workers_without_exceeding_lag(self): + for refs, steps, overlap in [([[] for _ in range(8)], 2, 8), + ([[], [0], [0], [0], [0]], 2, 4)]: + result = self.check_models(emissions(graph(refs), [(i, i + 1) for i in range(len(refs))])) + for mode in ('picture_ready', 'delayed_sealing'): + metrics = result['rps']['unit_duration_output_lag3'][mode]['full_picture_references'] + self.assertEqual(metrics, {'steps': steps, 'max_workers': 4, 'overlap_pictures': overlap, + 'non_wpp_overlap_pictures': overlap, 'max_pending_outputs': 3}) + + def test_strict_chain_stays_serial(self): + result = self.check_models(emissions(graph([[], [0], [1], [2], [3]]), + [(i, i + 1) for i in range(5)])) + for mode in ('picture_ready', 'delayed_sealing'): + metrics = result['rps']['unit_duration_output_lag3'][mode]['full_picture_references'] + self.assertEqual((metrics['steps'], metrics['max_workers'], metrics['overlap_pictures']), (5, 1, 0)) + + def test_four_record_burst_consumes_oldest_full_prefix_before_fourth_append(self): + data = emissions(graph([[], [0], [1], [], [], []]), [(0, 4), (1, 4), (2, 4), (3, 4)]) + result = self.check_models(data)['rps'] + metrics = result['unit_duration_output_lag3']['picture_ready']['full_picture_references'] + self.assertEqual(metrics, {'steps': 4, 'max_workers': 3, 'overlap_pictures': 3, + 'non_wpp_overlap_pictures': 3, 'max_pending_outputs': 3}) + self.assertEqual(result['unit_duration_coded_order']['4']['steps'], 3) + self.assertEqual(result['unit_duration_coded_order']['4']['max_workers'], 4) + + def test_skipped_sends_eof_bursts_and_all_skipped_records(self): + cases = [emissions(graph([[], [], [], [], []], skipped=[1]), + [(0, 1), (2, 5), (4, 5), (3, 5)]), + emissions(graph([[], [], []], skipped=[1, 2]), [(0, 3)]), + emissions(graph([[], []], skipped=[0, 1]), []), + emissions(graph([[] for _ in range(8)], wpp=list(range(8))), + [(i, 8) for i in reversed(range(8))])] + for data in cases: + result = self.check_models(data) + for mode in ('picture_ready', 'delayed_sealing'): + metrics = result['rps']['unit_duration_output_lag3'][mode]['full_picture_references'] + self.assertEqual(metrics['steps'], result['rps']['unit_duration_coded_order']['4']['steps']) + for mode in ('picture_ready', 'delayed_sealing'): + metrics = result['rps']['unit_duration_output_lag3'][mode]['full_picture_references'] + self.assertEqual(metrics['max_pending_outputs'], 3) + self.assertEqual(metrics['non_wpp_overlap_pictures'], 0) + + def test_consumption_never_reclaims_a_later_reference_or_invents_row_benefits(self): + data = emissions(graph([[], [0], [], [], [], [0]]), [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6)]) + result = self.check_models(data) + label = result['semantics']['unit_duration_output_lag3'] + self.assertIn('not implementability under the current Decoder send/receive API', label) + self.assertIn('no CTU-row or MV footprints', label) + self.assertIn('publication is not reclamation', label) + self.assertIn('logical pending emitted frames, not an allocated-byte/capacity bound', label) + + def test_all_four_vertex_dags_and_event_placements_match_tick_oracle(self): + pairs = list(itertools.combinations(range(4), 2)) + placements = list(itertools.product(*[range(index + 1, 5) for index in range(4)])) + for edge_mask in range(1 << len(pairs)): + refs = [[] for _ in range(4)] + for bit, (a, b) in enumerate(pairs): + if edge_mask & (1 << bit): + refs[b].append(a) + for placement in placements: + data = emissions(graph(refs, active=[targets[::2] for targets in refs], wpp=[0]), + sorted(enumerate(placement), key=lambda pair: pair[1])) + with self.subTest(edges=edge_mask, placement=placement): + self.check_models(data) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_hevc_inverse_reference.py b/scripts/tests/test_hevc_inverse_reference.py index 1da17a85f..90e877e26 100644 --- a/scripts/tests/test_hevc_inverse_reference.py +++ b/scripts/tests/test_hevc_inverse_reference.py @@ -72,8 +72,9 @@ def test_workflow_admits_both_exact_arms_before_builds_or_credentials(self): def test_existing_dispatch_preserves_reference_and_threading_unless_inverse_is_selected(self): text = (SCRIPTS.parent / '.github/workflows/hevc-reference.yml').read_text() self.assertIn('if: inputs.inverse', text) - self.assertIn("if: ${{ !inputs.inverse && inputs.baseline == '' && !inputs.sanitizer_only }}", text) - self.assertIn("if: ${{ !inputs.inverse && (inputs.baseline != '' || inputs.sanitizer_only) }}", text) + self.assertIn("if: ${{ !inputs.inverse && inputs.baseline == '' && !inputs.sanitizer_only && !inputs.thread_diagnostics_only && !inputs.thread_dependencies_only && !inputs.thread_rows_only }}", text) + self.assertIn("if: ${{ !inputs.inverse && inputs.baseline != 'native' && (inputs.baseline != '' || inputs.sanitizer_only || inputs.thread_diagnostics_only || inputs.thread_dependencies_only || inputs.thread_rows_only) }}", text) + self.assertIn("if: ${{ !inputs.inverse && inputs.baseline == 'native' }}", text) self.assertIn('uses: ./.github/workflows/hevc-inverse-reference.yml', text) for name in ('baseline', 'candidate', 'only'): self.assertIn(name + ': ${{ inputs.' + name + ' }}', text) diff --git a/scripts/tests/test_hevc_native_rows.py b/scripts/tests/test_hevc_native_rows.py new file mode 100644 index 000000000..0568afcd3 --- /dev/null +++ b/scripts/tests/test_hevc_native_rows.py @@ -0,0 +1,215 @@ +import copy +import json +import pathlib +import sys +import tempfile +import types +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) +import hevc_threading as threading + + +class NativeRows(unittest.TestCase): + def receipt(self, threads=4, check=True): + color = dict.fromkeys(threading.hevc_boundary.COLOR_FIELDS) + picture = dict(type='picture', timestamp=0, width=8, height=8, bit_depth=8, + packing='yuv420p', color=color, output_bytes=96, output_md5='a' * 32) + semantic = dict(headers=dict(nals=3, pictures=2, output_pictures=1, + warnings=dict(profile_bit_depth=False, invalid_sample_aspect_ratio=False, + temporal_nesting_nal_type=False)), decoded_pictures=2, + skipped_pictures=0, skipped_picture_hashes=0, skipped_reserved_hashes=0, + stats=dict(bins=7, ctus=2, memory_bytes=0), inter=dict(prediction_units=0)) + admission = dict(sps_id=0, sps_sha256=[0] * 32, ceiling=threads or 8, workers=2, lag=1, + frame_bytes=96, reserved_bytes=1 << 20, limit_bytes=1 << 30) + summary = dict(type='summary', complete=True, boundary=True, backend='native', semantic=semantic, + pictures=1, output_pictures=1, decoded_pictures=2, bins=7, requested_threads=threads, + host_threads=8, thread_limit=threads or 8, max_workers=1 if threads == 1 else 2, + parallel_pictures=0 if threads == 1 else 2, frame_parallel_pictures=0 if threads == 1 else 2, + row_parallel_pictures=0 if threads == 1 else 2, + max_memory_bytes=1 << 30, memory_bytes=1 << 20, input_bytes=128, sent_packets=2, + inherited_color=color, output_bytes=96 if check else None, + output_md5='a' * 32 if check else None, admissions=[] if threads == 1 else [admission]) + return [picture, summary] if check else [summary] + + def encoded(self, records): + return '\n'.join(json.dumps(record) for record in records).encode() + + def test_native_receipt_cannot_credit_the_old_complete_reference_protocol(self): + records = self.receipt() + frames, summary = threading.native_receipt(self.encoded(records), 4, check=True) + self.assertEqual(len(frames), 1) + self.assertEqual(summary['frame_parallel_pictures'], 2) + for key in ('boundary', 'backend', 'semantic', 'admissions'): + changed = copy.deepcopy(records) + changed[-1].pop(key) + with self.subTest(key=key), self.assertRaises(ValueError): + threading.native_receipt(self.encoded(changed), 4, check=True) + + def test_counter_types_admission_bytes_and_timed_hashing_fail_closed(self): + mutations = [lambda row: row.update(requested_threads=8), + lambda row: row.update(max_workers=5), + lambda row: row.update(bins=True), + lambda row: row['semantic']['stats'].update(bins=True), + lambda row: row['admissions'][0].update(lag=2), + lambda row: row['admissions'][0].update(reserved_bytes=1 << 30), + lambda row: row['admissions'][0].update(sps_sha256=[0]), + lambda row: row.update(pictures=2)] + mutations += [lambda row: row.update(row_parallel_pictures=3), + lambda row: row['admissions'][0].update(lag=0)] + for mutate in mutations: + records = self.receipt() + mutate(records[-1]) + with self.subTest(mutate=mutate), self.assertRaises(ValueError): + threading.native_receipt(self.encoded(records), 4, check=True) + records = self.receipt(check=False) + threading.native_receipt(self.encoded(records), 4, check=False) + records[-1]['output_md5'] = 'a' * 32 + with self.assertRaisesRegex(ValueError, 'disable_hashing'): + threading.native_receipt(self.encoded(records), 4, check=False) + records = self.receipt() + records[0].update(bit_depth=10, packing='yuv420p10le', output_bytes=192) + records[-1]['output_bytes'] = 192 + threading.native_receipt(self.encoded(records), 4, check=True) + records[0]['timestamp'] = True + with self.assertRaisesRegex(ValueError, 'frame_receipt'): + threading.native_receipt(self.encoded(records), 4, check=True) + + def conformance(self, directory, mutation=None, split_coverage=False): + source = directory / 'input' + source.write_bytes(b'fixture') + expected = self.receipt() + expected[0].update(picture_index=0, color_declarations=expected[0]['color']) + raw = dict(complete=True, passed=True, scope='complete_c1', + identities=dict.fromkeys(('registration', 'census', 'parser', 'acquisition'), 'b' * 64), + denominator=[dict(test='fixture', sha256='b' * 64)], + results=[dict(complete=True, test='fixture', sha256='b' * 64, + pictures=expected[:-1], summary={**expected[-1], **expected[-1]['semantic']})], + budgets=dict(max_memory_bytes=1 << 30)) + if split_coverage: + raw['denominator'].append(dict(test='legacy', sha256='b' * 64)) + raw['results'].append({**copy.deepcopy(raw['results'][0]), 'test': 'legacy'}) + scalar = directory / 'scalar.json' + scalar.write_text(json.dumps(raw)) + census = directory / 'census.json' + census.write_text('{}') + calls = [] + def child(command, out, label, **kwargs): + calls.append(command) + threads = int(command[3]) + records = self.receipt(threads) + records[-1]['input_bytes'] = source.stat().st_size + if split_coverage: + records[-1]['row_parallel_pictures'] = 0 + if out.name == '1': + records[-1]['admissions'] = [] + if mutation: + mutation(records, threads) + out.mkdir(parents=True, exist_ok=True) + output = out / (label + '.stdout') + output.write_bytes(self.encoded(records)) + return 0, output, out / (label + '.stderr') + args = ['--registration', str(directory / 'registration'), '--census', str(census), + '--data', str(directory), '--out', str(directory / 'out'), '--parser', '/headers', + '--native', '/native', '--scalar-result', str(scalar)] + selected = [dict(case=dict(test='fixture'), member=dict(sha256='b' * 64), source=source)] + if split_coverage: + selected.append(dict(case=dict(test='legacy'), member=dict(sha256='b' * 64), source=source)) + with patch.object(threading.sys, 'platform', 'linux'), \ + patch.object(threading.hevc_decode, 'streams', return_value=selected), \ + patch.object(threading.conformance, 'digest', return_value='b' * 64), \ + patch.object(threading.conformance, 'child', side_effect=child): + threading.native_conformance(args) + return calls + + def test_complete_native_conformance_executes_all_counts_and_the_frame_owner(self): + with tempfile.TemporaryDirectory() as temp: + calls = self.conformance(pathlib.Path(temp)) + self.assertEqual([call[3] for call in calls], ['1', '2', '4', '8', '0']) + self.assertTrue(all(call[2] == 'check' for call in calls)) + result = json.loads((pathlib.Path(temp) / 'out/result.json').read_text()) + self.assertTrue(result['complete']) + self.assertEqual(result['coverage']['4']['admitted_streams'], 1) + self.assertEqual(result['coverage']['0']['admitted_streams'], 1) + mutations = [lambda records, count: records[-1]['semantic']['stats'].update(ctus=3) if count == 4 else None, + lambda records, count: records[0].update(timestamp=1) if count == 4 else None, + lambda records, count: records[-1].update(admissions=[]), + lambda records, count: records[-1].update(row_parallel_pictures=0)] + for mutate in mutations: + with self.subTest(mutate=mutate), tempfile.TemporaryDirectory() as temp, self.assertRaises(ValueError): + self.conformance(pathlib.Path(temp), mutate) + + def test_native_timing_validates_before_calls_and_rechecks_every_timed_work_record(self): + with tempfile.TemporaryDirectory() as temp: + source = pathlib.Path(temp) / 'input' + source.write_bytes(b'fixture') + commands = [] + changed_work = [False] + no_row_overlap = [False] + def child(command, **kwargs): + commands.append(command) + if command[0] == '/native': + records = self.receipt(int(command[3]), check=command[2] == 'check') + records[-1]['input_bytes'] = source.stat().st_size + if no_row_overlap[0]: + records[-1]['row_parallel_pictures'] = 0 + if changed_work[0] and command[2] == 'decode': + records[-1]['semantic']['stats']['ctus'] += 1 + return self.encoded(records), dict(wall=1.0, user=0.9, system=0.1) + return b'frame=1\n', dict(wall=1.0, user=0.9, system=0.1) + args = types.SimpleNamespace(native='/native', repeats=3) + oracle = types.SimpleNamespace(ffmpeg='/qualified/ffmpeg') + verified = dict(frames=1, depth=8, output_md5='a' * 32) + with patch.object(threading.reference, 'verify', return_value=verified), \ + patch.object(threading.reference, 'command', side_effect=child): + row = threading.measure_native(args, oracle, source, 'd00') + self.assertEqual(row['status'], 'timed') + self.assertEqual([call[3] for call in commands[:5]], [1, 2, 4, 8, 0]) + self.assertEqual(len(row['warmups']), 5) + self.assertEqual(len(row['calls']), 15) + self.assertTrue(all(call['row_overlap'] == (call['arm'] != 'candidate_1') + for call in row['calls'] if call['arm'].startswith('candidate_'))) + self.assertTrue(all(call[0] in ('/native', oracle.ffmpeg) for call in commands)) + changed_work[0] = True + with patch.object(threading.reference, 'verify', return_value=verified), \ + patch.object(threading.reference, 'command', side_effect=child), \ + self.assertRaisesRegex(ValueError, 'timed_work_mismatch'): + threading.measure_native(args, oracle, source, 'd00') + changed_work[0] = False + no_row_overlap[0] = True + with patch.object(threading.reference, 'verify', return_value=verified), \ + patch.object(threading.reference, 'command', side_effect=child): + control = threading.measure_native(args, oracle, source, 'd00') + self.assertTrue(all(call['row_overlap'] is False for call in control['calls'] + if call['arm'].startswith('candidate_'))) + + def test_separate_admission_and_legacy_overlap_cannot_establish_row_coverage(self): + with tempfile.TemporaryDirectory() as temp, self.assertRaisesRegex(ValueError, 'real_coverage_missing'): + self.conformance(pathlib.Path(temp), split_coverage=True) + + def test_workflows_use_the_real_bridge_and_sanitizer_feature(self): + root = pathlib.Path(__file__).resolve().parents[2] + native = (root / '.github/workflows/hevc-native-rows.yml').read_text() + dispatch = (root / '.github/workflows/hevc-reference.yml').read_text() + reusable = (root / '.github/workflows/hevc-threading.yml').read_text() + self.assertIn("inputs.baseline == 'native'", dispatch) + self.assertIn('uses: ./.github/workflows/hevc-native-rows.yml', dispatch) + self.assertIn('workflow_call:', native) + self.assertIn('hevc_threading.py native-reference', native) + self.assertIn('--features native-hevc,test-support --example reference', native) + self.assertLess(native.index('git branch -r --contains'), native.index('cargo test')) + self.assertLess(native.index('git branch -r --contains'), native.index('secrets.')) + self.assertIn('hevc_threading.py native-conformance', reusable) + self.assertIn("counts='1 2 4 8'", reusable) + self.assertIn('if [ -z "$BASELINE" ]; then counts=1; fi', reusable) + self.assertIn('if [ -n "$BASELINE" ]; then\n python3 -I scripts/hevc_threading.py compare-conformance', reusable) + self.assertIn('--features native-hevc-rows-private --lib "$group"', reusable) + self.assertIn('dependent_raster_job_reads_a_building_predecessor_before_its_first_publication ... ok', reusable) + self.assertIn('capacity_probe_mismatch_preserves_legacy_frames_and_summary ... ok', reusable) + self.assertIn('rsa_oaep_md:sha256', native) + self.assertNotIn('path: ${{ runner.temp }}/private', native) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_hevc_row_dependency.py b/scripts/tests/test_hevc_row_dependency.py new file mode 100644 index 000000000..ca2fbcb12 --- /dev/null +++ b/scripts/tests/test_hevc_row_dependency.py @@ -0,0 +1,402 @@ +import copy +import itertools +import pathlib +import random +import sys +import unittest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) +from hevc_row_dependency import analyze +from hevc_dependency import analyze as analyze_header + + +def trace(counts, reads=None, *, rps=None, active=None, tiles=(), outputs=None): + reads = reads or {} + rps = rps if rps is not None else [sorted({target for row in range(count or 0) + for target in reads.get((index, row), {})}) + for index, count in enumerate(counts)] + active = active if active is not None else rps + headers = [] + pictures = [] + for index, count in enumerate(counts): + headers.append({'index': index, 'poc': index * 2 - 4, 'wpp': False, + 'reconstruction_required': count is not None, + 'slice_types': [1] if active[index] else [2], + 'rps': [[target, target * 2 - 4, False] for target in rps[index]], + 'active': [[target, target * 2 - 4, False] for target in active[index]]}) + if count is None: + continue + rows = [] + for row in range(count): + refs = [[target, target * 2 - 4, sample, motion] + for target, (sample, motion) in reads.get((index, row), {}).items()] + rows.append({'row': row, 'references': refs, + 'sample_calls': sum(ref[2] is not None for ref in refs), + 'motion_calls': sum(ref[3] is not None for ref in refs)}) + pictures.append({'index': index, 'poc': index * 2 - 4, 'width': 32, 'height': count * 16, + 'ctb_log2': 4, 'tiles': index in tiles, 'wpp': False, 'rows': rows}) + outputs = outputs if outputs is not None else [(index, len(counts)) for index, count in enumerate(counts) + if count is not None] + header = {'mode': 'header_dependencies', 'header_only': True, 'complete': True, + 'coded_pictures': len(counts), 'reconstruction_required': len(pictures), + 'output_pictures': len(outputs), 'nals': len(counts) + 3, 'pictures': headers, + 'output_events': [{'picture_index': index, 'poc': index * 2 - 4, 'before_picture': before} + for index, before in outputs]} + return {'mode': 'row_dependencies', 'complete': True, 'header_only': False, + 'requested_threads': 1, 'thread_limit': 1, 'max_workers': 1, + 'decoded_pictures': len(pictures), 'output_pictures': len(outputs), + 'header_dependencies': header, 'pictures': pictures} + + +def tick_oracle(data, lag, delayed): + pictures = {picture['index']: picture for picture in data['pictures']} + header = data['header_dependencies'] + rows_done = {index: 0 for index in pictures} + tile_work = {index: 0 for index in pictures} + finished = set() + active = set() + pending = [] + participants = set() + time = peak = inflight = pending_peak = 0 + + def ready(index): + picture = pictures[index] + if picture['tiles'] or lag is None: + return all(ref[0] in finished for ref in header['pictures'][index]['rps']) + row = picture['rows'][rows_done[index]] + for target, _, sample, motion in row['references']: + if pictures[target]['tiles']: + if target not in finished: + return False + continue + if sample is not None: + needed = min(sample + lag, len(pictures[target]['rows']) - 1) + if rows_done[target] <= needed: + return False + if motion is not None and rows_done[target] <= motion: + return False + return True + + def tick(): + nonlocal time, peak + jobs = [index for index in sorted(active) if ready(index)] + assert jobs + assert len(jobs) <= 4 + peak = max(peak, len(jobs)) + if len(jobs) > 1: + participants.update(jobs) + for index in jobs: + picture = pictures[index] + if picture['tiles']: + tile_work[index] += 1 + if tile_work[index] == len(picture['rows']): + rows_done[index] = len(picture['rows']) + else: + rows_done[index] += 1 + if rows_done[index] == len(picture['rows']): + finished.add(index) + active.remove(index) + time += 1 + + def fence_ready(boundary): + return all(index in finished for index in pictures if index < boundary) + + for call in range(header['coded_pictures'] + 1): + index = call - int(delayed) + if index in pictures: + required = {ref[0] for ref in header['pictures'][index]['rps']} if lag is None else set() + while len(active) == 4 or not required <= finished: + tick() + active.add(index) + inflight = max(inflight, len(active)) + if delayed and lag is None and call < header['coded_pictures']: + required = {ref[0] for ref in header['pictures'][call]['rps']} + while not required <= finished: + tick() + for event in header['output_events']: + if event['before_picture'] != call: + continue + if len(pending) == 3: + while not fence_ready(pending[0]): + tick() + pending.pop(0) + assert len(pending) < 3 + pending.append(call) + pending_peak = max(pending_peak, len(pending)) + while pending and fence_ready(pending[0]): + pending.pop(0) + while active: + tick() + assert all(fence_ready(boundary) for boundary in pending) + return {'steps': time, 'max_workers': peak, 'overlap_pictures': len(participants), + 'max_in_flight_pictures': inflight, 'max_pending_outputs': pending_peak} + + +class RowDependencies(unittest.TestCase): + def check_oracle(self, data): + result = analyze(data) + for mode in ('picture_ready', 'delayed_sealing'): + for name, lag in (('filter_lag1', 1), ('filter_lag2', 2), ('whole_reference_control', None)): + model = result['models'][mode][name] + expected = tick_oracle(data, lag, mode == 'delayed_sealing') + self.assertEqual({key: model[key] for key in expected}, expected) + self.assertLessEqual(model['max_workers'], 4) + self.assertLessEqual(model['max_in_flight_pictures'], 4) + self.assertLessEqual(model['max_pending_outputs'], 3) + self.assertLessEqual(model['critical_path_row_units'], model['steps']) + self.assertLessEqual(model['serial_work_over_steps'], model['four_worker_work_span_bound']) + self.assertLessEqual(model['four_worker_work_span_bound'], 4.0) + return result + + def test_forward_chain_has_explicit_sample_filter_margin_and_whole_reference_control(self): + data = trace([4, 4], {(1, row): {0: (row, None)} for row in range(4)}) + result = self.check_oracle(data) + for mode in result['models'].values(): + self.assertEqual(mode['filter_lag1']['steps'], 6) + self.assertEqual(mode['filter_lag1']['critical_path_row_units'], 6) + self.assertEqual(mode['filter_lag1']['max_workers'], 2) + self.assertEqual(mode['filter_lag1']['serial_row_work'], 8) + self.assertEqual(mode['filter_lag1']['normalized_picture_work'], 2) + self.assertEqual(mode['filter_lag1']['normalized_picture_span'], 1.5) + self.assertAlmostEqual(mode['filter_lag1']['serial_work_over_steps'], 8 / 6) + self.assertAlmostEqual(mode['filter_lag1']['four_worker_work_span_bound'], 8 / 6) + self.assertEqual(mode['filter_lag2']['steps'], 7) + self.assertEqual(mode['whole_reference_control']['steps'], 8) + self.assertEqual(mode['whole_reference_control']['max_workers'], 1) + + def test_far_future_reads_wait_for_complete_reference_not_just_current_row(self): + result = self.check_oracle(trace([4, 3], {(1, row): {0: (3, None)} for row in range(3)})) + for mode in result['models'].values(): + self.assertEqual(mode['filter_lag1']['steps'], 7) + self.assertEqual(mode['filter_lag2']['steps'], 7) + self.assertEqual(mode['filter_lag1']['overlap_pictures'], 0) + + def test_tmvp_waits_only_for_motion_row_and_may_reference_rps_outside_active(self): + data = trace([4, 4], {(1, row): {0: (None, row)} for row in range(4)}) + result = self.check_oracle(data) + for mode in result['models'].values(): + self.assertEqual(mode['filter_lag1']['steps'], 5) + self.assertEqual(mode['filter_lag2']['steps'], 5) + data = trace([2, 2, 2], {(2, 0): {0: (None, 1), 1: (0, None)}}, + rps=[[], [], [0, 1]], active=[[], [], [1]]) + self.assertEqual(self.check_oracle(data)['graph'], data) + + def test_header_references_without_actual_reads_do_not_become_row_dependencies(self): + data = trace([4, 4], rps=[[], [0]]) + result = self.check_oracle(data) + for mode in result['models'].values(): + self.assertEqual(mode['filter_lag1']['steps'], 4) + self.assertEqual(mode['filter_lag1']['max_workers'], 2) + self.assertEqual(mode['whole_reference_control']['steps'], 8) + + def test_unit_row_whole_reference_control_matches_existing_lag3_model(self): + for refs in ([[], [], [], [], []], [[], [0], [1], [2]], [[], [0], [0], [0], [0]], + [[], [0], [0], [1, 2], [0], [4], [5]]): + data = trace([1] * len(refs), {(index, 0): {target: (0, None) for target in targets} + for index, targets in enumerate(refs)}, + outputs=[(index, index + 1) for index in range(len(refs))]) + result = self.check_oracle(data) + control = analyze_header(data['header_dependencies'])['rps']['unit_duration_output_lag3'] + for mode in ('picture_ready', 'delayed_sealing'): + actual = result['models'][mode]['whole_reference_control'] + for key, value in control[mode]['full_picture_references'].items(): + if key != 'non_wpp_overlap_pictures': + self.assertEqual(actual[key], value) + + def test_tiled_current_or_reference_uses_atomic_whole_picture_fallback(self): + for tiles in ([0], [1], [0, 1]): + result = self.check_oracle(trace([4, 4], {(1, row): {0: (0, None)} for row in range(4)}, + tiles=tiles)) + self.assertEqual(result['tiled_picture_indices'], tiles) + for mode in result['models'].values(): + self.assertEqual(mode['filter_lag1']['steps'], 8) + self.assertEqual(mode['filter_lag1']['critical_path_row_units'], 8) + self.assertEqual(mode['filter_lag1']['max_workers'], 1) + self.assertIn('no raster-row entropy order', result['semantics']['tiles']) + + def test_unequal_row_counts_keep_equal_row_cost_normalization(self): + result = self.check_oracle(trace([4, 2], {(1, 0): {0: (0, None)}, (1, 1): {0: (3, None)}})) + metrics = result['models']['picture_ready']['filter_lag1'] + self.assertEqual(metrics['steps'], 5) + self.assertEqual(metrics['serial_row_work'], 6) + self.assertAlmostEqual(metrics['normalized_picture_steps'], 5 / 3) + self.assertEqual(metrics['normalized_picture_work'], 2) + + def test_four_worker_bound_is_distinct_from_scheduled_ratio_and_capped_at_four(self): + metrics = self.check_oracle(trace([2] * 8))['models']['picture_ready']['filter_lag1'] + self.assertEqual(metrics['unit_row_work_span_bound'], 8.0) + self.assertEqual(metrics['four_worker_work_span_bound'], 4.0) + self.assertEqual(metrics['serial_work_over_steps'], 4.0) + data = trace([1, 1, 1, 5, 1, 1, 1], outputs=[(0, 4), (1, 4), (2, 4), (3, 4)]) + metrics = self.check_oracle(data)['models']['picture_ready']['filter_lag1'] + self.assertGreater(metrics['four_worker_work_span_bound'], metrics['serial_work_over_steps']) + + def test_geometry_rejects_bigints_and_accepts_positive_u32_width_boundary(self): + for field in ('width', 'height', 'ctb_log2'): + for value in (1 << 32, 1 << 128): + data = trace([1]) + data['pictures'][0][field] = value + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + analyze(data) + data = trace([1]) + data['pictures'][0]['width'] = (1 << 32) - 1 + self.assertEqual(analyze(data)['models']['picture_ready']['filter_lag1']['steps'], 1) + + def test_skipped_sends_output_bursts_and_eof_preserve_lag_and_inflight_limits(self): + for data in (trace([2, None, 3, 1, 2, 4], outputs=[(0, 1), (2, 5), (3, 5), (4, 5), (5, 6)]), + trace([None, None], outputs=[]), trace([3]), + trace([2] * 8, outputs=[(index, 8) for index in reversed(range(8))])): + self.check_oracle(data) + result = analyze(trace([2] * 8)) + for mode in result['models'].values(): + self.assertEqual(mode['filter_lag1']['steps'], 4) + self.assertEqual(mode['filter_lag1']['max_workers'], 4) + self.assertEqual(mode['filter_lag1']['max_pending_outputs'], 3) + + def test_output_fences_include_unrelated_earlier_picture_rows(self): + data = trace([1, 1, 1, 5, 1, 1, 1], outputs=[(0, 4), (1, 4), (2, 4), (3, 4)]) + result = self.check_oracle(data) + self.assertEqual(result['models']['picture_ready']['filter_lag1']['steps'], 6) + self.assertIn('all-rows-of-all-earlier-pictures', result['semantics']['outputs']) + + def test_raw_capture_is_detached_and_replayable_with_explicit_hypothetical_limits(self): + data = trace([3, 2], {(1, 0): {0: (0, 0)}}) + original = copy.deepcopy(data) + result = analyze(data) + self.assertEqual(data, original) + self.assertEqual(analyze(result['graph']), result) + result['graph']['pictures'][0]['rows'].clear() + self.assertEqual(data, original) + self.assertIn('not current whole-picture filtering', result['semantics']['filter_lag1']) + self.assertIn('not implementability', result['semantics']['limits']) + self.assertIn('replace whole-reference input gates', result['semantics']['admission']) + + def test_small_exhaustive_read_maxima_and_random_multi_picture_traces_match_tick_oracle(self): + for reference_rows in range(1, 4): + for current_rows in range(1, 4): + for maxima in itertools.product(range(-1, reference_rows), repeat=current_rows): + for motion in (False, True): + reads = {(1, row): {0: (None, q) if motion else (q, None)} + for row, q in enumerate(maxima) if q >= 0} + self.check_oracle(trace([reference_rows, current_rows], reads, rps=[[], [0]])) + generator = random.Random(704) + for _ in range(80): + counts = [generator.randrange(1, 5) for _ in range(6)] + reads = {} + for index, count in enumerate(counts): + for row in range(count): + refs = {} + for target in range(index): + if generator.random() < 0.25: + refs[target] = (generator.randrange(counts[target]), + generator.randrange(counts[target]) if generator.random() < 0.5 else None) + reads[index, row] = refs + outputs = sorted(((index, generator.randrange(index + 1, 7)) for index in range(6)), + key=lambda pair: pair[1]) + self.check_oracle(trace(counts, reads, outputs=outputs, + tiles=[index for index in range(6) if generator.random() < 0.1])) + + def test_missing_root_picture_row_and_header_fields_are_value_errors(self): + original = trace([2, 2], {(1, 0): {0: (0, 0)}}) + for path in ((), ('pictures', 0), ('pictures', 1, 'rows', 0), ('header_dependencies',)): + value = original + for key in path: + value = value[key] + for field in value: + changed = copy.deepcopy(original) + target = changed + for key in path: + target = target[key] + del target[field] + with self.subTest(path=path, field=field), self.assertRaises(ValueError): + analyze(changed) + + def test_root_counts_modes_and_capture_worker_mutations_fail_closed(self): + original = trace([2, 2]) + mutations = {'mode': [None, 'header_dependencies', True], 'complete': [False, 1], + 'header_only': [True, 0], 'decoded_pictures': [True, -1, 1, 3, 2.0], + 'output_pictures': [None, True, -1, 1, 3, 2.0], + 'requested_threads': [True, 0, 2, 1.0], 'thread_limit': [True, 0, 2, 1.0], + 'max_workers': [True, 0, 2, 1.0], 'pictures': [None, {}, [], original['pictures'][::-1]], + 'header_dependencies': [None, {}, []]} + for field, values in mutations.items(): + for value in values: + changed = copy.deepcopy(original) + changed[field] = value + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + analyze(changed) + for value in (None, [], True, 1): + with self.assertRaises(ValueError): + analyze(value) + for mutation in ('wpp', 'events'): + changed = copy.deepcopy(original) + if mutation == 'wpp': + changed['header_dependencies']['pictures'][0]['wpp'] = True + else: + del changed['header_dependencies']['output_events'] + with self.assertRaises(ValueError): + analyze(changed) + + def test_geometry_row_counts_flags_and_calls_reject_numeric_booleans(self): + original = trace([2, 2], {(1, 0): {0: (0, 0)}}) + mutations = {'index': [True, 0, 2, 1.0], 'poc': [True, 0, -2.0], + 'width': [True, 0, -1, 32.0], 'height': [True, 0, 16, 48, 32.0, 32769], + 'ctb_log2': [True, 3, 7, 4.0], 'tiles': [0, 1, None], 'wpp': [True, 0, None], + 'rows': [None, {}, [], original['pictures'][1]['rows'][::-1]]} + for field, values in mutations.items(): + for value in values: + changed = copy.deepcopy(original) + changed['pictures'][1][field] = value + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + analyze(changed) + for field, values in {'row': [True, 1, -1, 0.0], 'sample_calls': [True, -1, 0, 1.0], + 'motion_calls': [True, -1, 0, 1.0]}.items(): + for value in values: + changed = copy.deepcopy(original) + changed['pictures'][1]['rows'][0][field] = value + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + analyze(changed) + changed = trace([2]) + changed['pictures'][0]['rows'][0]['sample_calls'] = 1 + with self.assertRaises(ValueError): + analyze(changed) + + def test_reference_shapes_identity_poc_footprints_and_counts_are_closed(self): + original = trace([2, 2], {(1, 0): {0: (0, None)}}) + for refs in (None, {}, [None], [[0]], [[0, -4, 0]], [[0, -4, 0, None, 0]], + [[True, -4, 0, None]], [[0.0, -4, 0, None]], [[0, True, 0, None]], + [[0, -3, 0, None]], [[-1, -4, 0, None]], [[1, -2, 0, None]], + [[2, 0, 0, None]], [[0, -4, True, None]], [[0, -4, 0.0, None]], + [[0, -4, -1, None]], [[0, -4, 2, None]], [[0, -4, None, None]], + [[0, -4, 0, True]], [[0, -4, 0, -1]], [[0, -4, 0, 2]], + [[0, -4, 0, None]] * 2, [[0, -4, 0, None]] * 17): + changed = copy.deepcopy(original) + changed['pictures'][1]['rows'][0]['references'] = refs + with self.subTest(refs=refs), self.assertRaises(ValueError): + analyze(changed) + changed = trace([2, None, 2], {(2, 0): {0: (0, None)}}) + changed['pictures'][1]['rows'][0]['references'][0][:2] = [1, -2] + with self.assertRaises(ValueError): + analyze(changed) + changed = trace([2, 2, 2], {(2, 0): {1: (0, None)}}) + changed['pictures'][2]['rows'][0]['references'][0][:2] = [0, -4] + with self.assertRaises(ValueError): + analyze(changed) + changed = trace([2, 2]) + changed['pictures'][1]['rows'][0].update(references=[[0, -4, 0, None]], sample_calls=1) + with self.assertRaises(ValueError): + analyze(changed) + + def test_2048_rows_sixteen_refs_and_partial_edge_ctu_geometry_are_supported(self): + data = trace([2048]) + data['pictures'][0]['height'] -= 1 + self.assertEqual(analyze(data)['models']['picture_ready']['filter_lag1']['steps'], 2048) + data = trace([1] * 17, {(16, 0): {index: (0, None) for index in range(16)}}) + self.check_oracle(data) + data['pictures'][-1]['rows'][0]['sample_calls'] = 15 + with self.assertRaises(ValueError): + analyze(data) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_hevc_threading.py b/scripts/tests/test_hevc_threading.py index 00336b2e5..658483bcc 100644 --- a/scripts/tests/test_hevc_threading.py +++ b/scripts/tests/test_hevc_threading.py @@ -1,6 +1,8 @@ import copy +import itertools import json import pathlib +import re import subprocess import sys import tempfile @@ -14,26 +16,82 @@ class ThreadingReference(unittest.TestCase): + def test_main_baseline_keeps_wavefront_threads_and_uses_only_the_sync_api(self): + root = pathlib.Path(__file__).resolve().parents[2] + source = (root / 'bench/hevc-reference/decode.rs').read_text() + self.assertIn('PictureDecoder::new(limits).with_threads(threads)', source) + baseline = source.split('#[cfg(hevc_synchronous_baseline)]', 1)[1].split( + '#[cfg(not(hevc_synchronous_baseline))]', 1)[0] + self.assertIn('decoder.read_nal(nal, None, &mut callback)', baseline) + self.assertNotIn('read_nal_scheduled', baseline) + self.assertIn('decoder.read_nal_scheduled(nal, None, None,', source) + self.assertNotIn('black_box(threads)', source) + workflow = (root / '.github/workflows/hevc-threading.yml').read_text() + self.assertIn('ref: ${{ inputs.baseline }}\n fetch-depth: 0', workflow) + self.assertIn('--cfg hevc_synchronous_baseline', workflow) + self.assertNotIn('hevc_serial_baseline', workflow) + def test_explicit_sanitizer_mode_overrides_reference_selection(self): workflow = pathlib.Path(__file__).resolve().parents[2] / '.github/workflows/hevc-threading.yml' job = workflow.read_text().split(' thread-sanitizer:', 1)[1] - self.assertIn("if: ${{ inputs.sanitizer_only || inputs.only == '' }}", job) + self.assertIn("if: ${{ !inputs.thread_diagnostics_only && !inputs.thread_dependencies_only && !inputs.thread_rows_only && (inputs.sanitizer_only || inputs.only == '') }}", job) + + def test_sanitizer_executes_both_scheduling_owners_and_rejects_zero_tests(self): + workflow = pathlib.Path(__file__).resolve().parents[2] / '.github/workflows/hevc-threading.yml' + job = workflow.read_text().split(' thread-sanitizer:', 1)[1] + self.assertIn('for group in wavefront decoder::scheduler::tests; do', job) + self.assertIn('--lib "$group" | tee', job) + self.assertIn('set -euo pipefail', job) + check = next(line.strip() for line in job.splitlines() if line.strip().startswith('grep -Eq')) + with tempfile.TemporaryDirectory() as temp: + log = pathlib.Path(temp) / 'threading-tests.log' + for count in (0, 1, 20): + log.write_text(f'test result: ok. {count} passed; 0 failed; 255 filtered out\n') + result = subprocess.run(['bash', '-c', check.replace('"$RUNNER_TEMP/threading-tests.log"', '"$1"'), + 'check', str(log)], capture_output=True, check=False) + self.assertEqual(result.returncode == 0, count > 0) def test_threaded_conformance_rejects_silent_serial_fallback(self): serial = dict(complete=True, summary=dict(decoded_pictures=3, parallel_pictures=0, max_workers=1, stats=dict(wpp_restarts=2))) - self.assertEqual(hevc_decode.threading_coverage([serial], 1), dict(parallel_pictures=0, max_workers=1)) + self.assertEqual(hevc_decode.threading_coverage([serial], 1), + dict(parallel_pictures=0, frame_parallel_pictures=0, max_workers=1)) for threads in (2, 4, 8): with self.assertRaisesRegex(ValueError, 'native_parallel_coverage_missing'): hevc_decode.threading_coverage([serial], threads) parallel = copy.deepcopy(serial) parallel['summary'].update(parallel_pictures=2, max_workers=2) self.assertEqual(hevc_decode.threading_coverage([serial, parallel], 4), - dict(parallel_pictures=2, max_workers=2)) + dict(parallel_pictures=2, frame_parallel_pictures=0, max_workers=2)) parallel['summary']['stats']['wpp_restarts'] = 0 with self.assertRaisesRegex(ValueError, 'native_parallel_wpp_work'): hevc_decode.threading_coverage([parallel], 4) + def test_frame_coverage_requires_a_complete_overlapping_pair(self): + frame = dict(complete=True, summary=dict(decoded_pictures=3, parallel_pictures=2, + frame_parallel_pictures=2, max_workers=2, stats=dict(wpp_restarts=0))) + self.assertEqual(hevc_decode.threading_coverage([frame], 4), + dict(parallel_pictures=2, frame_parallel_pictures=2, max_workers=2)) + for count in (True, -1, 1, 3, '2', None): + changed = copy.deepcopy(frame) + changed['summary']['frame_parallel_pictures'] = count + with self.subTest(count=count), self.assertRaisesRegex(ValueError, 'native_frame_thread_counters'): + hevc_decode.threading_coverage([changed], 4) + for workers in (0, 1, 5): + changed = copy.deepcopy(frame) + changed['summary']['max_workers'] = workers + with self.subTest(workers=workers), self.assertRaisesRegex(ValueError, 'native_thread_counters'): + hevc_decode.threading_coverage([changed], 4) + + def test_frame_coverage_does_not_hide_unproved_wavefront_credit(self): + mixed = dict(complete=True, summary=dict(decoded_pictures=4, parallel_pictures=3, + frame_parallel_pictures=2, max_workers=2, stats=dict(wpp_restarts=0))) + with self.assertRaisesRegex(ValueError, 'native_parallel_wpp_work'): + hevc_decode.threading_coverage([mixed], 4) + mixed['summary']['stats']['wpp_restarts'] = 2 + self.assertEqual(hevc_decode.threading_coverage([mixed], 4), + dict(parallel_pictures=3, frame_parallel_pictures=2, max_workers=2)) + def test_unverified_baseline_cannot_pass_or_continue_the_shard(self): with tempfile.TemporaryDirectory() as temp: out = pathlib.Path(temp) / 'out' @@ -107,6 +165,7 @@ def test_identity_ignores_only_resource_and_scheduling_counters(self): one = dict(output_md5='a' * 32, stats=dict(ctus=3, memory_bytes=100), parallel_pictures=0, max_workers=1) many = copy.deepcopy(one) many.update(parallel_pictures=1, max_workers=8) + many['frame_parallel_pictures'] = 2 many['stats']['memory_bytes'] = 500 self.assertEqual(threading.normalized(one), threading.normalized(many)) for owner, key in [(many, 'output_md5'), (many['stats'], 'ctus')]: @@ -122,6 +181,56 @@ def test_thread_count_reaches_every_conformance_control(self): del args.threads self.assertEqual(hevc_decode.command(args, '/input'), ['/native', '/input', '100', '200', '300']) + def test_complete_conformance_compares_every_semantic_record(self): + one = dict(complete=True, passed=True, scope='complete_c1', identities=dict(native='a' * 64), + results=[dict(complete=True, sha256='b' * 64, + summary=dict(decoded_pictures=2, parallel_pictures=0, max_workers=1, + stats=dict(bins=25, memory_bytes=100)), + pictures=[dict(output_md5='c' * 32, poc=1)])]) + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + for count in (1, 2, 4, 8): + directory = root / f'threads-{count}' + directory.mkdir() + many = copy.deepcopy(one) + many['results'][0]['summary'].update(frame_parallel_pictures=2, parallel_pictures=2, + max_workers=count) + many['results'][0]['summary']['stats']['memory_bytes'] = count * 100 + (directory / 'result.json').write_text(json.dumps(one if count == 1 else many)) + receipt = threading.compare_conformance(root) + self.assertEqual((receipt['passed'], receipt['streams'], receipt['pictures']), (True, 1, 2)) + self.assertEqual(receipt['threading']['4']['frame_parallel_pictures'], 2) + self.assertEqual(receipt['threading']['4']['picture_job_streams'], 1) + for field in ('bins', 'output_md5', 'poc', 'source'): + changed = copy.deepcopy(many) + if field == 'bins': + changed['results'][0]['summary']['stats']['bins'] += 1 + elif field == 'source': + changed['results'][0]['sha256'] = 'd' * 64 + else: + changed['results'][0]['pictures'][0][field] = 'changed' + (root / 'threads-8' / 'result.json').write_text(json.dumps(changed)) + with self.subTest(field=field), self.assertRaisesRegex(ValueError, 'thread_conformance_mismatch'): + threading.compare_conformance(root) + changed = copy.deepcopy(many) + changed['complete'] = False + (root / 'threads-8' / 'result.json').write_text(json.dumps(changed)) + with self.assertRaisesRegex(ValueError, 'thread_conformance_incomplete'): + threading.compare_conformance(root) + for count in (2, 4, 8): + changed = copy.deepcopy(many) + changed['results'][0]['summary']['frame_parallel_pictures'] = 0 + changed['results'][0]['summary']['max_workers'] = count + changed['results'][0]['summary']['stats']['wpp_restarts'] = 1 + (root / f'threads-{count}' / 'result.json').write_text(json.dumps(changed)) + changed = copy.deepcopy(one) + changed['results'][0]['summary']['stats']['wpp_restarts'] = 1 + (root / 'threads-1' / 'result.json').write_text(json.dumps(changed)) + with self.assertRaisesRegex(ValueError, 'native_picture_job_coverage_missing'): + threading.compare_conformance(root) + workflow = (pathlib.Path(__file__).resolve().parents[2] / '.github/workflows/hevc-threading.yml').read_text() + self.assertIn('hevc_threading.py compare-conformance --root', workflow) + def test_signature_requires_completion_and_supplies_threads(self): with patch.object(threading.reference, 'command', return_value=(b'{"type":"summary","complete":true}', {})) as child: for threads in (0, 4): @@ -152,6 +261,465 @@ def test_ratio_direction_and_every_requested_count(self): self.assertEqual(summary['candidate_one_over_threads']['4'], 4) self.assertEqual(summary['candidate_over_ffmpeg_one']['1'], 3) + def test_timed_receipts_retain_requested_host_and_frame_work(self): + args = types.SimpleNamespace(native='/candidate', main='/main', repeats=3) + oracle = types.SimpleNamespace(ffmpeg='/ffmpeg', ffprobe='/ffprobe') + def child(command, **unused): + if command[0] == oracle.ffmpeg: + return b'frame=2\n', dict(wall=1, user=0.5, system=0.1) + requested = command[-1] + limit = min(requested or 4, 4) + data = dict(pictures=2, decoded_pictures=2, bins=5, requested_threads=requested, + host_threads=4, thread_limit=limit, max_memory_bytes=8888) + if command[0] == args.native: + data.update(parallel_pictures=2 if limit > 1 else 0, + frame_parallel_pictures=2 if limit > 1 else 0, + max_workers=min(limit, 2)) + return json.dumps(data).encode(), dict(wall=1, user=0.5, system=0.1) + with patch.object(threading.conformance, 'digest', return_value='a' * 64), \ + patch.object(threading.reference, 'verify', return_value=dict(frames=2, depth=8)), \ + patch.object(threading, 'signature', return_value=[dict(output_md5='a')]), \ + patch.object(threading.reference, 'command', side_effect=child): + result = threading.measure(args, oracle, pathlib.Path('/input'), 'd00') + self.assertEqual(result['status'], 'timed') + for call in result['calls']: + if call['arm'].startswith('candidate_'): + requested = int(call['arm'].split('_')[1]) + self.assertEqual(call['requested_threads'], requested) + self.assertEqual(call['host_threads'], 4) + self.assertEqual(call['thread_limit'], min(requested or 4, 4)) + self.assertEqual(call['frame_parallel_pictures'], 0 if requested == 1 else 2) + + +class ThreadDiagnostics(unittest.TestCase): + def record(self): + return dict(pictures=3, decoded_pictures=4, bins=55, requested_threads=4, host_threads=2, + thread_limit=2, max_memory_bytes=8888, parallel_pictures=2, + frame_parallel_pictures=2, max_workers=2, + thread_diagnostics=[[key, 0] for key in sorted(threading.DIAGNOSTIC_KEYS)]) + + def test_requires_complete_unique_counters_and_picture_denominators(self): + valid = self.record() + self.assertEqual(len(threading.diagnostic_record(valid)['barrier_histogram']), 16) + events = threading.diagnostic_record(valid)['events'] + self.assertEqual(len(events), 19) + self.assertTrue({f'event_sei_type_{kind}' for kind in (0, 1, 5, 129, 132, 'other')} <= events.keys()) + serial = dict(valid, parallel_pictures=0, frame_parallel_pictures=0, max_workers=1) + self.assertEqual(threading.diagnostic_record(serial)['raw']['frame_parallel_pictures'], 0) + mutations = [] + for key in valid: + missing = copy.deepcopy(valid) + del missing[key] + mutations.append(missing) + for key, values in { + 'pictures': (0, 5, True), 'decoded_pictures': (0, 2, 33), 'bins': (0, -1, True), + 'requested_threads': (0, 1, 8), 'host_threads': (0, -1), 'thread_limit': (0, 1, 4), + 'max_workers': (0, 3), 'max_memory_bytes': (0,), + 'parallel_pictures': (-1, 5), 'frame_parallel_pictures': (-1, 1, 3, '2') + }.items(): + for value in values: + changed = copy.deepcopy(valid) + changed[key] = value + mutations.append(changed) + for counters in ([], None, valid['thread_diagnostics'][:-1], + valid['thread_diagnostics'] + valid['thread_diagnostics'][:1], + valid['thread_diagnostics'] + [['barrier_unknown', 0]]): + mutations.append(dict(valid, thread_diagnostics=counters)) + for value in (-1, True, '1', 1 << 64, 5): + changed = copy.deepcopy(valid) + changed['thread_diagnostics'][0][1] = value + mutations.append(changed) + for changed in mutations: + with self.subTest(changed=changed), self.assertRaises(ValueError): + threading.diagnostic_record(changed) + for key in threading.DIAGNOSTIC_KEYS: + missing = dict(valid, thread_diagnostics=[pair for pair in valid['thread_diagnostics'] if pair[0] != key]) + with self.subTest(key=key), self.assertRaisesRegex(ValueError, 'diagnostic_counters_missing'): + threading.diagnostic_record(missing) + + def test_histograms_allow_multiple_reasons_and_distinguish_events(self): + data = self.record() + for pair in data['thread_diagnostics']: + pair[1] = 4 if pair[0].startswith('barrier_') else 999 + row = dict(id='d00', status='decoded', **threading.diagnostic_record(data)) + second = dict(row, id='c00') + summary = threading.diagnostic_summary([row, second], {'d00', 'c00'}) + self.assertTrue(summary['complete']) + self.assertEqual(summary['pictures'], 6) + self.assertEqual(summary['decoded_pictures'], 8) + self.assertEqual(summary['frame_parallel_pictures'], 4) + self.assertEqual(summary['inputs_with_overlap'], 2) + self.assertEqual(summary['barrier_histogram'], {key: 8 for key in row['barrier_histogram']}) + self.assertEqual(row['events']['event_reconcile'], 999) + self.assertFalse(threading.diagnostic_summary([], set())['complete']) + self.assertFalse(threading.diagnostic_summary([row, row], {'d00'})['complete']) + self.assertEqual(threading.diagnostic_summary([row], {'d00', 'c00'})['missing_inputs'], 1) + + def run_diagnostics(self, inputs=None, output=None, dependencies=False, rows=False): + with tempfile.TemporaryDirectory() as temp: + out = pathlib.Path(temp) / 'out' + oracle = types.SimpleNamespace(ffmpeg='/qualified/ffmpeg', ffprobe='/qualified/ffprobe') + pool = types.SimpleNamespace(limits=types.SimpleNamespace(requests=0), close=lambda: None) + selected = threading.ROW_INPUT_IDS if rows else threading.INPUT_IDS + inputs = [(key, pathlib.Path('/' + key), 'd' * 64) for key in selected] if inputs is None else inputs + child = types.SimpleNamespace(stdout=json.dumps(self.record()).encode()) + def result(*args, **kwargs): + return output(*args, **kwargs) if output else child + with patch.object(threading.sys, 'platform', 'linux'), \ + patch.dict(threading.os.environ, GITHUB_ACTIONS='true'), \ + patch.object(threading.test_oracle, 'require', return_value=oracle) as require, \ + patch.object(threading.conformance, 'digest', return_value='c' * 64), \ + patch.object(threading.oracle_pin, 'load', return_value=dict(BUILD='qualified')), \ + patch.object(pathlib.Path, 'read_text', return_value='model name : fixture host\n'), \ + patch.object(threading.os, 'sched_getaffinity', return_value={0, 1}, create=True), \ + patch.object(threading.round_files, 'pool', return_value=pool), \ + patch.object(threading.reference, 'inputs', return_value=inputs) as prepare, \ + patch.object(threading.subprocess, 'run', side_effect=result) as run, \ + patch.object(threading.reference, 'command', side_effect=AssertionError('timing forbidden')), \ + patch.object(threading.reference, 'verify', side_effect=AssertionError('comparison forbidden')), \ + patch.object(threading.reference, 'profile', side_effect=AssertionError('profiling forbidden')): + extra = ['--dependencies-only'] if dependencies else [] + if rows: + extra.append('--rows-only') + code = threading.diagnostics(['--native', '/native', '--oracle-bin', '/caller-oracle', + '--out', str(out), '--candidate', 'a' * 40, *extra]) + require.assert_called_once_with(pathlib.Path('/caller-oracle/ffmpeg'), pathlib.Path('/caller-oracle/ffprobe')) + self.assertIs(prepare.call_args.args[1], oracle) + self.assertEqual(prepare.call_args.args[0].frames, 32) + self.assertEqual(prepare.call_args.args[0].only, set(selected)) + self.assertEqual(pool.limits.max_requests, 4000) + self.assertEqual(pool.limits.max_bytes, 32 << 30) + calls = run.call_args_list + return code, json.loads((out / 'results.json').read_text()), calls + + def test_each_existing_input_runs_once_in_decode_mode_without_timing(self): + code, report, calls = self.run_diagnostics() + self.assertEqual(code, 0) + self.assertEqual(len(calls), 30) + self.assertEqual(report['candidate'], 'a' * 40) + self.assertEqual(report['candidate_sha256'], 'c' * 64) + self.assertEqual(report['frames_cap'], 32) + for key, call, row in zip(threading.INPUT_IDS, calls, report['files']): + self.assertEqual(call.args[0], ['/native', '/' + key, 'decode', '4']) + self.assertEqual(call.kwargs, dict(capture_output=True, check=True, timeout=600, + env=threading.reference.CHILD_ENV)) + self.assertEqual(row['raw'], self.record()) + self.assertEqual(row['input_sha256'], 'c' * 64) + self.assertEqual(report['summary']['decoded_pictures'], 120) + self.assertEqual(report['summary']['frame_parallel_pictures'], 60) + self.assertNotIn('wall', json.dumps(report)) + self.assertNotIn('baseline', report) + + def test_header_replay_never_reconstructs_and_keeps_the_prepared_input_binding(self): + rows = [dict(index=i, poc=i, reconstruction_required=True, wpp=False, + slice_types=[2] if i == 0 else [1], + rps=[] if i == 0 else [[0, 0, False]], + active=[] if i == 0 else [[0, 0, False]]) for i in range(3)] + data = dict(mode='header_dependencies', header_only=True, complete=True, + coded_pictures=3, reconstruction_required=3, output_pictures=3, + nals=6, pictures=rows, output_events=[ + dict(picture_index=i, poc=i, before_picture=i + 1) for i in range(3)]) + child = lambda *a, **kw: types.SimpleNamespace(stdout=json.dumps(data).encode()) + code, report, calls = self.run_diagnostics(output=child, dependencies=True) + self.assertEqual((code, report['mode'], report['requested_threads']), (0, 'header_dependencies', 0)) + self.assertEqual((report['summary']['parsed_inputs'], report['summary']['coded_pictures']), (30, 90)) + for key, call, row in zip(threading.INPUT_IDS, calls, report['files']): + self.assertEqual(call.args[0], ['/native', '/' + key, 'dependencies', '1']) + self.assertEqual(row['status'], 'parsed') + self.assertEqual(row['input_sha256'], 'c' * 64) + self.assertEqual(row['dependency_ceiling']['active']['coded_admission_width'], 2) + self.assertEqual(row['dependency_ceiling']['active']['unit_duration_receive_barrier']['4'], + dict(steps=2, max_workers=2, overlap_pictures=2, + non_wpp_overlap_pictures=2)) + del data['output_events'] + missing, invalid, _ = self.run_diagnostics(output=child, dependencies=True) + self.assertEqual((missing, invalid['summary']['errors']), (1, 30)) + failed, bad, _ = self.run_diagnostics(dependencies=True) + self.assertEqual((failed, bad['summary']['errors']), (1, 30)) + self.assertTrue(all(row['failure_reason'] == 'dependency_record_invalid' for row in bad['files'])) + + def test_errors_missing_counters_and_empty_preparation_fail(self): + for output in (lambda *a, **kw: types.SimpleNamespace(stdout=b'{}'), + lambda *a, **kw: types.SimpleNamespace(stdout=b'not json')): + code, report, calls = self.run_diagnostics(output=output) + self.assertEqual(code, 1) + self.assertEqual(len(calls), 30) + self.assertEqual(report['summary']['errors'], 30) + def failed(*args, **kwargs): + raise subprocess.CalledProcessError(9, args[0], stderr=b'private decode log') + code, report, calls = self.run_diagnostics(output=failed) + self.assertEqual(code, 1) + self.assertEqual(report['summary']['errors'], 30) + self.assertEqual(report['files'][0]['exit_code'], 9) + self.assertNotIn('private decode log', json.dumps(report)) + code, report, calls = self.run_diagnostics(inputs=[]) + self.assertEqual((code, len(calls), report['summary']['missing_inputs']), (1, 0, 30)) + self.assertFalse(report['complete']) + code, report, calls = self.run_diagnostics(inputs=[('d00', '/input', 'd' * 64)]) + self.assertEqual((code, len(calls), report['summary']['missing_inputs']), (1, 1, 29)) + code, report, calls = self.run_diagnostics(inputs=[('d00', '/input', 'd' * 64)] * 2) + self.assertEqual((code, len(calls), report['summary']['preparation_errors']), (1, 1, 1)) + self.assertEqual(report['summary']['errors'], 1) + + def test_diagnostics_refuse_non_actions_or_non_linux_before_acquisition(self): + for host, actions in [('darwin', 'true'), ('linux', 'false')]: + with patch.object(threading.sys, 'platform', host), \ + patch.dict(threading.os.environ, GITHUB_ACTIONS=actions), \ + patch.object(threading.test_oracle, 'require') as require, \ + self.assertRaises(SystemExit): + threading.diagnostics(['--native', '/native', '--oracle-bin', '/oracle', + '--out', '/out', '--candidate', 'a' * 40]) + require.assert_not_called() + + def test_row_and_header_selections_are_exclusive_before_acquisition(self): + with patch.object(threading.test_oracle, 'require') as require, self.assertRaises(SystemExit): + threading.diagnostics(['--native', '/native', '--oracle-bin', '/oracle', '--out', '/out', + '--candidate', 'a' * 40, '--rows-only', '--dependencies-only']) + require.assert_not_called() + + def test_row_selector_is_not_available_in_ordinary_mode(self): + command = [sys.executable, '-I', str(pathlib.Path(threading.__file__)), + '--native', '/native', '--main', '/main', '--oracle-bin', '/oracle', '--out', '/out', + '--candidate', 'a' * 40, '--baseline', 'b' * 40, '--rows-only'] + result = subprocess.run(command, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 2) + self.assertIn(b'unrecognized arguments: --rows-only', result.stderr) + + def row_record(self): + headers = [dict(index=index, poc=index, reconstruction_required=True, wpp=False, + slice_types=[1] if index else [2], + rps=[[index - 1, index - 1, False]] if index else [], + active=[[index - 1, index - 1, False]] if index else []) for index in range(3)] + header = dict(mode='header_dependencies', complete=True, header_only=True, + coded_pictures=3, reconstruction_required=3, output_pictures=3, nals=6, + pictures=headers, output_events=[dict(picture_index=i, poc=i, before_picture=i + 1) + for i in range(3)]) + pictures = [dict(index=index, poc=index, width=32, height=48, ctb_log2=4, tiles=False, wpp=False, + rows=[dict(row=row, sample_calls=int(index > 0), motion_calls=int(index > 0), + references=[[index - 1, index - 1, row, row]] if index else []) + for row in range(3)]) for index in range(3)] + return dict(mode='row_dependencies', complete=True, header_only=False, requested_threads=1, + thread_limit=1, max_workers=1, decoded_pictures=3, output_pictures=3, + header_dependencies=header, pictures=pictures) + + def test_rows_decode_exactly_the_existing_24_non_wpp_inputs_with_hashes_and_no_timing(self): + expected = tuple([f'd{i:02}' for i in range(26) if i not in (3, 6, 7, 10, 14, 22)] + + [f'c{i:02}' for i in range(4)]) + self.assertEqual(threading.ROW_INPUT_IDS, expected) + data = self.row_record() + child = lambda *a, **kw: types.SimpleNamespace(stdout=json.dumps(data).encode()) + code, report, calls = self.run_diagnostics(output=child, rows=True) + self.assertEqual((code, report['passed'], report['complete']), (0, True, True)) + self.assertEqual((report['mode'], report['header_only'], report['requested_threads']), + ('row_dependencies', False, 1)) + self.assertEqual(report['selected_ids'], list(expected)) + self.assertEqual(report['summary'], dict(selected_inputs=24, attempted_inputs=24, decoded_inputs=24, + decoded_pictures=72, errors=0, missing_inputs=0, + preparation_errors=0, complete=True)) + self.assertEqual(len(calls), 24) + for key, call, row in zip(expected, calls, report['files']): + self.assertEqual(call.args[0], ['/native', '/' + key, 'rows', '1']) + self.assertEqual(call.kwargs, dict(capture_output=True, check=True, timeout=600, + env=threading.reference.CHILD_ENV)) + self.assertEqual((row['status'], row['input_sha256'], row['source_sha256']), + ('decoded', 'c' * 64, 'd' * 64)) + self.assertEqual(row['raw'], data) + self.assertEqual(row['row_dependency_ceiling'], threading.hevc_row_dependency.analyze(data)) + for field in ('baseline', 'wall', 'calls', 'warmups', 'barrier_histogram'): + self.assertNotIn(field, report) + self.assertIn('actual decoded inter-reference reads', report['scope']) + + def test_rows_missing_invalid_and_failed_decodes_never_receive_success_credit(self): + valid = self.row_record() + invalid = [{}, [], dict(valid, header_only=True), dict(valid, complete=False), + dict(valid, decoded_pictures=2), dict(valid, requested_threads=4)] + missing = copy.deepcopy(valid) + del missing['pictures'][0]['rows'] + invalid.append(missing) + changed = copy.deepcopy(valid) + changed['pictures'][1]['rows'][0]['references'][0][2] = 999 + invalid.append(changed) + for data in invalid: + child = lambda *a, **kw: types.SimpleNamespace(stdout=json.dumps(data).encode()) + with self.subTest(data=data): + code, report, calls = self.run_diagnostics(output=child, rows=True) + self.assertEqual((code, len(calls), report['summary']['errors'], + report['summary']['decoded_inputs'], report['summary']['decoded_pictures']), + (1, 24, 24, 0, 0)) + self.assertTrue(all(row['failure_reason'] == 'row_dependency_record_invalid' + for row in report['files'])) + child = lambda *a, **kw: types.SimpleNamespace(stdout=json.dumps(valid).encode()) + for inputs, attempted, missing, preparation_errors in [([], 0, 24, 0), + ([('d00', '/input', 'd' * 64)], 1, 23, 0), + ([('d00', '/input', 'd' * 64)] * 2, 1, 23, 1), + ([('d03', '/excluded-wpp', 'd' * 64)], 0, 24, 1)]: + code, report, calls = self.run_diagnostics(inputs=inputs, output=child, rows=True) + self.assertEqual((code, len(calls), report['summary']['missing_inputs'], + report['summary']['preparation_errors']), (1, attempted, missing, preparation_errors)) + self.assertFalse(report['complete']) + def failed(*args, **kwargs): + raise subprocess.TimeoutExpired(args[0], 600, stderr=b'private row decode log') + code, report, calls = self.run_diagnostics(output=failed, rows=True) + self.assertEqual((code, len(calls), report['summary']['errors'], report['summary']['decoded_pictures']), + (1, 24, 24, 0)) + self.assertNotIn('private row decode log', json.dumps(report)) + + def test_row_selection_filters_wpp_before_any_source_fetch_or_preparation(self): + roster = [dict(partition='development', sha256=f'{i + 1:064x}', size=100, ext='mp4') + for i in range(26)] + manifest = json.dumps(dict(files=[dict(sha256=item['sha256'], size=100, ext='mp4', key=f'd{i:02}') + for i, item in enumerate(roster)])).encode() + pool = types.SimpleNamespace(head=lambda *a: (len(manifest), 'etag'), get=lambda *a: manifest, + limits=object()) + args = types.SimpleNamespace(only=set(threading.ROW_INPUT_IDS), frames=32) + oracle = types.SimpleNamespace(ffmpeg='/qualified/ffmpeg', ffprobe='/qualified/ffprobe') + with tempfile.TemporaryDirectory() as temp, \ + patch.object(threading.conformance, 'roster', return_value=roster), \ + patch.object(threading.conformance, 'registration', return_value={}), \ + patch.object(threading.reference, 'CONFORMANCE_BITSTREAMS', ()), \ + patch.object(threading.reference, 'fetch', return_value=pathlib.Path(temp) / 'original') as fetch, \ + patch.object(threading.reference, 'command') as command: + prepared = list(threading.reference.inputs(args, oracle, pathlib.Path(temp), pool)) + expected = [identity for identity in threading.ROW_INPUT_IDS if identity.startswith('d')] + self.assertEqual([row[0] for row in prepared], expected) + self.assertEqual(fetch.call_count, 20) + self.assertEqual(command.call_count, 20) + self.assertEqual([call.args[0]['sha256'] for call in fetch.call_args_list], + [roster[int(identity[1:])]['sha256'] for identity in expected]) + for call in command.call_args_list: + self.assertEqual(call.args[0][0], oracle.ffmpeg) + self.assertIn('-frames:v', call.args[0]) + self.assertEqual(call.args[0][call.args[0].index('-frames:v') + 1], '32') + + def test_workflow_diagnostic_selector_excludes_every_other_job(self): + root = pathlib.Path(__file__).resolve().parents[2] + dispatch = (root / '.github/workflows/hevc-reference.yml').read_text() + reusable = (root / '.github/workflows/hevc-threading.yml').read_text() + def gate(workflow, job, values): + text = workflow.split(f'\n {job}:\n', 1)[1].splitlines()[0].strip().removeprefix('if: ') + text = text.strip("'").removeprefix('${{ ').removesuffix(' }}') + text = re.sub(r'inputs\.([a-z_]+)', lambda match: repr(values[match[1]]), text) + text = re.sub(r'!(?!=)', ' not ', text).replace('&&', ' and ').replace('||', ' or ') + return bool(eval(text)) + for inverse, diagnostics, dependencies, rows, sanitizer, baseline, only in itertools.product( + (False, True), (False, True), (False, True), (False, True), (False, True), ('', 'b' * 40), ('', 'd00')): + values = dict(thread_diagnostics_only=diagnostics, thread_dependencies_only=dependencies, + thread_rows_only=rows, sanitizer_only=sanitizer, baseline=baseline, only=only, inverse=inverse) + ordinary = gate(dispatch, 'reference', values) + threading_job = gate(dispatch, 'threading', values) + self.assertEqual(ordinary, not inverse and not diagnostics and not dependencies and not rows and not sanitizer and not baseline) + self.assertEqual(bool(threading_job), not inverse and bool(diagnostics or dependencies or rows or sanitizer or baseline)) + self.assertEqual(gate(dispatch, 'inverse-comparison', values), inverse) + self.assertEqual(gate(reusable, 'diagnostics', values), diagnostics or dependencies or rows) + for job in ('reference', 'conformance', 'thread-sanitizer'): + if diagnostics or dependencies or rows: + self.assertFalse(gate(reusable, job, values)) + self.assertIn('thread_diagnostics_only: ${{ inputs.thread_diagnostics_only }}', dispatch) + self.assertIn('thread_dependencies_only: ${{ inputs.thread_dependencies_only }}', dispatch) + self.assertIn('thread_rows_only: ${{ inputs.thread_rows_only }}', dispatch) + for workflow in (dispatch, reusable): + self.assertRegex(workflow, r'thread_rows_only:\n(?:[^\n]+\n)*? default: false\n type: boolean') + job = reusable.split(' diagnostics:', 1)[1].split('\n reference:', 1)[0] + self.assertEqual(job.count('uses: actions/checkout@'), 1) + self.assertIn('ref: ${{ inputs.candidate }}\n fetch-depth: 0', job) + self.assertLess(job.index('git rev-parse HEAD'), job.index('cargo build')) + self.assertLess(job.index('git branch -r --contains'), job.index('secrets.')) + self.assertIn("--cfg hevc_thread_diagnostics --check-cfg=cfg(hevc_thread_diagnostics) --check-cfg=cfg(hevc_synchronous_baseline)", job) + self.assertNotIn('--cfg hevc_synchronous_baseline', job) + for forbidden in ('valgrind', 'hevc_conformance.py', 'path: baseline'): + self.assertNotIn(forbidden, job) + self.assertIn('test -z "$ONLY"', job) + self.assertIn('test "$FRAMES" = 32', job) + self.assertIn('--candidate "$CANDIDATE" "${EXTRA[@]}" > "$RUNNER_TEMP/private/diagnostics.log" 2>&1', job) + self.assertIn('EXTRA+=(--dependencies-only)', job) + self.assertIn('EXTRA+=(--rows-only)', job) + self.assertIn('-p test_hevc_row_dependency.py', job) + self.assertIn('RUSTFLAGS=\'--cfg hevc_thread_diagnostics\' cargo test --locked -p coreav-hevc --features native-hevc --lib', job) + self.assertIn("grep -Eq '^test result: ok\\. [1-9][0-9]* passed; 0 failed;'", job) + self.assertIn('tar -czf records.tgz private', job) + self.assertIn('rsa_oaep_md:sha256', job) + self.assertIn('name: hevc-threading-diagnostics-${{ inputs.candidate }}', job) + self.assertIn('path: ${{ runner.temp }}/sealed', job) + upload = job.split('- uses: actions/upload-artifact@')[1] + self.assertNotIn('/private', upload) + + def test_ordinary_pr_ci_executes_cfg_tests_and_excludes_the_private_bridge_from_releases(self): + workflow = (pathlib.Path(__file__).resolve().parents[2] / '.github/workflows/hevc-conformance.yml').read_text() + step = workflow.split('- name: Check diagnostic-only extraction and row reads\n', 1)[1].split('\n - ', 1)[0] + self.assertIn("RUSTFLAGS='--cfg hevc_thread_diagnostics' cargo test", step) + self.assertIn('-p coreav-hevc -p coreav-kernels-hevc --features native-hevc --lib', step) + for name in ('decoder::scheduler::tests::header_only_dependency_replay_distinguishes_chains_and_siblings', + 'decoder::scheduler::tests::row_dependency_trace_observes_generated_predictions', + 'inter::tests::diagnostic_read_bounds_match_interpolation_support'): + self.assertIn(f"grep -Fx 'test {name} ... ok'", step) + self.assertIn('set -euo pipefail', step) + self.assertIn('for feature in release-default release-full; do', workflow) + self.assertIn('native-hevc-private|coreav-(kernels-|backend-)?hevc', workflow) + + def test_workflow_cfg_tests_require_both_row_owners_and_header_extraction(self): + workflow = (pathlib.Path(__file__).resolve().parents[2] / '.github/workflows/hevc-threading.yml').read_text() + step = workflow.split('- name: Check cfg-only dependency extraction and scheduling tests\n', 1)[1] + command = step.split(' run: |\n', 1)[1].split('\n - ', 1)[0] + decoder = ('decoder::scheduler::tests::header_only_dependency_replay_distinguishes_chains_and_siblings', + 'decoder::scheduler::tests::row_dependency_trace_observes_generated_predictions') + kernel = 'inter::tests::diagnostic_read_bounds_match_interpolation_support' + self.assertIn("RUSTFLAGS='--cfg hevc_thread_diagnostics' cargo test --locked -p coreav-hevc --features native-hevc --lib", command) + self.assertIn("RUSTFLAGS='--cfg hevc_thread_diagnostics' cargo test --locked -p coreav-kernels-hevc --features native-hevc --lib", command) + def log(names, count=2): + return '\n'.join([*(f'test {name} ... ok' for name in names), + f'test result: ok. {count} passed; 0 failed; 200 filtered out']) + '\n' + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + cargo = root / 'cargo' + cargo.write_text('#!/bin/bash\ncase "$*" in\n*coreav-kernels-hevc*) cat "$KERNEL_LOG"; exit "${KERNEL_EXIT:-0}";;\n*) cat "$DECODER_LOG"; exit "${DECODER_EXIT:-0}";;\nesac\n') + cargo.chmod(0o700) + command = command.replace('cargo test', f'"{cargo}" test') + env = dict(PATH='/usr/bin:/bin', RUNNER_TEMP=temp, DECODER_LOG=str(root / 'decoder.log'), + KERNEL_LOG=str(root / 'kernel.log')) + cases = [(log(decoder), log([kernel]), {}, True)] + cases.extend((log([name for name in decoder if name != missing]), log([kernel]), {}, False) + for missing in decoder) + cases += [(log(decoder), log(['inter::tests::unrelated']), {}, False), + (log(decoder, 0), log([kernel]), {}, False), + (log(decoder), log([kernel], 0), {}, False), + (log(decoder), log([kernel]), dict(DECODER_EXIT='1'), False), + (log(decoder), log([kernel]), dict(KERNEL_EXIT='1'), False)] + for decoder_log, kernel_log, extra, passes in cases: + (root / 'decoder.log').write_text(decoder_log) + (root / 'kernel.log').write_text(kernel_log) + result = subprocess.run(['bash', '-c', command], env=dict(env, **extra), capture_output=True, timeout=10) + self.assertEqual(result.returncode == 0, passes, result.stdout + result.stderr) + + def test_workflow_rejects_conflicting_modes_before_credentials_and_forwards_row_flag(self): + workflow = (pathlib.Path(__file__).resolve().parents[2] / '.github/workflows/hevc-threading.yml').read_text() + job = workflow.split(' diagnostics:\n', 1)[1].split('\n reference:', 1)[0] + guard = next(line.strip() for line in job.splitlines() if line.strip().startswith('test "$DEPENDENCIES_ONLY"')) + self.assertLess(job.index(guard), job.index('cargo test')) + self.assertLess(job.index(guard), job.index('secrets.')) + step = job.split('- name: Inspect each prepared input once\n', 1)[1] + command = step.split(' run: |\n', 1)[1].split('\n - ', 1)[0] + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + python = root / 'python' + (root / 'private').mkdir() + python.write_text('#!/bin/bash\nprintf "%s\\n" "$@"\n') + python.chmod(0o700) + command = command.replace('python3 -I candidate/scripts/hevc_threading.py', f'"{python}"') + for dependencies, rows in itertools.product(('false', 'true'), repeat=2): + env = dict(PATH='/usr/bin:/bin', RUNNER_TEMP=temp, CANDIDATE='a' * 40, + DEPENDENCIES_ONLY=dependencies, ROWS_ONLY=rows) + result = subprocess.run(['bash', '-c', guard], env=env, capture_output=True, timeout=10) + self.assertEqual(result.returncode == 0, (dependencies, rows) != ('true', 'true')) + if result.returncode: + continue + bash = '/opt/homebrew/bin/bash' if pathlib.Path('/opt/homebrew/bin/bash').exists() else 'bash' + subprocess.run([bash, '-c', command], env=env, capture_output=True, check=True, timeout=10) + args = (root / 'private/diagnostics.log').read_text().splitlines() + expected = ['diagnostics', '--native', str(pathlib.Path.cwd() / 'candidate/target/release/examples/reference'), + '--oracle-bin', str(root / 'oracle'), '--out', str(root / 'private'), '--candidate', 'a' * 40] + expected += ['--rows-only'] if rows == 'true' else ['--dependencies-only'] if dependencies == 'true' else [] + self.assertEqual(args, expected) + if __name__ == '__main__': unittest.main()