From d9d4d896379e6caf45603e497a0caf9475a942c9 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 30 Oct 2026 18:43:05 +0100 Subject: [PATCH] fix(stream): fall back by codec when a decoder refuses the stream A browser whose isConfigSupported says yes to prefer-hardware while configure fails rebuilt with that same setting twice a second, for good: a black screen. A rebuild now moves past the acceleration that failed before its first frame, down to prefer-software, and a configure that throws instead of calling error reaches the same rebuild. A codec this browser decodes under no acceleration ended the stream even when another codec was on offer. The engine now drops it from the offer and reconnects, once per codec per stream the player starts, and says which codec took over, including when that overrides the codec setting. H.264 can leave the offer too, so a pin on it can fall back. The overlay said WebCodecs whether the frames came from hardware or software. It now names the acceleration once a decoder has produced a frame. Fixes the fallback half of #59. --- packages/stream/build.sh | 2 +- packages/stream/rust/session.rs | 34 +++++++-- packages/stream/src/emscripten.d.ts | 2 + packages/stream/src/engine.test.ts | 36 ++++++++++ packages/stream/src/engine.ts | 63 +++++++++++++---- packages/stream/src/video.test.ts | 105 +++++++++++++++++++++++----- packages/stream/src/video.ts | 73 ++++++++++++------- 7 files changed, 251 insertions(+), 64 deletions(-) diff --git a/packages/stream/build.sh b/packages/stream/build.sh index e2d7d0a..7f66558 100755 --- a/packages/stream/build.sh +++ b/packages/stream/build.sh @@ -85,7 +85,7 @@ link=( -C link-arg=-sSTACK_SIZE=4MB # Every `_pf_*` the page calls, the ring's included: `src/carrier/feed.ts` writes datagrams # into it. - -C link-arg=-sEXPORTED_FUNCTIONS=_main,_pf_start,_pf_frame,_pf_key,_pf_console_text,_pf_console_pointer,_pf_console_pad,_pf_console_phase,_pf_console_state,_pf_console_leave_out,_pf_console_push,_pf_console_art,_pf_rx_base,_pf_rx_stride,_pf_rx_claim,_pf_rx_commit,_pf_rx_dropped,_pf_net_blast,_pf_net_drain,_pf_wt_close,_pf_wt_ctl_open,_pf_ctl_recv,_pf_uni_recv,_pf_session_hello,_pf_session_codecs,_pf_session_pump,_pf_session_phase,_pf_session_frames,_pf_device_init,_pf_device_set,_pf_device_sign,_pf_device_fingerprint_hex,_pf_cred_phase,_pf_cred_sign_ptr,_pf_cred_sign_len,_pf_cred_signed,_pf_pair_begin,_pf_input,_pf_gamepad,_pf_gamepad_arrival,_pf_gamepad_remove,_pf_audio_frames,_pf_audio_lost,_pf_session_audio_channels,_pf_session_audio,_pf_session_audio_layout,_pf_mic_send,_pf_session_access_seq,_pf_session_access_grants,_pf_session_access_secs,_pf_session_reset,_pf_session_reconfigure,_pf_hud_decoded,_pf_hud_presented,_pf_gate_decoded,_pf_gate_no_output,_pf_request_keyframe,_pf_hud_drain,_pf_hud_quality,_pf_hud_text,_pf_hud_text_ptr,_pf_session_host_caps,_pf_cursor_render,_pf_cursor_serial,_pf_cursor_flags,_malloc,_free + -C link-arg=-sEXPORTED_FUNCTIONS=_main,_pf_start,_pf_frame,_pf_key,_pf_console_text,_pf_console_pointer,_pf_console_pad,_pf_console_phase,_pf_console_state,_pf_console_leave_out,_pf_console_push,_pf_console_art,_pf_rx_base,_pf_rx_stride,_pf_rx_claim,_pf_rx_commit,_pf_rx_dropped,_pf_net_blast,_pf_net_drain,_pf_wt_close,_pf_wt_ctl_open,_pf_ctl_recv,_pf_uni_recv,_pf_session_hello,_pf_session_codecs,_pf_session_pump,_pf_session_phase,_pf_session_frames,_pf_device_init,_pf_device_set,_pf_device_sign,_pf_device_fingerprint_hex,_pf_cred_phase,_pf_cred_sign_ptr,_pf_cred_sign_len,_pf_cred_signed,_pf_pair_begin,_pf_input,_pf_gamepad,_pf_gamepad_arrival,_pf_gamepad_remove,_pf_audio_frames,_pf_audio_lost,_pf_session_audio_channels,_pf_session_audio,_pf_session_audio_layout,_pf_mic_send,_pf_session_access_seq,_pf_session_access_grants,_pf_session_access_secs,_pf_session_reset,_pf_session_reconfigure,_pf_hud_decoded,_pf_hud_presented,_pf_gate_decoded,_pf_gate_no_output,_pf_request_keyframe,_pf_hud_drain,_pf_hud_quality,_pf_video_decoder,_pf_hud_text,_pf_hud_text_ptr,_pf_session_host_caps,_pf_cursor_render,_pf_cursor_serial,_pf_cursor_flags,_malloc,_free # `stringToNewUTF8` is how the page hands a string across; `HEAPU8` is emscripten's view of # wasm memory, which `src/carrier/feed.ts` writes datagrams into. -C link-arg=-sEXPORTED_RUNTIME_METHODS=stringToNewUTF8,HEAPU8 diff --git a/packages/stream/rust/session.rs b/packages/stream/rust/session.rs index af025c2..9378a1b 100644 --- a/packages/stream/rust/session.rs +++ b/packages/stream/rust/session.rs @@ -90,6 +90,9 @@ struct Client { depth: u8, hdr: bool, full_range: bool, + /// How the page's decoder runs, as `pf_video_decoder` last said: `0` unknown, `1` hardware, + /// `2` software. The overlay's decoder line names it. + decoder_accel: u32, /// Access units delivered, so the page can tell "connected" from "streaming". frames: u64, /// The stats overlay's window. No clock handshake runs in a browser, so its offset stays @@ -147,6 +150,7 @@ impl Client { depth: 8, hdr: false, full_range: false, + decoder_accel: 0, frames: 0, hud: new_hud(), receipts: VecDeque::with_capacity(RECEIPTS), @@ -214,6 +218,7 @@ pub extern "C" fn pf_session_reset() { c.inbox.clear(); c.frames = 0; c.codec = 0; + c.decoder_accel = 0; c.width = 0; c.height = 0; c.asked = None; @@ -323,14 +328,17 @@ thread_local! { static PREFERRED: std::cell::Cell = const { std::cell::Cell::new(0) }; } -/// The codecs this browser decodes (`CODEC_*` bits), from the page's `isConfigSupported` probe, -/// whether it presents BT.2020 PQ, and the one codec bit to ask for first (`0`: the host's -/// order). H.264 is always kept: a GPU-less host has nothing else. A preference this browser -/// cannot decode is dropped rather than sent. +/// The codecs this browser decodes (`CODEC_*` bits), from the page's `isConfigSupported` probe +/// less those it failed to decode, whether it presents BT.2020 PQ, and the one codec bit to ask +/// for first (`0`: the host's order). An empty mask offers H.264: the host reads `0` as an old +/// HEVC-only client. A preference this browser cannot decode is dropped rather than sent. #[unsafe(no_mangle)] pub extern "C" fn pf_session_codecs(mask: u32, hdr: u32, preferred: u32) { - let supported = punktfunk_core::quic::CODEC_HEVC | punktfunk_core::quic::CODEC_AV1; - let offered = (mask as u8 & supported) | punktfunk_core::quic::CODEC_H264; + use punktfunk_core::quic::{CODEC_AV1, CODEC_H264, CODEC_HEVC}; + let offered = match mask as u8 & (CODEC_H264 | CODEC_HEVC | CODEC_AV1) { + 0 => CODEC_H264, + m => m, + }; CODECS.with(|c| c.set(offered)); HDR.with(|h| h.set(hdr != 0)); PREFERRED.with(|p| p.set(preferred as u8 & offered)); @@ -944,6 +952,13 @@ pub extern "C" fn pf_hud_decoded(pts_us: f64, decoded_ms: f64) { }); } +/// How the page's decoder runs, once it has produced a frame: `0` unknown, `1` hardware, +/// `2` software. +#[unsafe(no_mangle)] +pub extern "C" fn pf_video_decoder(accel: u32) { + CLIENT.with(|c| c.borrow_mut().decoder_accel = accel); +} + /// The page drew the frame stamped `pts_us`: decoded at `decoded_ms`, drawn at `presented_ms`. #[unsafe(no_mangle)] pub extern "C" fn pf_hud_presented(pts_us: f64, decoded_ms: f64, presented_ms: f64) { @@ -976,7 +991,12 @@ pub extern "C" fn pf_hud_drain(skipped: u32) { snap.auto_rate = c.bitrate_kbps == 0; (snap.width, snap.height, snap.refresh_hz) = (c.width, c.height, c.fps); snap.codec = hud::codec_label(c.codec).into(); - snap.decoder = "WebCodecs".into(); + snap.decoder = match c.decoder_accel { + 1 => "WebCodecs, hardware", + 2 => "WebCodecs, software", + _ => "WebCodecs", + } + .into(); let rtt = crate::transport::rtt_us(); snap.rtt_us = (rtt > 0).then_some(rtt); hud::quality(&snap, &mut c.quality); diff --git a/packages/stream/src/emscripten.d.ts b/packages/stream/src/emscripten.d.ts index 2a65196..1d34ead 100644 --- a/packages/stream/src/emscripten.d.ts +++ b/packages/stream/src/emscripten.d.ts @@ -108,6 +108,8 @@ export interface PunktfunkModule { _pf_hud_drain(skipped: number): void; /** The connection dot as of the last drain: `0` starting, `1` good, `2` fair, `3` poor. */ _pf_hud_quality(): number; + /** How the decoder that works runs, for the overlay: `0` unknown, `1` hardware, `2` software. */ + _pf_video_decoder?(accel: number): void; /** Format the last window at `tier` (0–3) in the Advanced vocabulary or not; byte length. */ _pf_hud_text(tier: number, advanced: number): number; /** Where `_pf_hud_text` left its bytes: `\t\n` per line. */ diff --git a/packages/stream/src/engine.test.ts b/packages/stream/src/engine.test.ts index 721590a..29f8140 100644 --- a/packages/stream/src/engine.test.ts +++ b/packages/stream/src/engine.test.ts @@ -249,3 +249,39 @@ test("the connection dot is the wire's rating, read once a second", () => { assert.equal(second(3), "poor"); assert.equal(second(0), "starting"); }); + +test("a codec this browser can't decode gives way to another on offer, once each", () => { + const { engine, mod } = engineIn("ready"); + const offers: number[] = []; + mod._pf_session_codecs = (mask: number) => offers.push(mask); + Object.assign(engine, { codecs: { mask: 0x03, tenBit: false } }); + const undecodable = (codec: number) => (engine as unknown as { undecodable(codec: number): void }).undecodable(codec); + engine.configure({ codec: "hevc" }); + engine.startStream({ width: 1280, height: 720 }); + const before = dialled.length; + undecodable(0x02); + assert.equal(engine.current.kind, "starting", "no error while H.264 is left"); + assert.deepEqual(offers, [0x03, 0x01], "the same stream, offered without HEVC"); + assert.equal(dialled.length, before + 1); + + // The session that took over names its codec once `Welcome` has. + Object.assign(mod, { _pf_session_phase: () => 2, _pf_session_audio_channels: () => 0, _pf_hud_text: () => 0 }); + const snapshot = () => ({ codec: 0x01, width: 0, height: 0, decoded: 0, dropped: 0, uploadMs: 0, backend: null }); + Object.assign(engine, { video: { snapshot, close() {} } }); + (engine as unknown as { updateSession(): void }).updateSession(); + const live = engine.current as import("./engine.ts").EngineState; + assert.equal( + live.kind === "streaming" && live.stats.launchNotice, + "This browser can't decode HEVC, so the stream uses H.264, ignoring the Video codec setting.", + ); + + // H.264 fails too: nothing is left, and HEVC is not offered again. + undecodable(0x01); + assert.equal(engine.current.kind, "error"); + assert.deepEqual(offers, [0x03, 0x01]); + + // A stream the player starts offers every codec again. + (engine as unknown as { set(s: unknown): void }).set({ kind: "ready", origin: ORIGIN }); + engine.startStream({ width: 1280, height: 720 }); + assert.equal(offers.at(-1), 0x03); +}); diff --git a/packages/stream/src/engine.ts b/packages/stream/src/engine.ts index e9e1e88..8776737 100644 --- a/packages/stream/src/engine.ts +++ b/packages/stream/src/engine.ts @@ -197,6 +197,8 @@ const TUNABLE_DEFAULTS: TunableOptions = { /** `CODEC_*` bits, as `Hello::preferred_codec` names one. */ const CODEC_BIT: Record = { auto: 0, h264: 0x01, hevc: 0x02, av1: 0x04 }; +/** A `CODEC_*` bit as the player reads it. */ +const CODEC_NAME: Record = { 0x01: "H.264", 0x02: "HEVC", 0x04: "AV1" }; export interface EngineOptions { /** The lower canvas: decoded video goes here and nowhere else. */ @@ -298,6 +300,11 @@ export class Engine { private streamed: StreamOptions | null = null; /** The stream is an SDR restart, which the next session says once. */ private sdrNotice = false; + /** Codec bits this browser failed to decode since the player started the stream. The offer + * leaves them out, so two failing codecs can't alternate. */ + private refused = 0; + /** The codec the stream fell back from, until the session that replaced it names its own. */ + private fellBackFrom = 0; /** A request for access, sent as `Hello` when the next control stream opens. Survives `reset`, * as `pendingPin` does, for the reconnect that clears a stale pairing first. */ private pendingKnock: StreamOptions | null = null; @@ -598,10 +605,9 @@ export class Engine { */ startStream(opts: StreamOptions): void { if (!this.origin || this.state.kind !== "ready") return; - // HDR is offered per stream: the window may have moved to another display, and the plane - // may have been switched to WebGL2, since the last one. - const hdr = this.tunable.hdr && this.codecs.tenBit && hdrDisplay(this.tunable.videoBackend, this.gpu); - this.mod._pf_session_codecs?.(this.codecs.mask, hdr ? 1 : 0, CODEC_BIT[this.tunable.codec]); + this.refused = 0; + this.fellBackFrom = 0; + this.offer(false); this.set({ kind: "starting", origin: this.origin }); this.streamed = opts; this.pendingStream = opts; @@ -609,27 +615,47 @@ export class Engine { } /** - * The browser converted an HDR frame itself, which nothing here can undo: the same stream again - * in SDR, on a fresh connection. The title keeps running on the host; `hdrDisplay` now says no - * for this browser build, so this happens once. + * What the next `Hello` offers: this browser's codecs less those it refused, the player's pick + * first, and HDR unless `sdr`. HDR is asked per stream: the window may have moved to another + * display, and the plane may have been switched to WebGL2, since the last one. */ - private restartInSdr(): void { + private offer(sdr: boolean): void { + const hdr = !sdr && this.tunable.hdr && this.codecs.tenBit && hdrDisplay(this.tunable.videoBackend, this.gpu); + this.mod._pf_session_codecs?.(this.codecs.mask & ~this.refused, hdr ? 1 : 0, CODEC_BIT[this.tunable.codec]); + } + + /** The same stream again on a fresh connection, offering what `offer` allows now. The title + * keeps running on the host. False when no stream is up to restart. */ + private restart(sdr: boolean): boolean { const origin = this.origin; const opts = this.streamed; - if (!origin || !opts || (this.state.kind !== "streaming" && this.state.kind !== "starting")) return; + if (!origin || !opts || (this.state.kind !== "streaming" && this.state.kind !== "starting")) return false; this.stopSession(); - this.mod._pf_session_codecs?.(this.codecs.mask, 0, CODEC_BIT[this.tunable.codec]); - this.sdrNotice = true; + this.offer(sdr); this.set({ kind: "starting", origin }); this.pendingStream = opts; this.dialBox(); + return true; } - /** The browser decodes this stream neither in hardware nor in software. The title keeps - * running on the host, as on a Leave. */ - private undecodable(): void { + /** The browser converted an HDR frame itself, which nothing here can undo: the same stream in + * SDR. `hdrDisplay` now says no for this browser build, so this happens once. */ + private restartInSdr(): void { + if (this.restart(true)) this.sdrNotice = true; + } + + /** The browser decodes the stream's codec under no acceleration. Another codec on offer takes + * over, once per codec per stream the player starts. With none left the stream ends, and the + * title keeps running on the host, as on a Leave. */ + private undecodable(codec: number): void { const origin = this.origin; if (!origin || (this.state.kind !== "streaming" && this.state.kind !== "starting")) return; + const fresh = (this.refused & codec) === 0; + this.refused |= codec; + if (fresh && (this.codecs.mask & ~this.refused) !== 0 && this.restart(false)) { + this.fellBackFrom = codec; + return; + } this.stopSession(); // `error` before the close, so the close it answers with finds nothing left to report. this.set({ @@ -683,7 +709,7 @@ export class Engine { this.opts.videoCanvas, this.tunable.videoBackend, () => setTimeout(() => this.restartInSdr(), 0), - () => this.undecodable(), + (codec) => this.undecodable(codec), ); this.video.attach(); this.tier = this.tunable.statsTier; @@ -1309,6 +1335,13 @@ export class Engine { } const now = performance.now(); + // A codec fallback names the codec that took over, which only `Welcome` says. + if (this.fellBackFrom && v?.codec) { + const pinned = this.refused & CODEC_BIT[this.tunable.codec] ? ", ignoring the Video codec setting" : ""; + const text = `This browser can't decode ${CODEC_NAME[this.fellBackFrom]}, so the stream uses ${CODEC_NAME[v.codec]}${pinned}.`; + this.launchNotice = { text, at: now }; + this.fellBackFrom = 0; + } this.readAccess(now); const frames = this.mod._pf_session_frames(); let second = false; diff --git a/packages/stream/src/video.test.ts b/packages/stream/src/video.test.ts index 3785df1..97a9feb 100644 --- a/packages/stream/src/video.test.ts +++ b/packages/stream/src/video.test.ts @@ -1,12 +1,23 @@ -// The decoder's recovery, against a stand-in WebCodecs that acts like a browser with no hardware -// decoder: it refuses `prefer-hardware` after `configure`, and `close()` on a closed codec throws. +// The decoder's recovery, against a stand-in WebCodecs that acts like a browser told what it +// decodes: `configure` refuses the rest after the fact, `isConfigSupported` may claim more than +// `configure` takes, and `close()` on a closed codec throws. import { test } from "node:test"; import assert from "node:assert/strict"; import type { PunktfunkModule } from "./emscripten.ts"; -let decodes: (accel: HardwareAcceleration) => boolean = () => true; +interface Browser { + /** What `configure` takes. */ + decodes: (accel: HardwareAcceleration) => boolean; + /** What `isConfigSupported` says yes to: `decodes`, unless it lies. */ + claims?: (accel: HardwareAcceleration) => boolean; + /** What `configure` throws on rather than calling `error`. */ + throws?: (accel: HardwareAcceleration) => boolean; +} +let browser: Browser = { decodes: () => true }; let decoders: FakeDecoder[] = []; +/** Every acceleration `isConfigSupported` was asked about, in order. */ +let probed: HardwareAcceleration[] = []; class FakeDecoder { state: CodecState = "unconfigured"; @@ -17,12 +28,16 @@ class FakeDecoder { decoders.push(this); } static async isConfigSupported(config: VideoDecoderConfig) { - return { supported: decodes(config.hardwareAcceleration ?? "no-preference"), config }; + const accel = config.hardwareAcceleration ?? "no-preference"; + probed.push(accel); + return { supported: (browser.claims ?? browser.decodes)(accel), config }; } configure(config: VideoDecoderConfig) { + const accel = config.hardwareAcceleration ?? "no-preference"; + if (browser.throws?.(accel)) throw new TypeError("Unsupported codec."); this.config = config; this.state = "configured"; - if (decodes(config.hardwareAcceleration ?? "no-preference")) return; + if (browser.decodes(accel)) return; this.state = "closed"; setTimeout(() => this.init.error(new DOMException("Unsupported configuration.", "OperationError")), 0); } @@ -30,6 +45,15 @@ class FakeDecoder { if (this.state === "closed") throw new DOMException("Cannot call 'close' on a closed codec.", "InvalidStateError"); this.state = "closed"; } + /** A decoded frame out. */ + emit() { + this.init.output({ timestamp: 0, close() {} } as unknown as VideoFrame); + } + /** A working decoder fails, as one does when the GPU goes away. */ + fail() { + this.state = "closed"; + this.init.error(new DOMException("Decoding error.", "EncodingError")); + } } globalThis.VideoDecoder = FakeDecoder as unknown as typeof VideoDecoder; const { VideoPipe } = await import("./video.ts"); @@ -37,38 +61,87 @@ const { VideoPipe } = await import("./video.ts"); /** A pipe with a stand-in plane, told the stream is H.264 at 1080p. */ function stream() { decoders = []; + probed = []; const asked: string[] = []; - let undecodable = 0; - const mod = { _pf_request_keyframe: () => asked.push("keyframe") } as unknown as PunktfunkModule; - const pipe = new VideoPipe(mod, {} as HTMLCanvasElement, "auto", undefined, () => undecodable++); + const refused: number[] = []; + const kinds: number[] = []; + const mod = { + _pf_request_keyframe: () => asked.push("keyframe"), + _pf_hud_decoded: () => {}, + _pf_gate_decoded: () => 1, + _pf_video_decoder: (accel: number) => kinds.push(accel), + } as unknown as PunktfunkModule; + const pipe = new VideoPipe(mod, {} as HTMLCanvasElement, "auto", undefined, (codec) => refused.push(codec)); // A real plane needs a GPU; the pipe only configures and destroys this one. - Object.assign(pipe, { plane: { configure() {}, setDynamicRange() {}, destroy() {} } }); + const plane = { configure() {}, setDynamicRange() {}, destroy() {}, takeUploadStats: () => ({ frames: 0 }) }; + Object.assign(pipe, { plane }); pipe.attach(); mod.__pfOnVideoConfig!(1, 1920, 1080, 8, false, false); - return { pipe, asked, undecodable: () => undecodable }; + return { pipe, asked, refused, kinds }; } const settle = () => new Promise((r) => setTimeout(r, 20)); +/** Wait for `done`: each rebuild after the first waits out the half-second throttle. */ +const until = async (done: () => boolean) => { + for (let i = 0; i < 150 && !done(); i++) await settle(); +}; test("a browser without a hardware decoder streams on the one it has", async () => { - decodes = (accel) => accel !== "prefer-hardware"; - const { pipe, asked, undecodable } = stream(); + browser = { decodes: (accel) => accel !== "prefer-hardware" }; + const { pipe, asked, refused } = stream(); await settle(); assert.equal(decoders.length, 2); assert.equal(decoders[1]!.config!.hardwareAcceleration, "no-preference"); assert.equal(decoders[1]!.state, "configured"); assert.deepEqual(asked, ["keyframe"], "the fresh decoder resumes on an IDR"); - assert.equal(undecodable(), 0); + assert.deepEqual(refused, []); pipe.close(); assert.equal(decoders[1]!.state, "closed"); }); test("a stream the browser cannot decode ends, and the pipe still closes", async () => { - decodes = () => false; - const { pipe, asked, undecodable } = stream(); + browser = { decodes: () => false }; + const { pipe, asked, refused } = stream(); await settle(); assert.equal(decoders.length, 1, "no decoder is built for a configuration the browser refuses"); - assert.equal(undecodable(), 1); + assert.deepEqual(refused, [1], "the pipe names the codec it could not decode"); assert.deepEqual(asked, []); assert.doesNotThrow(() => pipe.close()); }); + +test("a browser that claims what it then refuses moves down to software, then gives up", async () => { + browser = { decodes: () => false, claims: () => true }; + const { pipe, refused } = stream(); + await until(() => refused.length > 0); + assert.deepEqual( + decoders.map((d) => d.config!.hardwareAcceleration), + ["prefer-hardware", "no-preference", "prefer-software"], + ); + assert.deepEqual(probed, ["no-preference", "prefer-software"], "hardware is never asked about again"); + assert.deepEqual(refused, [1]); + pipe.close(); +}); + +test("a configure that throws reaches the rebuild like one that errors", async () => { + browser = { decodes: () => true, throws: (accel) => accel === "prefer-hardware" }; + const { pipe, asked, refused } = stream(); + await until(() => decoders.length === 2); + assert.equal(decoders[1]!.config!.hardwareAcceleration, "no-preference"); + assert.equal(decoders[1]!.state, "configured"); + assert.equal(pipe.snapshot().errors, 1); + assert.deepEqual(asked, ["keyframe"]); + assert.deepEqual(refused, []); + pipe.close(); +}); + +test("the first frame tells the overlay how it decodes, and a later error keeps that", async () => { + browser = { decodes: () => true }; + const { pipe, kinds } = stream(); + decoders[0]!.emit(); + decoders[0]!.emit(); + assert.deepEqual(kinds, [1], "hardware, said once"); + decoders[0]!.fail(); + await until(() => decoders.length === 2); + assert.equal(decoders[1]!.config!.hardwareAcceleration, "prefer-hardware", "a decoder that worked keeps its acceleration"); + pipe.close(); +}); diff --git a/packages/stream/src/video.ts b/packages/stream/src/video.ts index dbf4069..a013087 100644 --- a/packages/stream/src/video.ts +++ b/packages/stream/src/video.ts @@ -143,6 +143,8 @@ const PQ_PASSTHROUGH: VideoColorSpaceInit = { }; export interface VideoStats { + /** The wire codec `Welcome` named (`CODEC_*`); `0` before it. */ + codec: number; /** Access units handed to the decoder. */ submitted: number; /** Frames the decoder produced. A gap against `submitted` is loss or a decoder still filling. */ @@ -159,6 +161,14 @@ export interface VideoStats { backend: "webgpu" | "webgl2" | null; } +/** The order a decoder is tried in. One that fails before its first frame moves to the next: + * `isConfigSupported` can say yes to a configuration `configure` then refuses. */ +const ACCELERATIONS: HardwareAcceleration[] = ["prefer-hardware", "no-preference", "prefer-software"]; + +/** What `_pf_video_decoder` is told once a decoder works: `1` hardware, `2` software, `0` the + * browser's own pick, which it does not reveal. */ +const DECODER_KIND: Record = { "prefer-hardware": 1, "no-preference": 0, "prefer-software": 2 }; + /** * Which plane to draw on. * @@ -206,20 +216,22 @@ export class VideoPipe { /** This decoder has asked the host for that key frame. */ private askedKey = false; private closed = false; - private codec = 0; /** `Welcome`'s bit depth, HDR and range, for the decoder string, the plane and `PQ_PASSTHROUGH`. */ private depth = 8; private hdr = false; private fullRange = false; /** The first frame of an HDR configure has been checked for `PQ_PASSTHROUGH`. */ private hdrChecked = false; - /** `no-preference` once the browser refused `prefer-hardware` for this stream. */ + /** Where in `ACCELERATIONS` this stream's decoder is. */ private accel: HardwareAcceleration = "prefer-hardware"; + /** The decoder has produced a frame, so its acceleration works here. */ + private stuck = false; private rebuilding = false; private lastRebuildMs = -Infinity; /** When `pending` left the decoder, for the overlay's display stage. */ private pendingDecodedMs = 0; private stats: VideoStats = { + codec: 0, submitted: 0, decoded: 0, dropped: 0, @@ -237,8 +249,8 @@ export class VideoPipe { private readonly prefer: "auto" | "webgl2" | "webgpu" = "auto", /** The browser converted an HDR frame itself: this stream cannot be shown right. */ private readonly onHdrRefused?: () => void, - /** The browser decodes this stream neither in hardware nor in software. */ - private readonly onUndecodable?: () => void, + /** The browser decodes this stream's codec (`CODEC_*`) under no acceleration. */ + private readonly onUndecodable?: (codec: number) => void, ) {} /** Install the callbacks the glue calls. Idempotent, so a reconnect can call it again. */ @@ -247,12 +259,15 @@ export class VideoPipe { this.depth = depth; this.hdr = hdr; this.fullRange = fullRange; + // A new configuration from the host starts at hardware again; only a rebuild moves down. + this.accel = "prefer-hardware"; void this.configure(codec, width, height); }; this.mod.__pfOnAccessUnit = (data, ptsUs, key, flags) => this.submit(data, ptsUs, key, flags); } private async configure(codec: number, width: number, height: number): Promise { + this.stats.codec = codec; this.stats.width = width; this.stats.height = height; if (!this.plane) { @@ -266,20 +281,23 @@ export class VideoPipe { this.plane.configure(width, height); this.plane.setDynamicRange(this.hdr ? "high" : "standard"); - this.codec = codec; this.closeDecoder(); this.meta.clear(); this.needKey = true; this.askedKey = false; - const decoder = new VideoDecoder({ - output: (frame) => this.onFrame(frame), - error: (e) => { - this.stats.errors++; - console.error("punktfunk: decoder", e); - void this.rebuild(); - }, - }); - decoder.configure(this.config(this.accel)); + this.stuck = false; + const fail = (e: unknown) => { + this.stats.errors++; + console.error("punktfunk: decoder", e); + void this.rebuild(); + }; + const decoder = new VideoDecoder({ output: (frame) => this.onFrame(frame), error: fail }); + // A configuration the engine rejects outright throws here rather than calling `error`. + try { + decoder.configure(this.config(this.accel)); + } catch (e) { + fail(e); + } this.hdrChecked = false; this.decoder = decoder; } @@ -288,7 +306,7 @@ export class VideoPipe { * host sends parameter sets inline with every key frame, which lets a browser join mid-stream. */ private config(hardwareAcceleration: HardwareAcceleration): VideoDecoderConfig { return { - codec: codecString(this.codec, this.depth), + codec: codecString(this.stats.codec, this.depth), codedWidth: this.stats.width, codedHeight: this.stats.height, optimizeForLatency: true, @@ -340,30 +358,30 @@ export class VideoPipe { } } - // A decoder that errored is closed for good. Build a fresh one the way the browser still takes - // this stream — one without a hardware decoder for it refuses `prefer-hardware` — and resume on - // the next IDR; the gate holds the last picture meanwhile. At most twice a second. + // A decoder that errored is closed for good. Build a fresh one and resume on the next IDR; the + // gate holds the last picture meanwhile. One that never produced a frame failed its + // acceleration, so the next starts past it in `ACCELERATIONS`. At most twice a second. private async rebuild(): Promise { if (this.rebuilding || !this.plane) return; this.rebuilding = true; const wait = this.lastRebuildMs + 500 - performance.now(); if (wait > 0) await new Promise((r) => setTimeout(r, wait)); - const accel = await this.acceleration(); + const accel = await this.acceleration(ACCELERATIONS.indexOf(this.accel) + (this.stuck ? 0 : 1)); this.rebuilding = false; // Closed meanwhile, or a reconfigure already built a working decoder. if (!this.plane || this.decoder?.state === "configured") return; this.lastRebuildMs = performance.now(); - if (!accel) return this.onUndecodable?.(); + if (!accel) return this.onUndecodable?.(this.stats.codec); this.accel = accel; - void this.configure(this.codec, this.stats.width, this.stats.height); + void this.configure(this.stats.codec, this.stats.width, this.stats.height); this.mod._pf_request_keyframe(); this.askedKey = true; } - /** The first of `prefer-hardware` and `no-preference` the browser takes for this stream, or - * null when it decodes it neither way. */ - private async acceleration(): Promise { - for (const accel of ["prefer-hardware", "no-preference"] as const) { + /** The first of `ACCELERATIONS` from index `from` on that the browser says it takes for this + * stream, or null when none is left. */ + private async acceleration(from: number): Promise { + for (const accel of ACCELERATIONS.slice(from)) { try { if ((await VideoDecoder.isConfigSupported(this.config(accel))).supported) return accel; } catch { @@ -378,6 +396,11 @@ export class VideoPipe { // than queued. Presenting happens on the page's own rAF, which is where pacing belongs. private onFrame(frame: VideoFrame): void { this.stats.decoded++; + // The first frame proves the acceleration: the overlay names it, and a later error keeps it. + if (!this.stuck) { + this.stuck = true; + this.mod._pf_video_decoder?.(DECODER_KIND[this.accel]); + } // An engine that ignores the override hands over light it has already converted, which no // shader here can undo. Said once; the engine streams SDR instead. if (this.hdr && !this.hdrChecked) {