diff --git a/packages/stream/build.sh b/packages/stream/build.sh index e2d7d0a..7f66558 100755 --- a/packages/stream/build.sh +++ b/packages/stream/build.sh @@ -85,7 +85,7 @@ link=( -C link-arg=-sSTACK_SIZE=4MB # Every `_pf_*` the page calls, the ring's included: `src/carrier/feed.ts` writes datagrams # into it. - -C link-arg=-sEXPORTED_FUNCTIONS=_main,_pf_start,_pf_frame,_pf_key,_pf_console_text,_pf_console_pointer,_pf_console_pad,_pf_console_phase,_pf_console_state,_pf_console_leave_out,_pf_console_push,_pf_console_art,_pf_rx_base,_pf_rx_stride,_pf_rx_claim,_pf_rx_commit,_pf_rx_dropped,_pf_net_blast,_pf_net_drain,_pf_wt_close,_pf_wt_ctl_open,_pf_ctl_recv,_pf_uni_recv,_pf_session_hello,_pf_session_codecs,_pf_session_pump,_pf_session_phase,_pf_session_frames,_pf_device_init,_pf_device_set,_pf_device_sign,_pf_device_fingerprint_hex,_pf_cred_phase,_pf_cred_sign_ptr,_pf_cred_sign_len,_pf_cred_signed,_pf_pair_begin,_pf_input,_pf_gamepad,_pf_gamepad_arrival,_pf_gamepad_remove,_pf_audio_frames,_pf_audio_lost,_pf_session_audio_channels,_pf_session_audio,_pf_session_audio_layout,_pf_mic_send,_pf_session_access_seq,_pf_session_access_grants,_pf_session_access_secs,_pf_session_reset,_pf_session_reconfigure,_pf_hud_decoded,_pf_hud_presented,_pf_gate_decoded,_pf_gate_no_output,_pf_request_keyframe,_pf_hud_drain,_pf_hud_quality,_pf_hud_text,_pf_hud_text_ptr,_pf_session_host_caps,_pf_cursor_render,_pf_cursor_serial,_pf_cursor_flags,_malloc,_free + -C link-arg=-sEXPORTED_FUNCTIONS=_main,_pf_start,_pf_frame,_pf_key,_pf_console_text,_pf_console_pointer,_pf_console_pad,_pf_console_phase,_pf_console_state,_pf_console_leave_out,_pf_console_push,_pf_console_art,_pf_rx_base,_pf_rx_stride,_pf_rx_claim,_pf_rx_commit,_pf_rx_dropped,_pf_net_blast,_pf_net_drain,_pf_wt_close,_pf_wt_ctl_open,_pf_ctl_recv,_pf_uni_recv,_pf_session_hello,_pf_session_codecs,_pf_session_pump,_pf_session_phase,_pf_session_frames,_pf_device_init,_pf_device_set,_pf_device_sign,_pf_device_fingerprint_hex,_pf_cred_phase,_pf_cred_sign_ptr,_pf_cred_sign_len,_pf_cred_signed,_pf_pair_begin,_pf_input,_pf_gamepad,_pf_gamepad_arrival,_pf_gamepad_remove,_pf_audio_frames,_pf_audio_lost,_pf_session_audio_channels,_pf_session_audio,_pf_session_audio_layout,_pf_mic_send,_pf_session_access_seq,_pf_session_access_grants,_pf_session_access_secs,_pf_session_reset,_pf_session_reconfigure,_pf_hud_decoded,_pf_hud_presented,_pf_gate_decoded,_pf_gate_no_output,_pf_request_keyframe,_pf_hud_drain,_pf_hud_quality,_pf_video_decoder,_pf_hud_text,_pf_hud_text_ptr,_pf_session_host_caps,_pf_cursor_render,_pf_cursor_serial,_pf_cursor_flags,_malloc,_free # `stringToNewUTF8` is how the page hands a string across; `HEAPU8` is emscripten's view of # wasm memory, which `src/carrier/feed.ts` writes datagrams into. -C link-arg=-sEXPORTED_RUNTIME_METHODS=stringToNewUTF8,HEAPU8 diff --git a/packages/stream/rust/session.rs b/packages/stream/rust/session.rs index af025c2..9378a1b 100644 --- a/packages/stream/rust/session.rs +++ b/packages/stream/rust/session.rs @@ -90,6 +90,9 @@ struct Client { depth: u8, hdr: bool, full_range: bool, + /// How the page's decoder runs, as `pf_video_decoder` last said: `0` unknown, `1` hardware, + /// `2` software. The overlay's decoder line names it. + decoder_accel: u32, /// Access units delivered, so the page can tell "connected" from "streaming". frames: u64, /// The stats overlay's window. No clock handshake runs in a browser, so its offset stays @@ -147,6 +150,7 @@ impl Client { depth: 8, hdr: false, full_range: false, + decoder_accel: 0, frames: 0, hud: new_hud(), receipts: VecDeque::with_capacity(RECEIPTS), @@ -214,6 +218,7 @@ pub extern "C" fn pf_session_reset() { c.inbox.clear(); c.frames = 0; c.codec = 0; + c.decoder_accel = 0; c.width = 0; c.height = 0; c.asked = None; @@ -323,14 +328,17 @@ thread_local! { static PREFERRED: std::cell::Cell = const { std::cell::Cell::new(0) }; } -/// The codecs this browser decodes (`CODEC_*` bits), from the page's `isConfigSupported` probe, -/// whether it presents BT.2020 PQ, and the one codec bit to ask for first (`0`: the host's -/// order). H.264 is always kept: a GPU-less host has nothing else. A preference this browser -/// cannot decode is dropped rather than sent. +/// The codecs this browser decodes (`CODEC_*` bits), from the page's `isConfigSupported` probe +/// less those it failed to decode, whether it presents BT.2020 PQ, and the one codec bit to ask +/// for first (`0`: the host's order). An empty mask offers H.264: the host reads `0` as an old +/// HEVC-only client. A preference this browser cannot decode is dropped rather than sent. #[unsafe(no_mangle)] pub extern "C" fn pf_session_codecs(mask: u32, hdr: u32, preferred: u32) { - let supported = punktfunk_core::quic::CODEC_HEVC | punktfunk_core::quic::CODEC_AV1; - let offered = (mask as u8 & supported) | punktfunk_core::quic::CODEC_H264; + use punktfunk_core::quic::{CODEC_AV1, CODEC_H264, CODEC_HEVC}; + let offered = match mask as u8 & (CODEC_H264 | CODEC_HEVC | CODEC_AV1) { + 0 => CODEC_H264, + m => m, + }; CODECS.with(|c| c.set(offered)); HDR.with(|h| h.set(hdr != 0)); PREFERRED.with(|p| p.set(preferred as u8 & offered)); @@ -944,6 +952,13 @@ pub extern "C" fn pf_hud_decoded(pts_us: f64, decoded_ms: f64) { }); } +/// How the page's decoder runs, once it has produced a frame: `0` unknown, `1` hardware, +/// `2` software. +#[unsafe(no_mangle)] +pub extern "C" fn pf_video_decoder(accel: u32) { + CLIENT.with(|c| c.borrow_mut().decoder_accel = accel); +} + /// The page drew the frame stamped `pts_us`: decoded at `decoded_ms`, drawn at `presented_ms`. #[unsafe(no_mangle)] pub extern "C" fn pf_hud_presented(pts_us: f64, decoded_ms: f64, presented_ms: f64) { @@ -976,7 +991,12 @@ pub extern "C" fn pf_hud_drain(skipped: u32) { snap.auto_rate = c.bitrate_kbps == 0; (snap.width, snap.height, snap.refresh_hz) = (c.width, c.height, c.fps); snap.codec = hud::codec_label(c.codec).into(); - snap.decoder = "WebCodecs".into(); + snap.decoder = match c.decoder_accel { + 1 => "WebCodecs, hardware", + 2 => "WebCodecs, software", + _ => "WebCodecs", + } + .into(); let rtt = crate::transport::rtt_us(); snap.rtt_us = (rtt > 0).then_some(rtt); hud::quality(&snap, &mut c.quality); diff --git a/packages/stream/src/emscripten.d.ts b/packages/stream/src/emscripten.d.ts index 2a65196..1d34ead 100644 --- a/packages/stream/src/emscripten.d.ts +++ b/packages/stream/src/emscripten.d.ts @@ -108,6 +108,8 @@ export interface PunktfunkModule { _pf_hud_drain(skipped: number): void; /** The connection dot as of the last drain: `0` starting, `1` good, `2` fair, `3` poor. */ _pf_hud_quality(): number; + /** How the decoder that works runs, for the overlay: `0` unknown, `1` hardware, `2` software. */ + _pf_video_decoder?(accel: number): void; /** Format the last window at `tier` (0–3) in the Advanced vocabulary or not; byte length. */ _pf_hud_text(tier: number, advanced: number): number; /** Where `_pf_hud_text` left its bytes: `\t\n` per line. */ diff --git a/packages/stream/src/engine.test.ts b/packages/stream/src/engine.test.ts index 721590a..29f8140 100644 --- a/packages/stream/src/engine.test.ts +++ b/packages/stream/src/engine.test.ts @@ -249,3 +249,39 @@ test("the connection dot is the wire's rating, read once a second", () => { assert.equal(second(3), "poor"); assert.equal(second(0), "starting"); }); + +test("a codec this browser can't decode gives way to another on offer, once each", () => { + const { engine, mod } = engineIn("ready"); + const offers: number[] = []; + mod._pf_session_codecs = (mask: number) => offers.push(mask); + Object.assign(engine, { codecs: { mask: 0x03, tenBit: false } }); + const undecodable = (codec: number) => (engine as unknown as { undecodable(codec: number): void }).undecodable(codec); + engine.configure({ codec: "hevc" }); + engine.startStream({ width: 1280, height: 720 }); + const before = dialled.length; + undecodable(0x02); + assert.equal(engine.current.kind, "starting", "no error while H.264 is left"); + assert.deepEqual(offers, [0x03, 0x01], "the same stream, offered without HEVC"); + assert.equal(dialled.length, before + 1); + + // The session that took over names its codec once `Welcome` has. + Object.assign(mod, { _pf_session_phase: () => 2, _pf_session_audio_channels: () => 0, _pf_hud_text: () => 0 }); + const snapshot = () => ({ codec: 0x01, width: 0, height: 0, decoded: 0, dropped: 0, uploadMs: 0, backend: null }); + Object.assign(engine, { video: { snapshot, close() {} } }); + (engine as unknown as { updateSession(): void }).updateSession(); + const live = engine.current as import("./engine.ts").EngineState; + assert.equal( + live.kind === "streaming" && live.stats.launchNotice, + "This browser can't decode HEVC, so the stream uses H.264, ignoring the Video codec setting.", + ); + + // H.264 fails too: nothing is left, and HEVC is not offered again. + undecodable(0x01); + assert.equal(engine.current.kind, "error"); + assert.deepEqual(offers, [0x03, 0x01]); + + // A stream the player starts offers every codec again. + (engine as unknown as { set(s: unknown): void }).set({ kind: "ready", origin: ORIGIN }); + engine.startStream({ width: 1280, height: 720 }); + assert.equal(offers.at(-1), 0x03); +}); diff --git a/packages/stream/src/engine.ts b/packages/stream/src/engine.ts index e9e1e88..8776737 100644 --- a/packages/stream/src/engine.ts +++ b/packages/stream/src/engine.ts @@ -197,6 +197,8 @@ const TUNABLE_DEFAULTS: TunableOptions = { /** `CODEC_*` bits, as `Hello::preferred_codec` names one. */ const CODEC_BIT: Record = { auto: 0, h264: 0x01, hevc: 0x02, av1: 0x04 }; +/** A `CODEC_*` bit as the player reads it. */ +const CODEC_NAME: Record = { 0x01: "H.264", 0x02: "HEVC", 0x04: "AV1" }; export interface EngineOptions { /** The lower canvas: decoded video goes here and nowhere else. */ @@ -298,6 +300,11 @@ export class Engine { private streamed: StreamOptions | null = null; /** The stream is an SDR restart, which the next session says once. */ private sdrNotice = false; + /** Codec bits this browser failed to decode since the player started the stream. The offer + * leaves them out, so two failing codecs can't alternate. */ + private refused = 0; + /** The codec the stream fell back from, until the session that replaced it names its own. */ + private fellBackFrom = 0; /** A request for access, sent as `Hello` when the next control stream opens. Survives `reset`, * as `pendingPin` does, for the reconnect that clears a stale pairing first. */ private pendingKnock: StreamOptions | null = null; @@ -598,10 +605,9 @@ export class Engine { */ startStream(opts: StreamOptions): void { if (!this.origin || this.state.kind !== "ready") return; - // HDR is offered per stream: the window may have moved to another display, and the plane - // may have been switched to WebGL2, since the last one. - const hdr = this.tunable.hdr && this.codecs.tenBit && hdrDisplay(this.tunable.videoBackend, this.gpu); - this.mod._pf_session_codecs?.(this.codecs.mask, hdr ? 1 : 0, CODEC_BIT[this.tunable.codec]); + this.refused = 0; + this.fellBackFrom = 0; + this.offer(false); this.set({ kind: "starting", origin: this.origin }); this.streamed = opts; this.pendingStream = opts; @@ -609,27 +615,47 @@ export class Engine { } /** - * The browser converted an HDR frame itself, which nothing here can undo: the same stream again - * in SDR, on a fresh connection. The title keeps running on the host; `hdrDisplay` now says no - * for this browser build, so this happens once. + * What the next `Hello` offers: this browser's codecs less those it refused, the player's pick + * first, and HDR unless `sdr`. HDR is asked per stream: the window may have moved to another + * display, and the plane may have been switched to WebGL2, since the last one. */ - private restartInSdr(): void { + private offer(sdr: boolean): void { + const hdr = !sdr && this.tunable.hdr && this.codecs.tenBit && hdrDisplay(this.tunable.videoBackend, this.gpu); + this.mod._pf_session_codecs?.(this.codecs.mask & ~this.refused, hdr ? 1 : 0, CODEC_BIT[this.tunable.codec]); + } + + /** The same stream again on a fresh connection, offering what `offer` allows now. The title + * keeps running on the host. False when no stream is up to restart. */ + private restart(sdr: boolean): boolean { const origin = this.origin; const opts = this.streamed; - if (!origin || !opts || (this.state.kind !== "streaming" && this.state.kind !== "starting")) return; + if (!origin || !opts || (this.state.kind !== "streaming" && this.state.kind !== "starting")) return false; this.stopSession(); - this.mod._pf_session_codecs?.(this.codecs.mask, 0, CODEC_BIT[this.tunable.codec]); - this.sdrNotice = true; + this.offer(sdr); this.set({ kind: "starting", origin }); this.pendingStream = opts; this.dialBox(); + return true; } - /** The browser decodes this stream neither in hardware nor in software. The title keeps - * running on the host, as on a Leave. */ - private undecodable(): void { + /** The browser converted an HDR frame itself, which nothing here can undo: the same stream in + * SDR. `hdrDisplay` now says no for this browser build, so this happens once. */ + private restartInSdr(): void { + if (this.restart(true)) this.sdrNotice = true; + } + + /** The browser decodes the stream's codec under no acceleration. Another codec on offer takes + * over, once per codec per stream the player starts. With none left the stream ends, and the + * title keeps running on the host, as on a Leave. */ + private undecodable(codec: number): void { const origin = this.origin; if (!origin || (this.state.kind !== "streaming" && this.state.kind !== "starting")) return; + const fresh = (this.refused & codec) === 0; + this.refused |= codec; + if (fresh && (this.codecs.mask & ~this.refused) !== 0 && this.restart(false)) { + this.fellBackFrom = codec; + return; + } this.stopSession(); // `error` before the close, so the close it answers with finds nothing left to report. this.set({ @@ -683,7 +709,7 @@ export class Engine { this.opts.videoCanvas, this.tunable.videoBackend, () => setTimeout(() => this.restartInSdr(), 0), - () => this.undecodable(), + (codec) => this.undecodable(codec), ); this.video.attach(); this.tier = this.tunable.statsTier; @@ -1309,6 +1335,13 @@ export class Engine { } const now = performance.now(); + // A codec fallback names the codec that took over, which only `Welcome` says. + if (this.fellBackFrom && v?.codec) { + const pinned = this.refused & CODEC_BIT[this.tunable.codec] ? ", ignoring the Video codec setting" : ""; + const text = `This browser can't decode ${CODEC_NAME[this.fellBackFrom]}, so the stream uses ${CODEC_NAME[v.codec]}${pinned}.`; + this.launchNotice = { text, at: now }; + this.fellBackFrom = 0; + } this.readAccess(now); const frames = this.mod._pf_session_frames(); let second = false; diff --git a/packages/stream/src/video.test.ts b/packages/stream/src/video.test.ts index 3785df1..97a9feb 100644 --- a/packages/stream/src/video.test.ts +++ b/packages/stream/src/video.test.ts @@ -1,12 +1,23 @@ -// The decoder's recovery, against a stand-in WebCodecs that acts like a browser with no hardware -// decoder: it refuses `prefer-hardware` after `configure`, and `close()` on a closed codec throws. +// The decoder's recovery, against a stand-in WebCodecs that acts like a browser told what it +// decodes: `configure` refuses the rest after the fact, `isConfigSupported` may claim more than +// `configure` takes, and `close()` on a closed codec throws. import { test } from "node:test"; import assert from "node:assert/strict"; import type { PunktfunkModule } from "./emscripten.ts"; -let decodes: (accel: HardwareAcceleration) => boolean = () => true; +interface Browser { + /** What `configure` takes. */ + decodes: (accel: HardwareAcceleration) => boolean; + /** What `isConfigSupported` says yes to: `decodes`, unless it lies. */ + claims?: (accel: HardwareAcceleration) => boolean; + /** What `configure` throws on rather than calling `error`. */ + throws?: (accel: HardwareAcceleration) => boolean; +} +let browser: Browser = { decodes: () => true }; let decoders: FakeDecoder[] = []; +/** Every acceleration `isConfigSupported` was asked about, in order. */ +let probed: HardwareAcceleration[] = []; class FakeDecoder { state: CodecState = "unconfigured"; @@ -17,12 +28,16 @@ class FakeDecoder { decoders.push(this); } static async isConfigSupported(config: VideoDecoderConfig) { - return { supported: decodes(config.hardwareAcceleration ?? "no-preference"), config }; + const accel = config.hardwareAcceleration ?? "no-preference"; + probed.push(accel); + return { supported: (browser.claims ?? browser.decodes)(accel), config }; } configure(config: VideoDecoderConfig) { + const accel = config.hardwareAcceleration ?? "no-preference"; + if (browser.throws?.(accel)) throw new TypeError("Unsupported codec."); this.config = config; this.state = "configured"; - if (decodes(config.hardwareAcceleration ?? "no-preference")) return; + if (browser.decodes(accel)) return; this.state = "closed"; setTimeout(() => this.init.error(new DOMException("Unsupported configuration.", "OperationError")), 0); } @@ -30,6 +45,15 @@ class FakeDecoder { if (this.state === "closed") throw new DOMException("Cannot call 'close' on a closed codec.", "InvalidStateError"); this.state = "closed"; } + /** A decoded frame out. */ + emit() { + this.init.output({ timestamp: 0, close() {} } as unknown as VideoFrame); + } + /** A working decoder fails, as one does when the GPU goes away. */ + fail() { + this.state = "closed"; + this.init.error(new DOMException("Decoding error.", "EncodingError")); + } } globalThis.VideoDecoder = FakeDecoder as unknown as typeof VideoDecoder; const { VideoPipe } = await import("./video.ts"); @@ -37,38 +61,87 @@ const { VideoPipe } = await import("./video.ts"); /** A pipe with a stand-in plane, told the stream is H.264 at 1080p. */ function stream() { decoders = []; + probed = []; const asked: string[] = []; - let undecodable = 0; - const mod = { _pf_request_keyframe: () => asked.push("keyframe") } as unknown as PunktfunkModule; - const pipe = new VideoPipe(mod, {} as HTMLCanvasElement, "auto", undefined, () => undecodable++); + const refused: number[] = []; + const kinds: number[] = []; + const mod = { + _pf_request_keyframe: () => asked.push("keyframe"), + _pf_hud_decoded: () => {}, + _pf_gate_decoded: () => 1, + _pf_video_decoder: (accel: number) => kinds.push(accel), + } as unknown as PunktfunkModule; + const pipe = new VideoPipe(mod, {} as HTMLCanvasElement, "auto", undefined, (codec) => refused.push(codec)); // A real plane needs a GPU; the pipe only configures and destroys this one. - Object.assign(pipe, { plane: { configure() {}, setDynamicRange() {}, destroy() {} } }); + const plane = { configure() {}, setDynamicRange() {}, destroy() {}, takeUploadStats: () => ({ frames: 0 }) }; + Object.assign(pipe, { plane }); pipe.attach(); mod.__pfOnVideoConfig!(1, 1920, 1080, 8, false, false); - return { pipe, asked, undecodable: () => undecodable }; + return { pipe, asked, refused, kinds }; } const settle = () => new Promise((r) => setTimeout(r, 20)); +/** Wait for `done`: each rebuild after the first waits out the half-second throttle. */ +const until = async (done: () => boolean) => { + for (let i = 0; i < 150 && !done(); i++) await settle(); +}; test("a browser without a hardware decoder streams on the one it has", async () => { - decodes = (accel) => accel !== "prefer-hardware"; - const { pipe, asked, undecodable } = stream(); + browser = { decodes: (accel) => accel !== "prefer-hardware" }; + const { pipe, asked, refused } = stream(); await settle(); assert.equal(decoders.length, 2); assert.equal(decoders[1]!.config!.hardwareAcceleration, "no-preference"); assert.equal(decoders[1]!.state, "configured"); assert.deepEqual(asked, ["keyframe"], "the fresh decoder resumes on an IDR"); - assert.equal(undecodable(), 0); + assert.deepEqual(refused, []); pipe.close(); assert.equal(decoders[1]!.state, "closed"); }); test("a stream the browser cannot decode ends, and the pipe still closes", async () => { - decodes = () => false; - const { pipe, asked, undecodable } = stream(); + browser = { decodes: () => false }; + const { pipe, asked, refused } = stream(); await settle(); assert.equal(decoders.length, 1, "no decoder is built for a configuration the browser refuses"); - assert.equal(undecodable(), 1); + assert.deepEqual(refused, [1], "the pipe names the codec it could not decode"); assert.deepEqual(asked, []); assert.doesNotThrow(() => pipe.close()); }); + +test("a browser that claims what it then refuses moves down to software, then gives up", async () => { + browser = { decodes: () => false, claims: () => true }; + const { pipe, refused } = stream(); + await until(() => refused.length > 0); + assert.deepEqual( + decoders.map((d) => d.config!.hardwareAcceleration), + ["prefer-hardware", "no-preference", "prefer-software"], + ); + assert.deepEqual(probed, ["no-preference", "prefer-software"], "hardware is never asked about again"); + assert.deepEqual(refused, [1]); + pipe.close(); +}); + +test("a configure that throws reaches the rebuild like one that errors", async () => { + browser = { decodes: () => true, throws: (accel) => accel === "prefer-hardware" }; + const { pipe, asked, refused } = stream(); + await until(() => decoders.length === 2); + assert.equal(decoders[1]!.config!.hardwareAcceleration, "no-preference"); + assert.equal(decoders[1]!.state, "configured"); + assert.equal(pipe.snapshot().errors, 1); + assert.deepEqual(asked, ["keyframe"]); + assert.deepEqual(refused, []); + pipe.close(); +}); + +test("the first frame tells the overlay how it decodes, and a later error keeps that", async () => { + browser = { decodes: () => true }; + const { pipe, kinds } = stream(); + decoders[0]!.emit(); + decoders[0]!.emit(); + assert.deepEqual(kinds, [1], "hardware, said once"); + decoders[0]!.fail(); + await until(() => decoders.length === 2); + assert.equal(decoders[1]!.config!.hardwareAcceleration, "prefer-hardware", "a decoder that worked keeps its acceleration"); + pipe.close(); +}); diff --git a/packages/stream/src/video.ts b/packages/stream/src/video.ts index dbf4069..a013087 100644 --- a/packages/stream/src/video.ts +++ b/packages/stream/src/video.ts @@ -143,6 +143,8 @@ const PQ_PASSTHROUGH: VideoColorSpaceInit = { }; export interface VideoStats { + /** The wire codec `Welcome` named (`CODEC_*`); `0` before it. */ + codec: number; /** Access units handed to the decoder. */ submitted: number; /** Frames the decoder produced. A gap against `submitted` is loss or a decoder still filling. */ @@ -159,6 +161,14 @@ export interface VideoStats { backend: "webgpu" | "webgl2" | null; } +/** The order a decoder is tried in. One that fails before its first frame moves to the next: + * `isConfigSupported` can say yes to a configuration `configure` then refuses. */ +const ACCELERATIONS: HardwareAcceleration[] = ["prefer-hardware", "no-preference", "prefer-software"]; + +/** What `_pf_video_decoder` is told once a decoder works: `1` hardware, `2` software, `0` the + * browser's own pick, which it does not reveal. */ +const DECODER_KIND: Record = { "prefer-hardware": 1, "no-preference": 0, "prefer-software": 2 }; + /** * Which plane to draw on. * @@ -206,20 +216,22 @@ export class VideoPipe { /** This decoder has asked the host for that key frame. */ private askedKey = false; private closed = false; - private codec = 0; /** `Welcome`'s bit depth, HDR and range, for the decoder string, the plane and `PQ_PASSTHROUGH`. */ private depth = 8; private hdr = false; private fullRange = false; /** The first frame of an HDR configure has been checked for `PQ_PASSTHROUGH`. */ private hdrChecked = false; - /** `no-preference` once the browser refused `prefer-hardware` for this stream. */ + /** Where in `ACCELERATIONS` this stream's decoder is. */ private accel: HardwareAcceleration = "prefer-hardware"; + /** The decoder has produced a frame, so its acceleration works here. */ + private stuck = false; private rebuilding = false; private lastRebuildMs = -Infinity; /** When `pending` left the decoder, for the overlay's display stage. */ private pendingDecodedMs = 0; private stats: VideoStats = { + codec: 0, submitted: 0, decoded: 0, dropped: 0, @@ -237,8 +249,8 @@ export class VideoPipe { private readonly prefer: "auto" | "webgl2" | "webgpu" = "auto", /** The browser converted an HDR frame itself: this stream cannot be shown right. */ private readonly onHdrRefused?: () => void, - /** The browser decodes this stream neither in hardware nor in software. */ - private readonly onUndecodable?: () => void, + /** The browser decodes this stream's codec (`CODEC_*`) under no acceleration. */ + private readonly onUndecodable?: (codec: number) => void, ) {} /** Install the callbacks the glue calls. Idempotent, so a reconnect can call it again. */ @@ -247,12 +259,15 @@ export class VideoPipe { this.depth = depth; this.hdr = hdr; this.fullRange = fullRange; + // A new configuration from the host starts at hardware again; only a rebuild moves down. + this.accel = "prefer-hardware"; void this.configure(codec, width, height); }; this.mod.__pfOnAccessUnit = (data, ptsUs, key, flags) => this.submit(data, ptsUs, key, flags); } private async configure(codec: number, width: number, height: number): Promise { + this.stats.codec = codec; this.stats.width = width; this.stats.height = height; if (!this.plane) { @@ -266,20 +281,23 @@ export class VideoPipe { this.plane.configure(width, height); this.plane.setDynamicRange(this.hdr ? "high" : "standard"); - this.codec = codec; this.closeDecoder(); this.meta.clear(); this.needKey = true; this.askedKey = false; - const decoder = new VideoDecoder({ - output: (frame) => this.onFrame(frame), - error: (e) => { - this.stats.errors++; - console.error("punktfunk: decoder", e); - void this.rebuild(); - }, - }); - decoder.configure(this.config(this.accel)); + this.stuck = false; + const fail = (e: unknown) => { + this.stats.errors++; + console.error("punktfunk: decoder", e); + void this.rebuild(); + }; + const decoder = new VideoDecoder({ output: (frame) => this.onFrame(frame), error: fail }); + // A configuration the engine rejects outright throws here rather than calling `error`. + try { + decoder.configure(this.config(this.accel)); + } catch (e) { + fail(e); + } this.hdrChecked = false; this.decoder = decoder; } @@ -288,7 +306,7 @@ export class VideoPipe { * host sends parameter sets inline with every key frame, which lets a browser join mid-stream. */ private config(hardwareAcceleration: HardwareAcceleration): VideoDecoderConfig { return { - codec: codecString(this.codec, this.depth), + codec: codecString(this.stats.codec, this.depth), codedWidth: this.stats.width, codedHeight: this.stats.height, optimizeForLatency: true, @@ -340,30 +358,30 @@ export class VideoPipe { } } - // A decoder that errored is closed for good. Build a fresh one the way the browser still takes - // this stream — one without a hardware decoder for it refuses `prefer-hardware` — and resume on - // the next IDR; the gate holds the last picture meanwhile. At most twice a second. + // A decoder that errored is closed for good. Build a fresh one and resume on the next IDR; the + // gate holds the last picture meanwhile. One that never produced a frame failed its + // acceleration, so the next starts past it in `ACCELERATIONS`. At most twice a second. private async rebuild(): Promise { if (this.rebuilding || !this.plane) return; this.rebuilding = true; const wait = this.lastRebuildMs + 500 - performance.now(); if (wait > 0) await new Promise((r) => setTimeout(r, wait)); - const accel = await this.acceleration(); + const accel = await this.acceleration(ACCELERATIONS.indexOf(this.accel) + (this.stuck ? 0 : 1)); this.rebuilding = false; // Closed meanwhile, or a reconfigure already built a working decoder. if (!this.plane || this.decoder?.state === "configured") return; this.lastRebuildMs = performance.now(); - if (!accel) return this.onUndecodable?.(); + if (!accel) return this.onUndecodable?.(this.stats.codec); this.accel = accel; - void this.configure(this.codec, this.stats.width, this.stats.height); + void this.configure(this.stats.codec, this.stats.width, this.stats.height); this.mod._pf_request_keyframe(); this.askedKey = true; } - /** The first of `prefer-hardware` and `no-preference` the browser takes for this stream, or - * null when it decodes it neither way. */ - private async acceleration(): Promise { - for (const accel of ["prefer-hardware", "no-preference"] as const) { + /** The first of `ACCELERATIONS` from index `from` on that the browser says it takes for this + * stream, or null when none is left. */ + private async acceleration(from: number): Promise { + for (const accel of ACCELERATIONS.slice(from)) { try { if ((await VideoDecoder.isConfigSupported(this.config(accel))).supported) return accel; } catch { @@ -378,6 +396,11 @@ export class VideoPipe { // than queued. Presenting happens on the page's own rAF, which is where pacing belongs. private onFrame(frame: VideoFrame): void { this.stats.decoded++; + // The first frame proves the acceleration: the overlay names it, and a later error keeps it. + if (!this.stuck) { + this.stuck = true; + this.mod._pf_video_decoder?.(DECODER_KIND[this.accel]); + } // An engine that ignores the override hands over light it has already converted, which no // shader here can undo. Said once; the engine streams SDR instead. if (this.hdr && !this.hdrChecked) {