Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/stream/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ link=(
-C link-arg=-sSTACK_SIZE=4MB
# Every `_pf_*` the page calls, the ring's included: `src/carrier/feed.ts` writes datagrams
# into it.
-C link-arg=-sEXPORTED_FUNCTIONS=_main,_pf_start,_pf_frame,_pf_key,_pf_console_text,_pf_console_pointer,_pf_console_pad,_pf_console_phase,_pf_console_state,_pf_console_leave_out,_pf_console_push,_pf_console_art,_pf_rx_base,_pf_rx_stride,_pf_rx_claim,_pf_rx_commit,_pf_rx_dropped,_pf_net_blast,_pf_net_drain,_pf_wt_close,_pf_wt_ctl_open,_pf_ctl_recv,_pf_uni_recv,_pf_session_hello,_pf_session_codecs,_pf_session_pump,_pf_session_phase,_pf_session_frames,_pf_device_init,_pf_device_set,_pf_device_sign,_pf_device_fingerprint_hex,_pf_cred_phase,_pf_cred_sign_ptr,_pf_cred_sign_len,_pf_cred_signed,_pf_pair_begin,_pf_input,_pf_gamepad,_pf_gamepad_arrival,_pf_gamepad_remove,_pf_audio_frames,_pf_audio_lost,_pf_session_audio_channels,_pf_session_audio,_pf_session_audio_layout,_pf_mic_send,_pf_session_access_seq,_pf_session_access_grants,_pf_session_access_secs,_pf_session_reset,_pf_session_reconfigure,_pf_hud_decoded,_pf_hud_presented,_pf_gate_decoded,_pf_gate_no_output,_pf_request_keyframe,_pf_hud_drain,_pf_hud_quality,_pf_hud_text,_pf_hud_text_ptr,_pf_session_host_caps,_pf_cursor_render,_pf_cursor_serial,_pf_cursor_flags,_malloc,_free
-C link-arg=-sEXPORTED_FUNCTIONS=_main,_pf_start,_pf_frame,_pf_key,_pf_console_text,_pf_console_pointer,_pf_console_pad,_pf_console_phase,_pf_console_state,_pf_console_leave_out,_pf_console_push,_pf_console_art,_pf_rx_base,_pf_rx_stride,_pf_rx_claim,_pf_rx_commit,_pf_rx_dropped,_pf_net_blast,_pf_net_drain,_pf_wt_close,_pf_wt_ctl_open,_pf_ctl_recv,_pf_uni_recv,_pf_session_hello,_pf_session_codecs,_pf_session_pump,_pf_session_phase,_pf_session_frames,_pf_device_init,_pf_device_set,_pf_device_sign,_pf_device_fingerprint_hex,_pf_cred_phase,_pf_cred_sign_ptr,_pf_cred_sign_len,_pf_cred_signed,_pf_pair_begin,_pf_input,_pf_gamepad,_pf_gamepad_arrival,_pf_gamepad_remove,_pf_audio_frames,_pf_audio_lost,_pf_session_audio_channels,_pf_session_audio,_pf_session_audio_layout,_pf_mic_send,_pf_session_access_seq,_pf_session_access_grants,_pf_session_access_secs,_pf_session_reset,_pf_session_reconfigure,_pf_hud_decoded,_pf_hud_presented,_pf_gate_decoded,_pf_gate_no_output,_pf_request_keyframe,_pf_hud_drain,_pf_hud_quality,_pf_video_decoder,_pf_hud_text,_pf_hud_text_ptr,_pf_session_host_caps,_pf_cursor_render,_pf_cursor_serial,_pf_cursor_flags,_malloc,_free
# `stringToNewUTF8` is how the page hands a string across; `HEAPU8` is emscripten's view of
# wasm memory, which `src/carrier/feed.ts` writes datagrams into.
-C link-arg=-sEXPORTED_RUNTIME_METHODS=stringToNewUTF8,HEAPU8
Expand Down
34 changes: 27 additions & 7 deletions packages/stream/rust/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ struct Client {
depth: u8,
hdr: bool,
full_range: bool,
/// How the page's decoder runs, as `pf_video_decoder` last said: `0` unknown, `1` hardware,
/// `2` software. The overlay's decoder line names it.
decoder_accel: u32,
/// Access units delivered, so the page can tell "connected" from "streaming".
frames: u64,
/// The stats overlay's window. No clock handshake runs in a browser, so its offset stays
Expand Down Expand Up @@ -147,6 +150,7 @@ impl Client {
depth: 8,
hdr: false,
full_range: false,
decoder_accel: 0,
frames: 0,
hud: new_hud(),
receipts: VecDeque::with_capacity(RECEIPTS),
Expand Down Expand Up @@ -214,6 +218,7 @@ pub extern "C" fn pf_session_reset() {
c.inbox.clear();
c.frames = 0;
c.codec = 0;
c.decoder_accel = 0;
c.width = 0;
c.height = 0;
c.asked = None;
Expand Down Expand Up @@ -323,14 +328,17 @@ thread_local! {
static PREFERRED: std::cell::Cell<u8> = const { std::cell::Cell::new(0) };
}

/// The codecs this browser decodes (`CODEC_*` bits), from the page's `isConfigSupported` probe,
/// whether it presents BT.2020 PQ, and the one codec bit to ask for first (`0`: the host's
/// order). H.264 is always kept: a GPU-less host has nothing else. A preference this browser
/// cannot decode is dropped rather than sent.
/// The codecs this browser decodes (`CODEC_*` bits), from the page's `isConfigSupported` probe
/// less those it failed to decode, whether it presents BT.2020 PQ, and the one codec bit to ask
/// for first (`0`: the host's order). An empty mask offers H.264: the host reads `0` as an old
/// HEVC-only client. A preference this browser cannot decode is dropped rather than sent.
#[unsafe(no_mangle)]
pub extern "C" fn pf_session_codecs(mask: u32, hdr: u32, preferred: u32) {
let supported = punktfunk_core::quic::CODEC_HEVC | punktfunk_core::quic::CODEC_AV1;
let offered = (mask as u8 & supported) | punktfunk_core::quic::CODEC_H264;
use punktfunk_core::quic::{CODEC_AV1, CODEC_H264, CODEC_HEVC};
let offered = match mask as u8 & (CODEC_H264 | CODEC_HEVC | CODEC_AV1) {
0 => CODEC_H264,
m => m,
};
CODECS.with(|c| c.set(offered));
HDR.with(|h| h.set(hdr != 0));
PREFERRED.with(|p| p.set(preferred as u8 & offered));
Expand Down Expand Up @@ -944,6 +952,13 @@ pub extern "C" fn pf_hud_decoded(pts_us: f64, decoded_ms: f64) {
});
}

/// How the page's decoder runs, once it has produced a frame: `0` unknown, `1` hardware,
/// `2` software.
#[unsafe(no_mangle)]
pub extern "C" fn pf_video_decoder(accel: u32) {
CLIENT.with(|c| c.borrow_mut().decoder_accel = accel);
}

/// The page drew the frame stamped `pts_us`: decoded at `decoded_ms`, drawn at `presented_ms`.
#[unsafe(no_mangle)]
pub extern "C" fn pf_hud_presented(pts_us: f64, decoded_ms: f64, presented_ms: f64) {
Expand Down Expand Up @@ -976,7 +991,12 @@ pub extern "C" fn pf_hud_drain(skipped: u32) {
snap.auto_rate = c.bitrate_kbps == 0;
(snap.width, snap.height, snap.refresh_hz) = (c.width, c.height, c.fps);
snap.codec = hud::codec_label(c.codec).into();
snap.decoder = "WebCodecs".into();
snap.decoder = match c.decoder_accel {
1 => "WebCodecs, hardware",
2 => "WebCodecs, software",
_ => "WebCodecs",
}
.into();
let rtt = crate::transport::rtt_us();
snap.rtt_us = (rtt > 0).then_some(rtt);
hud::quality(&snap, &mut c.quality);
Expand Down
2 changes: 2 additions & 0 deletions packages/stream/src/emscripten.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ export interface PunktfunkModule {
_pf_hud_drain(skipped: number): void;
/** The connection dot as of the last drain: `0` starting, `1` good, `2` fair, `3` poor. */
_pf_hud_quality(): number;
/** How the decoder that works runs, for the overlay: `0` unknown, `1` hardware, `2` software. */
_pf_video_decoder?(accel: number): void;
/** Format the last window at `tier` (0–3) in the Advanced vocabulary or not; byte length. */
_pf_hud_text(tier: number, advanced: number): number;
/** Where `_pf_hud_text` left its bytes: `<role>\t<text>\n` per line. */
Expand Down
36 changes: 36 additions & 0 deletions packages/stream/src/engine.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -249,3 +249,39 @@ test("the connection dot is the wire's rating, read once a second", () => {
assert.equal(second(3), "poor");
assert.equal(second(0), "starting");
});

test("a codec this browser can't decode gives way to another on offer, once each", () => {
const { engine, mod } = engineIn("ready");
const offers: number[] = [];
mod._pf_session_codecs = (mask: number) => offers.push(mask);
Object.assign(engine, { codecs: { mask: 0x03, tenBit: false } });
const undecodable = (codec: number) => (engine as unknown as { undecodable(codec: number): void }).undecodable(codec);
engine.configure({ codec: "hevc" });
engine.startStream({ width: 1280, height: 720 });
const before = dialled.length;
undecodable(0x02);
assert.equal(engine.current.kind, "starting", "no error while H.264 is left");
assert.deepEqual(offers, [0x03, 0x01], "the same stream, offered without HEVC");
assert.equal(dialled.length, before + 1);

// The session that took over names its codec once `Welcome` has.
Object.assign(mod, { _pf_session_phase: () => 2, _pf_session_audio_channels: () => 0, _pf_hud_text: () => 0 });
const snapshot = () => ({ codec: 0x01, width: 0, height: 0, decoded: 0, dropped: 0, uploadMs: 0, backend: null });
Object.assign(engine, { video: { snapshot, close() {} } });
(engine as unknown as { updateSession(): void }).updateSession();
const live = engine.current as import("./engine.ts").EngineState;
assert.equal(
live.kind === "streaming" && live.stats.launchNotice,
"This browser can't decode HEVC, so the stream uses H.264, ignoring the Video codec setting.",
);

// H.264 fails too: nothing is left, and HEVC is not offered again.
undecodable(0x01);
assert.equal(engine.current.kind, "error");
assert.deepEqual(offers, [0x03, 0x01]);

// A stream the player starts offers every codec again.
(engine as unknown as { set(s: unknown): void }).set({ kind: "ready", origin: ORIGIN });
engine.startStream({ width: 1280, height: 720 });
assert.equal(offers.at(-1), 0x03);
});
63 changes: 48 additions & 15 deletions packages/stream/src/engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,8 @@ const TUNABLE_DEFAULTS: TunableOptions = {

/** `CODEC_*` bits, as `Hello::preferred_codec` names one. */
const CODEC_BIT: Record<TunableOptions["codec"], number> = { auto: 0, h264: 0x01, hevc: 0x02, av1: 0x04 };
/** A `CODEC_*` bit as the player reads it. */
const CODEC_NAME: Record<number, string> = { 0x01: "H.264", 0x02: "HEVC", 0x04: "AV1" };

export interface EngineOptions {
/** The lower canvas: decoded video goes here and nowhere else. */
Expand Down Expand Up @@ -298,6 +300,11 @@ export class Engine {
private streamed: StreamOptions | null = null;
/** The stream is an SDR restart, which the next session says once. */
private sdrNotice = false;
/** Codec bits this browser failed to decode since the player started the stream. The offer
* leaves them out, so two failing codecs can't alternate. */
private refused = 0;
/** The codec the stream fell back from, until the session that replaced it names its own. */
private fellBackFrom = 0;
/** A request for access, sent as `Hello` when the next control stream opens. Survives `reset`,
* as `pendingPin` does, for the reconnect that clears a stale pairing first. */
private pendingKnock: StreamOptions | null = null;
Expand Down Expand Up @@ -598,38 +605,57 @@ export class Engine {
*/
startStream(opts: StreamOptions): void {
if (!this.origin || this.state.kind !== "ready") return;
// HDR is offered per stream: the window may have moved to another display, and the plane
// may have been switched to WebGL2, since the last one.
const hdr = this.tunable.hdr && this.codecs.tenBit && hdrDisplay(this.tunable.videoBackend, this.gpu);
this.mod._pf_session_codecs?.(this.codecs.mask, hdr ? 1 : 0, CODEC_BIT[this.tunable.codec]);
this.refused = 0;
this.fellBackFrom = 0;
this.offer(false);
this.set({ kind: "starting", origin: this.origin });
this.streamed = opts;
this.pendingStream = opts;
this.dialBox();
}

/**
* The browser converted an HDR frame itself, which nothing here can undo: the same stream again
* in SDR, on a fresh connection. The title keeps running on the host; `hdrDisplay` now says no
* for this browser build, so this happens once.
* What the next `Hello` offers: this browser's codecs less those it refused, the player's pick
* first, and HDR unless `sdr`. HDR is asked per stream: the window may have moved to another
* display, and the plane may have been switched to WebGL2, since the last one.
*/
private restartInSdr(): void {
private offer(sdr: boolean): void {
const hdr = !sdr && this.tunable.hdr && this.codecs.tenBit && hdrDisplay(this.tunable.videoBackend, this.gpu);
this.mod._pf_session_codecs?.(this.codecs.mask & ~this.refused, hdr ? 1 : 0, CODEC_BIT[this.tunable.codec]);
}

/** The same stream again on a fresh connection, offering what `offer` allows now. The title
* keeps running on the host. False when no stream is up to restart. */
private restart(sdr: boolean): boolean {
const origin = this.origin;
const opts = this.streamed;
if (!origin || !opts || (this.state.kind !== "streaming" && this.state.kind !== "starting")) return;
if (!origin || !opts || (this.state.kind !== "streaming" && this.state.kind !== "starting")) return false;
this.stopSession();
this.mod._pf_session_codecs?.(this.codecs.mask, 0, CODEC_BIT[this.tunable.codec]);
this.sdrNotice = true;
this.offer(sdr);
this.set({ kind: "starting", origin });
this.pendingStream = opts;
this.dialBox();
return true;
}

/** The browser decodes this stream neither in hardware nor in software. The title keeps
* running on the host, as on a Leave. */
private undecodable(): void {
/** The browser converted an HDR frame itself, which nothing here can undo: the same stream in
* SDR. `hdrDisplay` now says no for this browser build, so this happens once. */
private restartInSdr(): void {
if (this.restart(true)) this.sdrNotice = true;
}

/** The browser decodes the stream's codec under no acceleration. Another codec on offer takes
* over, once per codec per stream the player starts. With none left the stream ends, and the
* title keeps running on the host, as on a Leave. */
private undecodable(codec: number): void {
const origin = this.origin;
if (!origin || (this.state.kind !== "streaming" && this.state.kind !== "starting")) return;
const fresh = (this.refused & codec) === 0;
this.refused |= codec;
if (fresh && (this.codecs.mask & ~this.refused) !== 0 && this.restart(false)) {
this.fellBackFrom = codec;
return;
}
this.stopSession();
// `error` before the close, so the close it answers with finds nothing left to report.
this.set({
Expand Down Expand Up @@ -683,7 +709,7 @@ export class Engine {
this.opts.videoCanvas,
this.tunable.videoBackend,
() => setTimeout(() => this.restartInSdr(), 0),
() => this.undecodable(),
(codec) => this.undecodable(codec),
);
this.video.attach();
this.tier = this.tunable.statsTier;
Expand Down Expand Up @@ -1309,6 +1335,13 @@ export class Engine {
}

const now = performance.now();
// A codec fallback names the codec that took over, which only `Welcome` says.
if (this.fellBackFrom && v?.codec) {
const pinned = this.refused & CODEC_BIT[this.tunable.codec] ? ", ignoring the Video codec setting" : "";
const text = `This browser can't decode ${CODEC_NAME[this.fellBackFrom]}, so the stream uses ${CODEC_NAME[v.codec]}${pinned}.`;
this.launchNotice = { text, at: now };
this.fellBackFrom = 0;
}
this.readAccess(now);
const frames = this.mod._pf_session_frames();
let second = false;
Expand Down
Loading
Loading