From 709cdbc0a8e473403cea9d89fb479934b1bcd718 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:30:50 +0100 Subject: [PATCH 01/48] fix(stream/transport): count a datagram as dropped only when given up A full ring used to count a drop at the first refused claim, but the feed then drains the ring and stores the datagram. The phantom drop reached the bitrate driver as this receiver's own loss, which marks the window a link fault and can cut the rate with nothing lost. The claim now only refuses. The feed reports a datagram it really gives up through pf_rx_lost: one the drained ring still cannot take, or one longer than a slot. --- packages/stream/build.sh | 2 +- packages/stream/rust/transport.rs | 35 ++++++++++++++++-------- packages/stream/src/carrier/feed.test.ts | 20 ++++++++++---- packages/stream/src/carrier/feed.ts | 4 +++ packages/stream/src/emscripten.d.ts | 2 ++ 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/packages/stream/build.sh b/packages/stream/build.sh index b224647..cdf8b9d 100755 --- a/packages/stream/build.sh +++ b/packages/stream/build.sh @@ -85,7 +85,7 @@ link=( -C link-arg=-sSTACK_SIZE=4MB # Every `_pf_*` the page calls, the ring's included: `src/carrier/feed.ts` writes datagrams # into it. - -C link-arg=-sEXPORTED_FUNCTIONS=_main,_pf_start,_pf_frame,_pf_key,_pf_console_text,_pf_console_pointer,_pf_console_pad,_pf_console_phase,_pf_console_state,_pf_console_leave_out,_pf_console_push,_pf_console_art,_pf_console_avatar,_pf_rx_base,_pf_rx_stride,_pf_rx_claim,_pf_rx_commit,_pf_rx_dropped,_pf_net_blast,_pf_net_drain,_pf_wt_close,_pf_wt_ctl_open,_pf_ctl_recv,_pf_uni_recv,_pf_session_hello,_pf_session_codecs,_pf_session_pump,_pf_session_phase,_pf_session_frames,_pf_device_init,_pf_device_set,_pf_device_sign,_pf_device_fingerprint_hex,_pf_cred_phase,_pf_cred_sign_ptr,_pf_cred_sign_len,_pf_cred_signed,_pf_pair_begin,_pf_input,_pf_gamepad,_pf_gamepad_arrival,_pf_gamepad_remove,_pf_audio_frames,_pf_audio_lost,_pf_session_audio_channels,_pf_session_audio,_pf_session_audio_layout,_pf_mic_send,_pf_session_access_seq,_pf_session_access_grants,_pf_session_access_secs,_pf_session_reset,_pf_session_reconfigure,_pf_hud_decoded,_pf_hud_presented,_pf_gate_decoded,_pf_gate_no_output,_pf_request_keyframe,_pf_hud_drain,_pf_hud_quality,_pf_video_decoder,_pf_hud_text,_pf_hud_text_ptr,_pf_session_host_caps,_pf_cursor_render,_pf_cursor_serial,_pf_cursor_flags,_malloc,_free + -C link-arg=-sEXPORTED_FUNCTIONS=_main,_pf_start,_pf_frame,_pf_key,_pf_console_text,_pf_console_pointer,_pf_console_pad,_pf_console_phase,_pf_console_state,_pf_console_leave_out,_pf_console_push,_pf_console_art,_pf_console_avatar,_pf_rx_base,_pf_rx_stride,_pf_rx_claim,_pf_rx_commit,_pf_rx_lost,_pf_rx_dropped,_pf_net_blast,_pf_net_drain,_pf_wt_close,_pf_wt_ctl_open,_pf_ctl_recv,_pf_uni_recv,_pf_session_hello,_pf_session_codecs,_pf_session_pump,_pf_session_phase,_pf_session_frames,_pf_device_init,_pf_device_set,_pf_device_sign,_pf_device_fingerprint_hex,_pf_cred_phase,_pf_cred_sign_ptr,_pf_cred_sign_len,_pf_cred_signed,_pf_pair_begin,_pf_input,_pf_gamepad,_pf_gamepad_arrival,_pf_gamepad_remove,_pf_audio_frames,_pf_audio_lost,_pf_session_audio_channels,_pf_session_audio,_pf_session_audio_layout,_pf_mic_send,_pf_session_access_seq,_pf_session_access_grants,_pf_session_access_secs,_pf_session_reset,_pf_session_reconfigure,_pf_hud_decoded,_pf_hud_presented,_pf_gate_decoded,_pf_gate_no_output,_pf_request_keyframe,_pf_hud_drain,_pf_hud_quality,_pf_video_decoder,_pf_hud_text,_pf_hud_text_ptr,_pf_session_host_caps,_pf_cursor_render,_pf_cursor_serial,_pf_cursor_flags,_malloc,_free # `stringToNewUTF8` is how the page hands a string across; `HEAPU8` is emscripten's view of # wasm memory, which `src/carrier/feed.ts` writes datagrams into. -C link-arg=-sEXPORTED_RUNTIME_METHODS=stringToNewUTF8,HEAPU8 diff --git a/packages/stream/rust/transport.rs b/packages/stream/rust/transport.rs index 16badbe..80136d9 100644 --- a/packages/stream/rust/transport.rs +++ b/packages/stream/rust/transport.rs @@ -41,8 +41,8 @@ struct Ring { head: usize, /// Next slot the pump will read. tail: usize, - /// Datagrams dropped because the pump did not drain in time. FEC covers the gap; the count is - /// what says whether it had to. + /// Datagrams the page dropped ([`pf_rx_lost`]). FEC covers the gap; the count is what says + /// whether it had to, and the bitrate driver reads it as this receiver's own loss. dropped: u64, } @@ -113,21 +113,29 @@ pub extern "C" fn pf_rx_stride() -> u32 { /// Claim the slot to write the next datagram into, or `-1` when the ring is full. /// -/// Full means the pump is behind. Dropping here is the right answer and the protocol's: FEC and -/// the next keyframe recover a lost datagram, where growing a queue only adds latency to every -/// packet behind it. +/// Full means the pump is behind. The page drains it and claims again; a refusal is not a drop +/// until the page gives the datagram up and says so with [`pf_rx_lost`]. #[unsafe(no_mangle)] pub extern "C" fn pf_rx_claim() -> i32 { RING.with(|r| { - let mut r = r.borrow_mut(); + let r = r.borrow(); if r.is_full() { - r.dropped += 1; - return -1; + -1 + } else { + r.head as i32 } - r.head as i32 }) } +/// The page dropped a datagram: the ring stayed full after a drain, or it is longer than a slot. +/// +/// Dropping is the protocol's answer: FEC and the next keyframe recover a lost datagram, where +/// growing a queue only adds latency to every packet behind it. +#[unsafe(no_mangle)] +pub extern "C" fn pf_rx_lost() { + RING.with(|r| r.borrow_mut().dropped += 1); +} + /// Publish the slot `pf_rx_claim` handed out. `len` is clamped to the slot. /// /// Every datagram opens with its `punktfunk/2` kind. Only media is published, moved to the @@ -178,7 +186,7 @@ fn on_host_event(payload: &[u8]) { } } -/// Datagrams dropped because the ring was full. Read by the page for the §5.4 measurement. +/// Datagrams the page dropped ([`pf_rx_lost`]). Read by the page for the §5.4 measurement. #[unsafe(no_mangle)] pub extern "C" fn pf_rx_dropped() -> u32 { RING.with(|r| u32::try_from(r.borrow().dropped).unwrap_or(u32::MAX)) @@ -330,14 +338,17 @@ mod tests { assert!(t.recv().unwrap().is_none()); } + /// A full ring refuses without counting: only the page knows whether a drain then made room. #[test] - fn a_full_ring_drops_rather_than_grows() { + fn a_full_ring_refuses_and_only_a_given_up_datagram_counts() { drain(); for i in 0..(RING_SLOTS - 1) { assert!(push(&[i as u8]), "slot {i} should be free"); } assert!(!push(&[0xff]), "the ring is full and must refuse"); - assert_eq!(pf_rx_dropped(), 1, "a refusal is counted, not silent"); + assert_eq!(pf_rx_dropped(), 0, "a refusal is not yet a drop"); + pf_rx_lost(); + assert_eq!(pf_rx_dropped(), 1, "a datagram given up is counted"); } #[test] diff --git a/packages/stream/src/carrier/feed.test.ts b/packages/stream/src/carrier/feed.test.ts index a1fe19a..6201f22 100644 --- a/packages/stream/src/carrier/feed.test.ts +++ b/packages/stream/src/carrier/feed.test.ts @@ -19,7 +19,8 @@ const { wasmFeed } = await import("./feed.ts"); const BASE = 64; const STRIDE = 8; -function fakeModule(slots = 4) { +/** `drains`: whether the pump empties the ring, as a live session's does. */ +function fakeModule(slots = 4, drains = true) { const heap = new Uint8Array(256); const log: string[] = []; let free = slots; @@ -30,6 +31,7 @@ function fakeModule(slots = 4) { _pf_rx_stride: () => STRIDE, _pf_rx_claim: () => (free > 0 ? slots - free-- : -1), _pf_rx_commit: (slot: number, len: number) => log.push(`commit ${slot} ${len}`), + _pf_rx_lost: () => log.push("lost"), _malloc: (n: number) => { log.push(`malloc ${n}`); return (next += 16) - 16; @@ -39,7 +41,7 @@ function fakeModule(slots = 4) { _pf_uni_recv: (p: number, n: number) => log.push(`uni ${[...heap.subarray(p, p + n)]}`), __pfOnData: () => { log.push("pump"); - free = slots; + if (drains) free = slots; }, }; return { mod, heap, log }; @@ -66,14 +68,22 @@ test("a full ring is drained before the datagram, not dropped", () => { const feed = feedOf(mod); feed.datagram(new Uint8Array([7])); feed.datagram(new Uint8Array([8, 9])); - assert.deepEqual(log, ["commit 0 1", "pump", "commit 0 2"]); + assert.deepEqual(log, ["commit 0 1", "pump", "commit 0 2"], "no drop is reported"); assert.deepEqual([...heap.subarray(BASE, BASE + 2)], [8, 9]); }); -test("a datagram longer than a slot is not written", () => { +test("a ring still full after the drain drops the datagram and says so once", () => { + const { mod, log } = fakeModule(1, false); + const feed = feedOf(mod); + feed.datagram(new Uint8Array([7])); + feed.datagram(new Uint8Array([8, 9])); + assert.deepEqual(log, ["commit 0 1", "pump", "lost"]); +}); + +test("a datagram longer than a slot is not written, and is a drop", () => { const { mod, log } = fakeModule(); feedOf(mod).datagram(new Uint8Array(STRIDE + 1)); - assert.deepEqual(log, []); + assert.deepEqual(log, ["lost"]); }); test("control and stream bytes cross through a buffer freed after the call", () => { diff --git a/packages/stream/src/carrier/feed.ts b/packages/stream/src/carrier/feed.ts index f28b12c..4a2c50c 100644 --- a/packages/stream/src/carrier/feed.ts +++ b/packages/stream/src/carrier/feed.ts @@ -43,9 +43,13 @@ export function wasmFeed(mod: PunktfunkModule): Feed { slot = mod._pf_rx_claim(); } // A slot holds any datagram the protocol allows; a longer one would run into the next. + // Only a datagram given up here is a drop: the bitrate driver reads it as this + // receiver's own loss. if (slot >= 0 && bytes.length <= mod._pf_rx_stride()) { mod.HEAPU8.set(bytes, mod._pf_rx_base() + slot * mod._pf_rx_stride()); mod._pf_rx_commit(slot, bytes.length); + } else { + mod._pf_rx_lost(); } // The session is pumped once per burst, on a task right after it: a frame starts // decoding when its last datagram lands, not at the next animation frame. diff --git a/packages/stream/src/emscripten.d.ts b/packages/stream/src/emscripten.d.ts index a5f6a23..82f93c9 100644 --- a/packages/stream/src/emscripten.d.ts +++ b/packages/stream/src/emscripten.d.ts @@ -142,6 +142,8 @@ export interface PunktfunkModule { _pf_rx_stride(): number; _pf_rx_claim(): number; _pf_rx_commit(slot: number, len: number): void; + /** A datagram the feed gave up: the ring stayed full after a drain, or it outgrew a slot. */ + _pf_rx_lost(): void; _pf_rx_dropped(): number; _malloc(bytes: number): number; From 1430eb5012413fbd62ce8f1a4cc39a0eb4240ec3 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:32:00 +0100 Subject: [PATCH 02/48] fix(stream/abr): discard the window a host pipeline rebuild straddles The host announces each capture and encoder rebuild with PipelineGap: a resize, a source mode change, a capture recovery. The page skipped the message, so the bitrate driver judged the window in flight as the link with no frames, old-epoch drops and a keyframe ask, and could cut the rate for a stall on the host. The page now hands the newest gap to the next tick, which discards that window as the native pump does. --- packages/stream/rust/abr.rs | 46 ++++++++++++++++++++++++++++----- packages/stream/rust/session.rs | 10 +++++-- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/packages/stream/rust/abr.rs b/packages/stream/rust/abr.rs index 0922b0f..cb3aac2 100644 --- a/packages/stream/rust/abr.rs +++ b/packages/stream/rust/abr.rs @@ -32,6 +32,8 @@ struct Since { encode_lat: (u64, u32), host_timings: Vec, keyframe_asks: u32, + /// The newest host pipeline gap, ms: one discard per window. + gap_ms: Option, } impl Since { @@ -41,6 +43,25 @@ impl Since { self.acks.push((ack.bitrate_kbps, ack.reason)); } } + + /// One `PipelineGap`. The newest wins: a tick discards one window however many landed. + fn on_pipeline_gap(&mut self, gap_ms: u32) { + self.gap_ms = Some(gap_ms); + } + + /// What was read, as the driver's inputs. The session's own counters go in at the tick. + fn into_inputs(self) -> Inputs { + Inputs { + gap_ms: self.gap_ms, + mode: self.mode, + acks: self.acks, + decode_lat: self.decode_lat, + encode_lat: self.encode_lat, + host_timings: self.host_timings, + keyframe_asks: self.keyframe_asks, + ..Inputs::default() + } + } } /// One session's driver, and what it has heard since its last tick. @@ -113,6 +134,12 @@ impl Abr { self.since.mode = Some(mode); } + /// The host rebuilt its pipeline and nothing flowed for `gap_ms`: the window in flight + /// describes the rebuild, not the link, and the next tick discards it. + pub fn on_pipeline_gap(&mut self, gap_ms: u32) { + self.since.on_pipeline_gap(gap_ms); + } + /// One access unit for the decoder. A repeat is the host's idle keepalive, not content. pub fn on_au(&mut self, flags: u32) { self.feed.driver.on_au(flags & USER_FLAG_REPEAT != 0); @@ -152,13 +179,7 @@ impl Abr { sock_drops: Some(ring_drops.saturating_sub(self.drops_at_start)), shard_delays: session.take_shard_delays().collect(), probe_active: false, - mode: s.mode, - acks: s.acks, - decode_lat: s.decode_lat, - encode_lat: s.encode_lat, - host_timings: s.host_timings, - keyframe_asks: s.keyframe_asks, - ..Inputs::default() + ..s.into_inputs() }); let unsent = apply(tick.actions, send); let d = &mut self.feed.driver; @@ -242,6 +263,17 @@ mod tests { assert_eq!(since.acks, vec![(8_000, short), (20_000, None)]); } + /// The newest host pipeline gap reaches one tick, which discards its window; the next + /// tick has none. + #[test] + fn the_newest_pipeline_gap_reaches_one_tick() { + let mut since = Since::default(); + since.on_pipeline_gap(401); + since.on_pipeline_gap(120); + assert_eq!(std::mem::take(&mut since).into_inputs().gap_ms, Some(120)); + assert_eq!(since.into_inputs().gap_ms, None); + } + /// A rate the control stream took is sent; one it did not goes back, as every probe does. #[test] fn an_unsent_rate_goes_back_to_the_driver() { diff --git a/packages/stream/rust/session.rs b/packages/stream/rust/session.rs index 9378a1b..5b30234 100644 --- a/packages/stream/rust/session.rs +++ b/packages/stream/rust/session.rs @@ -28,8 +28,8 @@ use punktfunk_core::quic::v2::msg::{Redirect, V2Message}; use punktfunk_core::quic::v2::registry as reg; use punktfunk_core::quic::{ AccessUpdate, AuthChallenge, BitrateChanged, CursorRenderMode, CursorShape, Hello, - LaunchOutcome, LinkFacts, PairChallenge, PairResult, Reconfigure, Reconfigured, Refused, - SetBitrate, + LaunchOutcome, LinkFacts, PairChallenge, PairResult, PipelineGap, Reconfigure, Reconfigured, + Refused, SetBitrate, }; use punktfunk_core::reanchor::{GateVerdict, ReanchorGate}; use punktfunk_core::session::{MediaV2, Session}; @@ -678,6 +678,12 @@ fn on_frame(c: &mut Client, ty: u64, body: &[u8]) { abr.on_ack(body); } } + // The host rebuilt capture and encoder: the bitrate window in flight is not the link's. + PipelineGap::TYPE => { + if let (Ok(gap), Some(abr)) = (PipelineGap::from_body(body), c.abr.as_mut()) { + abr.on_pipeline_gap(gap.gap_ms); + } + } AccessUpdate::TYPE => { if let Ok(u) = AccessUpdate::from_body(body) { c.grants = u.grants; From a57bd7f35f2ad8999de004f6110734cf0a5af091 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:33:54 +0100 Subject: [PATCH 03/48] fix(stream/abr): count the page's own keyframe asks as starved The bitrate driver cuts on keyframe asks a loss stands behind, and two in one window are enough. The page counted every ask that way, including a decoder error streak and the hold a rebuilt decoder raises, with the hold's half-second re-asks. Nothing was lost there, and a lower rate cannot feed a struggling decoder. The hold now remembers whether the page raised it on an open gate, the rule core's RecoveryDriver applies. Those asks reach the driver as starved asks. A loss that re-arms the hold makes it the loss's again, and a rebuilt decoder's ask, which the page did not count before, is counted by the same rule. --- packages/stream/rust/abr.rs | 30 ++++++++++-- packages/stream/rust/recovery.rs | 79 +++++++++++++++++++++++++++++++- packages/stream/rust/session.rs | 48 +++++++++++-------- 3 files changed, 135 insertions(+), 22 deletions(-) diff --git a/packages/stream/rust/abr.rs b/packages/stream/rust/abr.rs index cb3aac2..eab1f68 100644 --- a/packages/stream/rust/abr.rs +++ b/packages/stream/rust/abr.rs @@ -31,7 +31,9 @@ struct Since { decode_lat: (u64, u32), encode_lat: (u64, u32), host_timings: Vec, + /// Keyframe asks a loss stood behind, and those the page's own hold made. keyframe_asks: u32, + starved_keyframe_asks: u32, /// The newest host pipeline gap, ms: one discard per window. gap_ms: Option, } @@ -44,6 +46,15 @@ impl Since { } } + fn on_keyframe_ask(&mut self, starved: bool) { + let n = if starved { + &mut self.starved_keyframe_asks + } else { + &mut self.keyframe_asks + }; + *n += 1; + } + /// One `PipelineGap`. The newest wins: a tick discards one window however many landed. fn on_pipeline_gap(&mut self, gap_ms: u32) { self.gap_ms = Some(gap_ms); @@ -59,6 +70,7 @@ impl Since { encode_lat: self.encode_lat, host_timings: self.host_timings, keyframe_asks: self.keyframe_asks, + starved_keyframe_asks: self.starved_keyframe_asks, ..Inputs::default() } } @@ -161,9 +173,10 @@ impl Abr { } } - /// A recovery keyframe ask went out. - pub fn on_keyframe_ask(&mut self) { - self.since.keyframe_asks += 1; + /// A keyframe ask went out. `starved`: the page's own hold made it, with nothing lost + /// ([`crate::recovery::Freeze`]); the driver reads only the others as a reason to cut. + pub fn on_keyframe_ask(&mut self, starved: bool) { + self.since.on_keyframe_ask(starved); } /// Tick the driver with what was read since the last pump, and send what it asks for. @@ -274,6 +287,17 @@ mod tests { assert_eq!(since.into_inputs().gap_ms, None); } + /// A starved decoder's asks reach the driver apart from those a loss stood behind. + #[test] + fn starved_asks_are_counted_apart() { + let mut since = Since::default(); + for starved in [true, false, true] { + since.on_keyframe_ask(starved); + } + let inputs = since.into_inputs(); + assert_eq!((inputs.keyframe_asks, inputs.starved_keyframe_asks), (1, 2)); + } + /// A rate the control stream took is sent; one it did not goes back, as every probe does. #[test] fn an_unsent_rate_goes_back_to_the_driver() { diff --git a/packages/stream/rust/recovery.rs b/packages/stream/rust/recovery.rs index 5cdafd9..e2168dd 100644 --- a/packages/stream/rust/recovery.rs +++ b/packages/stream/rust/recovery.rs @@ -1,12 +1,52 @@ //! Which access units the page decodes after a loss, and what it asks the host for. The hold -//! itself is `punktfunk_core::reanchor`; this is the frame-index bookkeeping around it. +//! itself is `punktfunk_core::reanchor`; this is the frame-index bookkeeping around it, and who +//! raised the hold. use punktfunk_core::packet::{ FLAG_SOF, RFI_MAX_RANGE, USER_FLAG_RECOVERY_ANCHOR, USER_FLAG_RECOVERY_POINT, }; use punktfunk_core::quic::v2::dgram::Feedback; +use punktfunk_core::reanchor::ReanchorGate; use std::time::{Duration, Instant}; +/// The post-loss hold, and whether the page raised it on an unbroken stream rather than a loss. +/// +/// The page's own holds come from its decoder: an error streak, or a rebuilt decoder's ask. +/// Their keyframe asks, the hold's re-asks included, go to the bitrate driver as a starved +/// decoder's, which a lower rate cannot feed. Core's `RecoveryDriver` keeps the same flag. +pub struct Freeze { + gate: ReanchorGate, + own: bool, +} + +impl Default for Freeze { + /// Open, for a session whose reassembler has dropped nothing yet. + fn default() -> Freeze { + Freeze { + gate: ReanchorGate::new(0), + own: false, + } + } +} + +impl Freeze { + /// One gate event; `page` when the page's decoder raised it, not the wire. An event that + /// arms decides who holds: the page keeps a hold it raised on an open gate or already held. + pub fn step(&mut self, page: bool, event: impl FnOnce(&mut ReanchorGate) -> T) -> T { + let (arms, held) = (self.gate.arms(), self.gate.is_holding()); + let out = event(&mut self.gate); + if self.gate.arms() != arms { + self.own = page && (self.own || !held); + } + out + } + + /// A keyframe ask this hold makes now is a starved decoder's, not a loss's. + pub fn starved(&self) -> bool { + self.own + } +} + /// An open ask repeats this often: a feedback datagram can be lost. const REPEAT: Duration = Duration::from_millis(16); /// An ask nothing answered closes here; the hold's own re-ask takes over. @@ -199,6 +239,43 @@ mod tests { assert!(asks.resend(t0 + GIVE_UP).is_none(), "given up"); } + /// A hold the page raised on an open gate asks as starved, its backstop re-ask too, until a + /// loss re-arms it. + #[test] + fn a_page_hold_asks_starved_until_a_loss_re_arms_it() { + use punktfunk_core::reanchor::REANCHOR_FREEZE_MAX; + let t0 = Instant::now(); + let mut f = Freeze::default(); + f.step(true, |g| g.arm(t0)); + assert!(f.starved(), "a rebuilt decoder's ask"); + assert!(f.step(false, |g| g.poll(0, t0 + REANCHOR_FREEZE_MAX))); + assert!(f.starved(), "the backstop re-ask of the page's hold"); + f.step(false, |g| g.arm_expecting_drops(t0, 2)); + assert!(!f.starved(), "a gap makes the hold the loss's"); + assert!(f.step(false, |g| g.poll(9, t0 + REANCHOR_FREEZE_MAX))); + assert!(!f.starved(), "a drop climb is a loss"); + } + + /// The page's streak inside a loss's hold leaves it the loss's; on an open gate it is the + /// page's. + #[test] + fn a_no_output_streak_is_the_page_s_only_on_an_open_gate() { + let t0 = Instant::now(); + let mut f = Freeze::default(); + f.step(false, |g| g.arm_expecting_drops(t0, 1)); + let trips = (0..3) + .filter(|_| f.step(true, |g| g.on_no_output(t0))) + .count(); + assert_eq!(trips, 1); + assert!(!f.starved(), "held by a loss"); + f.step(false, |g| g.on_decoded(u32::from(FLAG_SOF), true, t0)); + let trips = (0..3) + .filter(|_| f.step(true, |g| g.on_no_output(t0))) + .count(); + assert_eq!(trips, 1); + assert!(f.starved(), "raised on an open gate"); + } + #[test] fn the_index_wraps() { let (mut next, mut pending) = (Some(u32::MAX), None); diff --git a/packages/stream/rust/session.rs b/packages/stream/rust/session.rs index 5b30234..49f9535 100644 --- a/packages/stream/rust/session.rs +++ b/packages/stream/rust/session.rs @@ -15,7 +15,7 @@ use crate::abr::Abr; use crate::credential; -use crate::recovery::{self, Ask, Asks, Step}; +use crate::recovery::{self, Ask, Asks, Freeze, Step}; use crate::transport::{send_feedback, WebTransportDatagrams}; use punktfunk_core::config::{CompositorPref, GamepadPref, Mode, Role}; use punktfunk_core::error::PunktfunkError; @@ -31,7 +31,7 @@ use punktfunk_core::quic::{ LaunchOutcome, LinkFacts, PairChallenge, PairResult, PipelineGap, Reconfigure, Reconfigured, Refused, SetBitrate, }; -use punktfunk_core::reanchor::{GateVerdict, ReanchorGate}; +use punktfunk_core::reanchor::GateVerdict; use punktfunk_core::session::{MediaV2, Session}; use std::cell::RefCell; use std::collections::VecDeque; @@ -109,8 +109,8 @@ struct Client { /// The connection dot's last three windows. quality: hud::QualityState, /// The shared post-loss freeze: the page withholds decoded frames until the stream - /// re-anchors, as every native client does. - gate: ReanchorGate, + /// re-anchors, as every native client does. It knows whether the page raised it. + freeze: Freeze, /// The frame index the next access unit should carry; a jump is a loss. next_index: Option, /// A lost range not yet asked for (the throttle held it), widened by later gaps. @@ -158,7 +158,7 @@ impl Client { snap: None, text: String::new(), quality: hud::QualityState::default(), - gate: ReanchorGate::new(0), + freeze: Freeze::default(), next_index: None, pending_rfi: None, asks: Asks::default(), @@ -746,7 +746,7 @@ fn on_server_hello(c: &mut Client, server: ServerHello) { c.grants = welcome.grants; c.access_secs = welcome.expires_in_secs; c.access_seq = c.access_seq.wrapping_add(1); - c.gate = ReanchorGate::new(0); + c.freeze = Freeze::default(); c.next_index = None; c.pending_rfi = None; c.asks = Asks::default(); @@ -795,7 +795,7 @@ pub extern "C" fn pf_session_pump() -> u32 { receipts, codec, frames, - gate, + freeze, next_index, pending_rfi, last_ask, @@ -823,7 +823,11 @@ pub extern "C" fn pf_session_pump() -> u32 { // every frame, so a wave can heal it) and ask for the range. match recovery::on_index(next_index, pending_rfi, frame.frame_index) { Step::Deliver => {} - Step::Gap(gap) => gate.arm_expecting_drops(Instant::now(), u64::from(gap)), + Step::Gap(gap) => { + freeze.step(false, |g| { + g.arm_expecting_drops(Instant::now(), u64::from(gap)) + }); + } Step::Straggler => continue, } asks.on_frame(frame.frame_index, frame.flags); @@ -859,14 +863,16 @@ pub extern "C" fn pf_session_pump() -> u32 { // no re-anchor for 500 ms re-asks: both want an IDR. Otherwise the lost range goes out // as RFI, which a capable host answers with a clean P or a wave instead of an IDR. let now = Instant::now(); - let want_keyframe = gate.poll(session.stats().frames_dropped, now); + let dropped = session.stats().frames_dropped; + let want_keyframe = freeze.step(false, |g| g.poll(dropped, now)); let throttled = last_ask.is_some_and(|t| now.duration_since(t) < ASK_THROTTLE); if !throttled { if let Some(ask) = recovery::take_ask(want_keyframe, pending_rfi) { *last_ask = Some(now); send_feedback(&asks.ask(ask, now)); + // A range too wide for RFI asks for a keyframe too, and that one is a loss's. if let (Ask::Keyframe, Some(abr)) = (ask, abr.as_mut()) { - abr.on_keyframe_ask(); + abr.on_keyframe_ask(want_keyframe && freeze.starved()); } } } @@ -893,40 +899,46 @@ pub extern "C" fn pf_gate_decoded(flags: u32, key: i32) -> i32 { CLIENT.with(|c| { let verdict = c .borrow_mut() - .gate - .on_decoded(flags, key != 0, Instant::now()); + .freeze + .step(false, |g| g.on_decoded(flags, key != 0, Instant::now())); i32::from(verdict == GateVerdict::Present) }) } /// The decoder refused an access unit or failed. Three in a row hold the picture and ask for an -/// IDR, the same streak the native clients count. +/// IDR, the same streak the native clients count. The bitrate driver hears the ask as a starved +/// decoder's unless a loss already holds the picture. #[unsafe(no_mangle)] pub extern "C" fn pf_gate_no_output() { CLIENT.with(|c| { - let mut c = c.borrow_mut(); + let c = &mut *c.borrow_mut(); let now = Instant::now(); - if c.gate.on_no_output(now) && c.phase == Phase::Live { + if c.freeze.step(true, |g| g.on_no_output(now)) && c.phase == Phase::Live { c.last_ask = Some(now); send_feedback(&c.asks.ask(Ask::Keyframe, now)); if let Some(abr) = c.abr.as_mut() { - abr.on_keyframe_ask(); + abr.on_keyframe_ask(c.freeze.starved()); } } }); } /// The page rebuilt its decoder, which can resume only on an IDR: hold, and ask for one now. +/// The bitrate driver hears the ask as a starved decoder's unless a loss already holds the +/// picture. #[unsafe(no_mangle)] pub extern "C" fn pf_request_keyframe() { CLIENT.with(|c| { - let mut c = c.borrow_mut(); + let c = &mut *c.borrow_mut(); let now = Instant::now(); - c.gate.arm(now); + c.freeze.step(true, |g| g.arm(now)); c.pending_rfi = None; if c.phase == Phase::Live { c.last_ask = Some(now); send_feedback(&c.asks.ask(Ask::Keyframe, now)); + if let Some(abr) = c.abr.as_mut() { + abr.on_keyframe_ask(c.freeze.starved()); + } } }); } From 905aeebb008e7a7316b391241a163ba2c6030e14 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:31:01 +0100 Subject: [PATCH 04/48] fix(web/console): keep the web shell out when a launch hold ends A launch hold that ended short of the stream (B before the dial landed, a failed start, a stream that closed under the hold) handed the web shell the raw screen. Home, the library or an error card then covered the console as an opaque page while the pad drove the console underneath, until the next render. The hold's end now goes through the same filter as every render: the web shell draws only its own sheets. --- apps/web/src/ui/console.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/web/src/ui/console.ts b/apps/web/src/ui/console.ts index 185027c..6e56458 100644 --- a/apps/web/src/ui/console.ts +++ b/apps/web/src/ui/console.ts @@ -286,12 +286,17 @@ export class ConsoleUi implements Ui { this.screen = screen; // A page-level change of screen counts as a move, as a key does. if (was?.kind !== screen.kind) this.inputAt = performance.now(); - const web = (WEB_SCREENS.has(screen.kind) && !this.held) || !this.started; - this.fallback.render(web ? screen : { kind: "console" }); + this.fallback.render(this.shellScreen(screen)); if (!this.started) return; this.follow(was, screen); } + /** What the web shell draws for `screen`: one of its own sheets while no launch hold covers + * it, every screen before the console is up, and otherwise nothing. */ + private shellScreen(screen: Screen): Screen { + return (WEB_SCREENS.has(screen.kind) && !this.held) || !this.started ? screen : { kind: "console" }; + } + destroy(): void { this.mounted = false; this.endWake(false); @@ -359,11 +364,12 @@ export class ConsoleUi implements Ui { } if (drew) this.cost(now - before, now, width, height); this.state = this.engine.console.state(); - // A launch hold covers the stream until the game is up: the page's own overlay waits. + // A launch hold covers the stream until the game is up: the page's own overlay waits. A + // hold that ends short of the stream (cancelled, failed) leaves a console screen. const holding = !!(this.state & CONSOLE_STATE.HOLDS_LAUNCH); if (holding !== this.held && this.screen) { this.held = holding; - this.fallback.render(holding ? { kind: "console" } : this.screen); + this.fallback.render(this.shellScreen(this.screen)); } const showing = this.showing(); // The canvas takes the pointer only while the console is on it; over a stream the pointer From c038f8d18b588a20ece18b55ce1031c54d72ea7e Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:31:39 +0100 Subject: [PATCH 05/48] fix(web/console): let the page's settings sheet show in console mode The sidebar on the accept, trust and link sheets and the stream overlay both open the page's settings. Console mode drew nothing for that screen and left settingsOpen set, so every later screen read as settings. Opened over a stream, the end of that stream never reached the console: a hidden canvas over an empty page until a reload. The sheet is now one of the web shell's screens, closed by its own Done, Back or sidebar. The console follows the flow around it, so a stream that ends while the sheet is open still ends there. --- apps/web/src/ui/console.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/web/src/ui/console.ts b/apps/web/src/ui/console.ts index 6e56458..0c2a8c7 100644 --- a/apps/web/src/ui/console.ts +++ b/apps/web/src/ui/console.ts @@ -102,8 +102,9 @@ const RED_KEY = 403; * measuring page prices it once more: a remote with no spare key still gets every screen. */ const QUIET_MS = 15_000; -/** The screens the console cannot draw; everything else is the console's. */ -const WEB_SCREENS = new Set(["accept", "trust", "link", "streaming"]); +/** The screens the console cannot draw, and the page's settings sheet, which the web shell's + * sidebar and stream overlay open and only it can close; everything else is the console's. */ +const WEB_SCREENS = new Set(["accept", "trust", "link", "streaming", "settings"]); /** How long the console waits on a `FetchProfiles` answer before it dials without one * (`pf_console_ui` `PROFILES_WAIT`). */ @@ -130,6 +131,9 @@ export class ConsoleUi implements Ui { /** The web shell was told to stand aside for a launch hold over the stream. */ private held = false; private screen: Screen | null = null; + /** The last screen `follow` saw. The settings sheet is no step of a flow, so the screen after + * it follows the one before it: a stream it opened over still ends. */ + private followed: Screen | null = null; /** Host rows by the key the console knows them by, and the origin each stands for. */ private origins = new Map(); private hosts: HostCard[] = []; @@ -287,8 +291,9 @@ export class ConsoleUi implements Ui { // A page-level change of screen counts as a move, as a key does. if (was?.kind !== screen.kind) this.inputAt = performance.now(); this.fallback.render(this.shellScreen(screen)); - if (!this.started) return; - this.follow(was, screen); + if (!this.started || screen.kind === "settings") return; + this.follow(this.followed, screen); + this.followed = screen; } /** What the web shell draws for `screen`: one of its own sheets while no launch hold covers From 7ba979fbd979bf9b6794e25496c66d744f24e709 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:32:15 +0100 Subject: [PATCH 06/48] fix(web/console): drop pending steps when a connect ends A Launch, Pair or RequestAccess for a host not yet reached waits for that host to come up. When the connect failed, or was backed out on the page's accept or trust sheet, the step stayed. The next visit to that host's library started the old stream unasked, sent an old PIN, or knocked again. An error or a return home now drops them, as Back on the console already does. In console mode the page shows home only with the engine on no host, so no flow passes through it mid-way. --- apps/web/src/ui/console.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/apps/web/src/ui/console.ts b/apps/web/src/ui/console.ts index 0c2a8c7..02bf63e 100644 --- a/apps/web/src/ui/console.ts +++ b/apps/web/src/ui/console.ts @@ -624,6 +624,8 @@ export class ConsoleUi implements Ui { this.hosts = s.hosts; this.pushHosts(); if (s.error) this.fetchFailed("The host did not answer", s.error); + // Home only with the engine on no host: what the console asked for will not follow. + this.dropPending(); if (was?.kind === "streaming" || (this.dialled && was?.kind !== "home")) { this.dialled = false; c.phase(3); @@ -662,6 +664,7 @@ export class ConsoleUi implements Ui { return; case "error": this.dialled = false; + this.dropPending(); this.fetchFailed(s.head, s.text); c.phase(2, `${s.head}. ${s.text}`); this.pushPair("Idle"); @@ -671,6 +674,15 @@ export class ConsoleUi implements Ui { } } + /** Forget the steps waiting on a connect that ended, so a later visit to that host does not + * play, pair or knock unasked. */ + private dropPending(): void { + this.pendingPlay = null; + this.profilesAsk = null; + this.pendingPin = null; + this.pendingKnock = null; + } + /** The shelf the console is waiting on could not be reached: say so on it, with a retry. */ private fetchFailed(title: string, body: string): void { if (!this.fetching) return; @@ -851,10 +863,7 @@ export class ConsoleUi implements Ui { return; } if (a === "CancelConnect") { - this.pendingPlay = null; - this.profilesAsk = null; - this.pendingPin = null; - this.pendingKnock = null; + this.dropPending(); this.dialled = false; // A request for access is withdrawn, so a late approval admits nothing. if (this.engine.current.kind === "awaiting-approval") this.actions?.cancelRequest(); From 27403ef90afbb30b72fb1494708f0697b0a4c361 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:33:20 +0100 Subject: [PATCH 07/48] fix(web/console): tell apart listed hosts the page gives no plane A visitor without a pass gets the page server's hosts without their plane address. Every such row then took the page's own hostname as its address and key, so the console's commands for any of them reached the last one listed. A row behind the page's server with no plane is now addressed by its route, which no two hosts share. Rows with a plane, and hosts dialled directly, keep their address. The row helpers move out of ConsoleUi so a test can pin this without a browser. --- apps/web/src/ui/console-hosts.test.ts | 45 ++++++++++++++++++++++++ apps/web/src/ui/console-hosts.ts | 50 +++++++++++++++++++++++++++ apps/web/src/ui/console.ts | 34 +++--------------- 3 files changed, 99 insertions(+), 30 deletions(-) create mode 100644 apps/web/src/ui/console-hosts.test.ts create mode 100644 apps/web/src/ui/console-hosts.ts diff --git a/apps/web/src/ui/console-hosts.test.ts b/apps/web/src/ui/console-hosts.test.ts new file mode 100644 index 0000000..adb697b --- /dev/null +++ b/apps/web/src/ui/console-hosts.test.ts @@ -0,0 +1,45 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { hostRow, originFor } from "./console-hosts.ts"; +import type { HostCard } from "./types.ts"; + +/** The rows as `ConsoleUi.pushHosts` builds them, and its key → origin map. */ +function rows(cards: HostCard[]) { + const all = cards.map((h) => hostRow(h, h.fingerprint)); + return { all, origins: new Map(all.map((r, i) => [r.key, cards[i]!.origin])) }; +} + +test("hosts the page's server lists without a plane each get their own row", () => { + const cards: HostCard[] = [ + { origin: "https://page.example/h/desk", name: "desk" }, + { origin: "https://page.example/h/den", name: "den" }, + ]; + const { all, origins } = rows(cards); + assert.notEqual(all[0]!.key, all[1]!.key); + assert.notEqual(all[0]!.addr, all[1]!.addr); + // A Pair names the key; a Launch for an unpaired host names only the address. + for (const [i, h] of cards.entries()) { + assert.equal(originFor(origins, cards, all[i]!.key, ""), h.origin); + assert.equal(originFor(origins, cards, "", all[i]!.addr), h.origin); + } +}); + +test("a host with a plane, and one dialled directly, keep their address", () => { + const cards: HostCard[] = [ + { origin: "https://page.example/h/desk", name: "desk", plane: "192.168.1.20" }, + { origin: "https://192.168.1.31:47990", name: "deck" }, + ]; + const { all, origins } = rows(cards); + assert.equal(all[0]!.addr, "192.168.1.20"); + assert.equal(all[1]!.addr, "192.168.1.31"); + assert.equal(all[1]!.key, "192.168.1.31:47990"); + assert.equal(originFor(origins, cards, "", "192.168.1.20"), cards[0]!.origin); +}); + +test("a paired row is keyed by its fingerprint", () => { + const cards: HostCard[] = [{ origin: "https://page.example/h/desk", name: "desk", fingerprint: "ab".repeat(32) }]; + const { all, origins } = rows(cards); + assert.equal(all[0]!.key, "ab".repeat(32)); + assert.equal(all[0]!.paired, true); + assert.equal(originFor(origins, cards, "ab".repeat(32), ""), cards[0]!.origin); +}); diff --git a/apps/web/src/ui/console-hosts.ts b/apps/web/src/ui/console-hosts.ts new file mode 100644 index 0000000..fc888f7 --- /dev/null +++ b/apps/web/src/ui/console-hosts.ts @@ -0,0 +1,50 @@ +// A known host as the console's Home draws it, and back: the row the page pushes, and the origin +// a row's key or address names when the console raises a command for it. + +import type { ConsoleHostRow } from "@punktfunk/stream"; +import type { HostCard } from "./types.ts"; + +/** Where a row says its host is: its plane address, else its own hostname. A host the page's + * server routes without telling its plane is its route, which no two hosts share. */ +export function hostAddr(h: HostCard): string { + const url = new URL(h.origin); + return h.plane ?? (url.pathname === "/" ? url.hostname : h.origin); +} + +/** The console's row for `h`, pinned to `fp` when this browser paired with it. */ +export function hostRow(h: HostCard, fp: string | undefined): ConsoleHostRow { + const addr = hostAddr(h); + const mgmt = Number(new URL(h.origin).port) || 47990; + return { + key: fp ?? `${addr}:${mgmt}`, + id: h.origin, + name: h.label ?? h.name ?? addr, + addr, + port: 9778, + fp_hex: fp ?? "", + paired: !!fp, + saved: true, + online: h.reach === "ok", + mgmt_port: mgmt, + can_wake: !!h.wake && h.reach !== "ok", + last_used: h.seen ? Math.floor(h.seen / 1000) : null, + os: "", + pin: null, + bound_preset: null, + profile: h.profile ?? null, + }; +} + +/** The origin a console key or address stands for: `origins` maps each row's key to its origin, + * and `cards` are the rows' hosts. */ +export function originFor( + origins: ReadonlyMap, + cards: readonly HostCard[], + key: string, + addr: string, +): string | undefined { + const byKey = origins.get(key); + if (byKey) return byKey; + for (const [k, o] of origins) if (k.startsWith(`${addr}:`)) return o; + return cards.find((h) => hostAddr(h) === addr)?.origin; +} diff --git a/apps/web/src/ui/console.ts b/apps/web/src/ui/console.ts index 02bf63e..1a34cf4 100644 --- a/apps/web/src/ui/console.ts +++ b/apps/web/src/ui/console.ts @@ -37,6 +37,7 @@ import { tvBack, } from "@punktfunk/stream"; import { FrameMeter, type Row, type Step, Sweep, table } from "./console-cost.ts"; +import { hostRow, originFor } from "./console-hosts.ts"; import { coverBytes } from "./cover.ts"; import type { Actions, HostCard, Screen, Ui } from "./types.ts"; @@ -791,42 +792,18 @@ export class ConsoleUi implements Ui { const known = new Map(this.engine.knownHosts().map((h) => [h.origin, h])); this.origins.clear(); const rows = this.hosts.map((h) => { - const row = this.rowOf(h, known.get(h.origin)?.fingerprint ?? h.fingerprint); + const row = hostRow(h, known.get(h.origin)?.fingerprint ?? h.fingerprint); this.origins.set(row.key, h.origin); return row; }); this.engine.console.push(CONSOLE_PUSH.HOSTS, rows); } - private rowOf(h: HostCard, fp: string | undefined): ConsoleHostRow { - const url = new URL(h.origin); - const addr = h.plane ?? url.hostname; - const mgmt = Number(url.port) || 47990; - return { - key: fp ?? `${addr}:${mgmt}`, - id: h.origin, - name: h.label ?? h.name ?? addr, - addr, - port: 9778, - fp_hex: fp ?? "", - paired: !!fp, - saved: true, - online: h.reach === "ok", - mgmt_port: mgmt, - can_wake: !!h.wake && h.reach !== "ok", - last_used: h.seen ? Math.floor(h.seen / 1000) : null, - os: "", - pin: null, - bound_preset: null, - profile: h.profile ?? null, - }; - } - /** The row for an origin, from the home list or, for one only just added, from what the * engine knows. */ private row(origin: string): ConsoleHostRow { const card = this.hosts.find((h) => h.origin === origin) ?? ({ origin } as HostCard); - return this.rowOf(card, this.fingerprint(origin)); + return hostRow(card, this.fingerprint(origin)); } private fingerprint(origin: string): string | undefined { @@ -1096,10 +1073,7 @@ export class ConsoleUi implements Ui { /** The origin a console key or address stands for. */ private originOf(key: string, addr: string): string | undefined { - const byKey = this.origins.get(key); - if (byKey) return byKey; - for (const [k, o] of this.origins) if (k.startsWith(`${addr}:`)) return o; - return this.hosts.find((h) => (h.plane ?? new URL(h.origin).hostname) === addr)?.origin; + return originFor(this.origins, this.hosts, key, addr); } } From 19008777985a938487351d0287c536a6f6c2e00c Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:34:08 +0100 Subject: [PATCH 08/48] fix(web/console): take the wake card down when no wake went out A wake the page's server refused, or one it never sent, came out as a toast while the console's wake card went on counting to a minute and then said the host never came back. The wake action now resolves whether a packet went out, and the console takes its card down when none did. A sent wake that the host does not answer still ends on the card's own timeout, which offers to try again. --- apps/web/src/app.ts | 14 +++++++++----- apps/web/src/ui/console.ts | 10 +++++++--- apps/web/src/ui/shell.tsx | 2 +- apps/web/src/ui/types.ts | 5 +++-- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/apps/web/src/app.ts b/apps/web/src/app.ts index 1992977..7ce363a 100644 --- a/apps/web/src/app.ts +++ b/apps/web/src/app.ts @@ -204,7 +204,7 @@ class App { hostAction: (id) => void this.hostAction(id), endGame: () => void this.endGame(), sendLog: () => void this.sendLog(), - wake: (origin) => void this.wake(origin), + wake: (origin) => this.wake(origin), disconnect: (quit) => { this.menuOpen = false; engine.leave(quit); @@ -650,18 +650,20 @@ class App { /** * Wake a host through the page's server, which sits on its network where a browser cannot send * a magic packet. Then ask after it every few seconds, as it boots, until it answers or a minute - * has gone. + * has gone. `false` when no packet went out: refused or failed, which the page has said. */ - private async wake(origin: string): Promise { + private async wake(origin: string): Promise { const url = this.configured.get(origin)?.wake; - if (!url || this.waking.has(origin)) return; + if (!url) return false; + if (this.waking.has(origin)) return true; this.waking.add(origin); this.redrawLists(); try { const r = await serverFetch()(url, { method: "POST" }); if (r.status === 403 || r.status === 429) { const why = r.status === 429 ? sentence(TOO_MANY) : `Only a browser paired with ${this.nameOf(origin)} can wake it from this page.`; - return this.ui.notify(why, "error"); + this.ui.notify(why, "error"); + return false; } if (!r.ok) throw new Error(`wake refused (${r.status})`); let up = false; @@ -672,9 +674,11 @@ class App { up = now === "ok"; } if (!up) this.ui.notify(`${this.nameOf(origin)} didn't answer within a minute of waking. It may still be starting.`, "error"); + return true; } catch (e) { console.warn("punktfunk: wake", e); this.ui.notify(`Couldn't wake ${this.nameOf(origin)} — this page's server didn't send the wake packet.`, "error"); + return false; } finally { this.waking.delete(origin); this.redrawLists(); diff --git a/apps/web/src/ui/console.ts b/apps/web/src/ui/console.ts index 1a34cf4..4d7ea00 100644 --- a/apps/web/src/ui/console.ts +++ b/apps/web/src/ui/console.ts @@ -1042,7 +1042,8 @@ export class ConsoleUi implements Ui { } /** Wake a host and keep the console's wake card current: seconds while the page's own wake - * polls it, then online (the console connects by itself if asked to) or timed out. */ + * polls it, then online (the console connects by itself if asked to) or timed out. A wake + * that never went out takes the card down; the page's toast says why. */ private startWake(req: ConsoleReq, key: string, origin: string, then: boolean): void { this.endWake(false); const name = this.hosts.find((h) => h.origin === origin)?.name ?? key; @@ -1060,8 +1061,11 @@ export class ConsoleUi implements Ui { if (online && !then) window.setTimeout(() => this.endWake(true), 1500); } }; - this.waking = { req, key, origin, name, then, since, timer: window.setInterval(tick, 1000) }; - this.actions?.wake(origin); + const waking = { req, key, origin, name, then, since, timer: window.setInterval(tick, 1000) }; + this.waking = waking; + void this.actions?.wake(origin).then((sent) => { + if (!sent && this.waking === waking) this.endWake(true); + }); tick(); } diff --git a/apps/web/src/ui/shell.tsx b/apps/web/src/ui/shell.tsx index a31e748..6573f6e 100644 --- a/apps/web/src/ui/shell.tsx +++ b/apps/web/src/ui/shell.tsx @@ -80,7 +80,7 @@ export class WebShell implements Ui { export const noop: Actions = { connect() {}, streamDesktop() {}, browse() {}, navigate() {}, pair() {}, requestAccess() {}, cancelRequest() {}, retry() {}, back() {}, play() {}, - forget() {}, wake() {}, disconnect() {}, + forget() {}, wake: async () => false, disconnect() {}, openTools() {}, hostAction() {}, endGame() {}, sendLog() {}, followLink() {}, copyLink() {}, setAdding() {}, rename() {}, openSettings() {}, setSettings() {}, toggleCapture() {}, diff --git a/apps/web/src/ui/types.ts b/apps/web/src/ui/types.ts index 636628c..9f7ff89 100644 --- a/apps/web/src/ui/types.ts +++ b/apps/web/src/ui/types.ts @@ -221,8 +221,9 @@ export interface Actions { * running one. Unsaved progress is lost, so the renderer asks first. */ endGame(): void; sendLog(): void; - /** Send a host the magic packet, through the page's server. */ - wake(origin: string): void; + /** Send a host the magic packet, through the page's server. `false` when none went out: the + * page has said why. */ + wake(origin: string): Promise; /** Name a host something this browser will remember. An empty label drops the name. */ rename(origin: string, label: string): void; /** Leave the stream for the page it started from. `quit` ends the title too; without it the From c5b4c104859969564059e2c69d8ddb450968b8e5 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:34:17 +0100 Subject: [PATCH 09/48] fix(web/console): clear the cover scratch before each cover Every cover and profile picture is drawn onto one shared canvas that keeps the last image. A picture with transparent areas showed the previous cover through them. --- apps/web/src/ui/cover.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/web/src/ui/cover.ts b/apps/web/src/ui/cover.ts index 0494c3d..df2d9dc 100644 --- a/apps/web/src/ui/cover.ts +++ b/apps/web/src/ui/cover.ts @@ -75,6 +75,8 @@ export async function coverBytes(blob: Blob): Promise { const { width: w, height: h } = bitmap; scratch ??= new OffscreenCanvas(COVER_W, COVER_H).getContext("2d", { willReadFrequently: true }); if (!scratch) return null; + // The scratch still holds the last cover, which a transparent one would show through. + scratch.clearRect(0, 0, w, h); scratch.drawImage(bitmap, 0, 0); bitmap.close(); return bmp(w, h, scratch.getImageData(0, 0, w, h).data); From cd8716b8bc2eaa1b66f61ca520ccf9f1d90efe3c Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:34:29 +0100 Subject: [PATCH 10/48] fix(web/console): open the full keyboard for text fields on a TV Every field that did not take digits asked the set for the decimal pad: a number pad with a dot. The device name on the pair screen, the library's search and a host's name take letters. Digit fields keep the number pad. Every other field gets the full keyboard, which types the dot of an address too. --- apps/web/src/ui/console.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/web/src/ui/console.ts b/apps/web/src/ui/console.ts index 4d7ea00..201679f 100644 --- a/apps/web/src/ui/console.ts +++ b/apps/web/src/ui/console.ts @@ -543,8 +543,8 @@ export class ConsoleUi implements Ui { } this.imeText = field.text; this.ime.value = field.text; - // A PIN or a port wants the number pad; an address wants the dot beside the digits. - this.ime.inputMode = field.digits ? "numeric" : "decimal"; + // A PIN or a port wants the number pad; a name, a title or an address the full keyboard. + this.ime.inputMode = field.digits ? "numeric" : "text"; this.ime.focus(); this.ime.setSelectionRange(field.text.length, field.text.length); } @@ -1086,7 +1086,6 @@ export class ConsoleUi implements Ui { function imeField(): HTMLInputElement { const input = document.createElement("input"); input.type = "text"; - input.inputMode = "decimal"; input.autocomplete = "off"; input.tabIndex = -1; input.setAttribute("aria-hidden", "true"); From 054f7f248c432f675b9a7a013e6b22bd10130984 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:28:08 +0100 Subject: [PATCH 11/48] fix(stream/input): release a held key wherever its keyup lands A key the host holds was let go only when its keyup reached the picture. Opening the quick menu with its chord moves focus into the menu before Ctrl, Alt and Shift come up, so their keyups were kept local and the host held all three. A held S, D, O, Q, V, M or Enter released under the chord modifiers matched a chord and was dropped the same way. macOS sends no keyup for a key pressed while Cmd is down, so Cmd+C left C held and repeating on the host. Keys that go down under Meta are now let go when Meta comes up; keys held from before it stay down. --- packages/stream/src/input.test.ts | 63 +++++++++++++++++++++++++++++++ packages/stream/src/input.ts | 44 ++++++++++++++++----- 2 files changed, 98 insertions(+), 9 deletions(-) diff --git a/packages/stream/src/input.test.ts b/packages/stream/src/input.test.ts index 107b48d..800cc79 100644 --- a/packages/stream/src/input.test.ts +++ b/packages/stream/src/input.test.ts @@ -162,3 +162,66 @@ test("a wheel turned down scrolls down: the wire is up-positive, the browser dow [6, 1, 120], ]); }); + +/** A pipe whose sends are recorded as `kind:code`, and a key event aimed at `target`. */ +const keyPipe = () => { + const sent: string[] = []; + const mod = { + _pf_input: (kind: number, code: number) => sent.push(`${kind}:${code}`), + _pf_gamepad: (pad: number, ...rest: number[]) => sent.push(`pad${pad}:${rest.join(",")}`), + _pf_gamepad_arrival: () => {}, + }; + const pipe = new InputPipe(mod as never, {} as HTMLCanvasElement, { streamWidth: 1920, streamHeight: 1080, pointer: "absolute", deadzone: 0.05 }); + const key = (code: string, down: boolean, target: unknown, mods: Partial = {}) => { + let prevented = false; + (pipe as unknown as { key(e: unknown, down: boolean): void }).key( + { code, target, repeat: false, ctrlKey: false, altKey: false, shiftKey: false, metaKey: false, ...mods, preventDefault: () => (prevented = true) }, + down, + ); + return prevented; + }; + return { sent, pipe, key }; +}; + +const PICTURE = { tagName: "CANVAS", closest: () => null }; +const IN_MENU = { tagName: "BUTTON", closest: () => ({}) }; +const CHORD = { ctrlKey: true, altKey: true, shiftKey: true }; + +test("a key the host holds is let go wherever its release lands", () => { + const { sent, key } = keyPipe(); + // Ctrl+Alt+Shift+O opens the menu, which takes focus before the modifiers come up. + key("ControlLeft", true, PICTURE, { ctrlKey: true }); + key("AltLeft", true, PICTURE, { ctrlKey: true, altKey: true }); + key("ShiftLeft", true, PICTURE, CHORD); + key("KeyO", true, PICTURE, CHORD); + assert.equal(key("KeyO", false, IN_MENU, CHORD), false, "never sent, so the menu keeps it"); + assert.equal(key("ShiftLeft", false, IN_MENU, { ctrlKey: true, altKey: true }), false, "the menu's event keeps its default"); + key("AltLeft", false, IN_MENU, { ctrlKey: true }); + key("ControlLeft", false, IN_MENU); + assert.deepEqual(sent, ["0:162", "0:164", "0:160", "1:160", "1:164", "1:162"]); + + sent.length = 0; + key("KeyW", true, IN_MENU); + key("KeyW", false, IN_MENU); + assert.deepEqual(sent, [], "a key that went down in the menu stays there"); + + // S held, then released with the chord's modifiers down: the release is not the stats chord. + key("KeyS", true, PICTURE); + assert.equal(key("KeyS", false, PICTURE, CHORD), true); + assert.deepEqual(sent, ["0:83", "1:83"]); +}); + +test("Meta's release lets go of the keys pressed under it, not those held before it", () => { + const { sent, key } = keyPipe(); + key("KeyW", true, PICTURE); + key("MetaLeft", true, PICTURE, { metaKey: true }); + key("KeyV", true, PICTURE, { metaKey: true }); + key("KeyV", false, PICTURE, { metaKey: true }); + // macOS sends no keyup for C while Cmd is down. + key("KeyC", true, PICTURE, { metaKey: true }); + key("MetaLeft", false, PICTURE); + assert.deepEqual(sent, ["0:87", "0:91", "0:86", "1:86", "0:67", "1:91", "1:67"]); + sent.length = 0; + key("KeyW", false, PICTURE); + assert.deepEqual(sent, ["1:87"]); +}); diff --git a/packages/stream/src/input.ts b/packages/stream/src/input.ts index fbf9bba..33de6f5 100644 --- a/packages/stream/src/input.ts +++ b/packages/stream/src/input.ts @@ -89,6 +89,9 @@ const BUTTON_BITS: ReadonlyArray = [ [16, 4], ]; +/** Both Meta keys' virtual-key codes. */ +const META: ReadonlySet = new Set([0x5b, 0x5c]); + /** One wheel notch on the wire. */ const WHEEL_NOTCH = 120; @@ -163,6 +166,9 @@ export class InputPipe { private touches = new Map(); /** Keys and mouse buttons (`MouseEvent.buttons`) the host holds down, for a lost focus to let go of. */ private keys = new Set(); + /** Keys that went down while Meta was held. macOS sends no keyup for those, so Meta's own + * release lets go of them. */ + private underMeta = new Set(); private held = 0; /** Per pad: when the escape chord went down (`0` up), whether its hold fired, and whether * the menu chord is down — each chord fires on its edge, not every frame it is held. */ @@ -288,11 +294,18 @@ export class InputPipe { // --- keyboard -------------------------------------------------------------------------- private key(e: KeyboardEvent, down: boolean): void { - // A field elsewhere on the page keeps its keys: the HUD has none, but a consumer's might. + // A field elsewhere on the page keeps its keys (the HUD has none, but a consumer's might), and + // so does the page's own menu while it has focus. const t = e.target as HTMLElement | null; - if (t && (t.tagName === "INPUT" || t.tagName === "TEXTAREA" || t.isContentEditable)) return; - // The page's own menu, while it has focus, keeps its keys too. - if (t?.closest?.("[data-pf-keys=local]")) return; + const local = + !!t && (t.tagName === "INPUT" || t.tagName === "TEXTAREA" || t.isContentEditable || !!t.closest?.("[data-pf-keys=local]")); + const vk = VK[e.code]; + // A key the host holds is let go wherever its release lands: in the menu, a field, or as a chord. + if (!down && vk !== undefined && this.keys.has(vk)) { + if (!local) e.preventDefault(); + return this.up(vk); + } + if (local) return; const chord = e.ctrlKey && e.altKey && e.shiftKey ? KEY_CHORDS[e.code] @@ -306,21 +319,34 @@ export class InputPipe { if (down && !e.repeat) this.opts.onChord?.(chord); return; } - const vk = VK[e.code]; if (vk === undefined) return; // Before the repeat check: a held Tab's own repeat would walk focus off the canvas. e.preventDefault(); + if (!down) return this.up(vk); // Held keys repeat on the host; the browser's own repeat would double them. - if (down && e.repeat) return; - if (down) this.keys.add(vk); - else this.keys.delete(vk); - this.mod._pf_input(down ? KIND.KEY_DOWN : KIND.KEY_UP, vk, 0, 0, 0); + if (e.repeat) return; + this.keys.add(vk); + if (e.metaKey && !META.has(vk)) this.underMeta.add(vk); + this.mod._pf_input(KIND.KEY_DOWN, vk, 0, 0, 0); + } + + /** The host lets go of `vk`. A Meta release also lets go of the keys pressed under it that + * are still down: their own keyups never come on macOS. */ + private up(vk: number): void { + this.keys.delete(vk); + this.underMeta.delete(vk); + this.mod._pf_input(KIND.KEY_UP, vk, 0, 0, 0); + if (!META.has(vk)) return; + const under = [...this.underMeta]; + this.underMeta.clear(); + for (const k of under) if (this.keys.has(k)) this.up(k); } /** Let go of every key and button the host holds. */ private release(): void { for (const vk of this.keys) this.mod._pf_input(KIND.KEY_UP, vk, 0, 0, 0); this.keys.clear(); + this.underMeta.clear(); this.press(0); } From 4d3bd5e3c3dea8fd766918797c67c4701e534c90 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:28:12 +0100 Subject: [PATCH 12/48] fix(stream/input): capture the mouse on press outside pointer lock In absolute mode a drag released over the HUD toolbar, which appears on any pointer move, or over another overlay never reached the canvas. The host kept the button down until the pointer came back over the picture, then moved the drag to the re-entry point before letting go. --- packages/stream/src/input.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/stream/src/input.ts b/packages/stream/src/input.ts index 33de6f5..8da5350 100644 --- a/packages/stream/src/input.ts +++ b/packages/stream/src/input.ts @@ -368,6 +368,14 @@ export class InputPipe { e.preventDefault(); if (phase === "down") { this.canvas.focus(); + // Unlocked, a release over the HUD or another overlay still comes here. + if (!this.captured) { + try { + this.canvas.setPointerCapture(e.pointerId); + } catch { + // A pointer the browser no longer tracks: nothing to capture. + } + } // Pointer lock can only be taken from a gesture, so the first click in capture mode is // what arms it. if (this.opts.pointer === "capture" && !this.captured) this.lock(); From f25718d6218f32af3b0f01126471214af29c1bac Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:28:19 +0100 Subject: [PATCH 13/48] fix(stream/input): let go of the pads on blur or a hidden tab Pads are read once a frame, and a hidden tab runs no frames, so a stick or trigger held as the tab hid stayed applied on the host until it came back. A lost focus now sends every attached pad at rest, and a hidden tab counts as a lost focus. A window that blurs but keeps drawing sends the real pad again on the next frame, which differs from the rest state just sent. --- packages/stream/src/input.test.ts | 9 +++++++++ packages/stream/src/input.ts | 13 ++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/stream/src/input.test.ts b/packages/stream/src/input.test.ts index 800cc79..b5602c3 100644 --- a/packages/stream/src/input.test.ts +++ b/packages/stream/src/input.test.ts @@ -225,3 +225,12 @@ test("Meta's release lets go of the keys pressed under it, not those held before key("KeyW", false, PICTURE); assert.deepEqual(sent, ["1:87"]); }); + +test("a lost focus lets go of keys and pads", () => { + const { sent, pipe, key } = keyPipe(); + const internals = pipe as unknown as { arrive(g: unknown): void; release(): void }; + internals.arrive({ index: 2, mapping: "standard", id: "" }); + key("KeyA", true, PICTURE); + internals.release(); + assert.deepEqual(sent, ["0:65", "1:65", "pad2:0,0,0,0,0,0,0"]); +}); diff --git a/packages/stream/src/input.ts b/packages/stream/src/input.ts index 8da5350..d56b023 100644 --- a/packages/stream/src/input.ts +++ b/packages/stream/src/input.ts @@ -254,6 +254,9 @@ export class InputPipe { this.on(c, "contextmenu", (e: Event) => e.preventDefault()); // A release that lands in another window or tab never comes back here. this.on(window, "blur", () => this.release()); + this.on(document, "visibilitychange", () => { + if (document.visibilityState === "hidden") this.release(); + }); // The pad set is polled; these only keep the arrival and removal events honest. this.on(window, "gamepadconnected", (e: GamepadEvent) => this.arrive(e.gamepad)); this.on(window, "gamepaddisconnected", (e: GamepadEvent) => this.leave(e.gamepad.index)); @@ -342,12 +345,14 @@ export class InputPipe { for (const k of under) if (this.keys.has(k)) this.up(k); } - /** Let go of every key and button the host holds. */ + /** Let go of every key, button and pad the host holds. The next poll sends a pad that is still + * in use again. */ private release(): void { for (const vk of this.keys) this.mod._pf_input(KIND.KEY_UP, vk, 0, 0, 0); this.keys.clear(); this.underMeta.clear(); this.press(0); + for (const pad of this.pads) this.mod._pf_gamepad(pad, 0, 0, 0, 0, 0, 0, 0); } /** Send what changed in `MouseEvent.buttons`. A second button pressed or released while one is @@ -502,6 +507,12 @@ export class InputPipe { fn: (e: WindowEventMap[K]) => void, opts?: AddEventListenerOptions, ): void; + private on( + target: Document, + type: K, + fn: (e: DocumentEventMap[K]) => void, + opts?: AddEventListenerOptions, + ): void; private on( target: HTMLElement, type: K, From 90061cbfae32ea706354ea33eebf45e0a8adfbde Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:30:47 +0100 Subject: [PATCH 14/48] fix(stream/mic): close a capture that a later toggle overtook Turning the microphone on, off and on again while the first permission prompt was up let both prompts proceed. The second overwrote the first's stream and context, and the first returned without stopping either, so a capture stayed open, recording indicator and all, until the page closed. Every start and stop now moves a generation, and a start that finds it moved after an await stops its own tracks and closes its own context. --- packages/stream/src/mic.test.ts | 99 +++++++++++++++++++++++++++++++++ packages/stream/src/mic.ts | 23 ++++++-- 2 files changed, 116 insertions(+), 6 deletions(-) create mode 100644 packages/stream/src/mic.test.ts diff --git a/packages/stream/src/mic.test.ts b/packages/stream/src/mic.test.ts new file mode 100644 index 0000000..528c3c8 --- /dev/null +++ b/packages/stream/src/mic.test.ts @@ -0,0 +1,99 @@ +// The microphone: a start overtaken by a stop, or by a stop and another start, closes what it +// opened, so exactly one capture is ever live. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { MicPipe } from "./mic.ts"; + +/** Browser stand-ins for one test; `prompts` resolves each `getUserMedia` the page asked for. */ +function fakeBrowser() { + const tracks: Array<{ stopped: boolean }> = []; + const contexts: Array<{ closed: boolean }> = []; + const prompts: Array<() => void> = []; + const stubs: Record = { + navigator: { + mediaDevices: { + getUserMedia: () => + new Promise((resolve) => + prompts.push(() => { + const track = { stopped: false, stop: () => (track.stopped = true) }; + tracks.push(track); + resolve({ getTracks: () => [track] }); + }), + ), + }, + }, + AudioContext: class { + closed = false; + sampleRate = 48_000; + destination = {}; + audioWorklet = { addModule: async () => {} }; + constructor() { + contexts.push(this); + } + createMediaStreamSource = () => ({ connect: (n: unknown) => n }); + createGain = () => ({ gain: { value: 1 }, connect: (n: unknown) => n }); + resume = async () => {}; + close = async () => { + this.closed = true; + }; + }, + AudioWorkletNode: class { + port = { onmessage: null }; + connect = (n: unknown) => n; + }, + AudioEncoder: class { + state = "configured"; + configure() {} + close() {} + }, + }; + const saved = Object.keys(stubs).map((k) => [k, Object.getOwnPropertyDescriptor(globalThis, k)] as const); + for (const [k, v] of Object.entries(stubs)) Object.defineProperty(globalThis, k, { value: v, configurable: true, writable: true }); + const restore = () => { + for (const [k, d] of saved) { + if (d) Object.defineProperty(globalThis, k, d); + else delete (globalThis as Record)[k]; + } + }; + return { tracks, contexts, prompts, restore }; +} + +test("on, off and on during the prompt leaves one capture live, and off closes it", async () => { + const b = fakeBrowser(); + try { + const mic = new MicPipe({} as never); + const first = mic.start(); + mic.stop(); + const second = mic.start(); + for (const allow of b.prompts) allow(); + await Promise.all([first, second]); + assert.equal(mic.state, "on"); + assert.deepEqual(b.tracks.map((t) => t.stopped), [true, false], "the overtaken prompt's track is stopped"); + assert.equal(b.contexts.length, 1); + mic.stop(); + assert.deepEqual(b.tracks.map((t) => t.stopped), [true, true]); + await Promise.resolve(); + assert.ok(b.contexts.every((c) => c.closed)); + } finally { + b.restore(); + } +}); + +test("a stop while the worklet loads closes that start's capture", async () => { + const b = fakeBrowser(); + try { + const mic = new MicPipe({} as never); + const started = mic.start(); + b.prompts[0]!(); + // Let `getUserMedia` settle and the worklet load begin, then turn the microphone off. + await new Promise((r) => setTimeout(r, 0)); + mic.stop(); + await started; + assert.equal(mic.state, "off"); + assert.deepEqual(b.tracks.map((t) => t.stopped), [true]); + assert.ok(b.contexts.every((c) => c.closed)); + } finally { + b.restore(); + } +}); diff --git a/packages/stream/src/mic.ts b/packages/stream/src/mic.ts index aac0e70..efbc07a 100644 --- a/packages/stream/src/mic.ts +++ b/packages/stream/src/mic.ts @@ -47,6 +47,9 @@ export class MicPipe { private encoder: AudioEncoder | null = null; private seq = 0; private ts = 0; + /** Moved by every `start` and `stop`: a `start` that finds it moved after an await was + * overtaken, and drops what it opened. */ + private gen = 0; constructor(private readonly mod: PunktfunkModule) {} @@ -66,6 +69,7 @@ export class MicPipe { /** Start capturing. Must follow a gesture: the permission prompt is the browser's. */ async start(): Promise { if (this._state === "on" || this._state === "starting" || this._state === "unsupported") return; + const gen = ++this.gen; this._state = "starting"; let stream: MediaStream; try { @@ -73,17 +77,21 @@ export class MicPipe { audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true }, }); } catch { - this._state = "denied"; + if (gen === this.gen) this._state = "denied"; return; } - if (this._state !== "starting") { - // Turned off while the prompt was up. + if (gen !== this.gen) { + // Turned off, or off and on again, while the prompt was up. for (const t of stream.getTracks()) t.stop(); return; } this.stream = stream; const ctx = new AudioContext({ sampleRate: SAMPLE_RATE }); this.ctx = ctx; + const drop = () => { + for (const t of stream.getTracks()) t.stop(); + void ctx.close().catch(() => {}); + }; // Any step below can fail on a browser short of one piece; the microphone must not stay open. try { const url = URL.createObjectURL(new Blob([WORKLET], { type: "application/javascript" })); @@ -92,7 +100,7 @@ export class MicPipe { } finally { URL.revokeObjectURL(url); } - if (this.ctx !== ctx) return; + if (gen !== this.gen) return drop(); const encoder = new AudioEncoder({ output: (chunk) => this.send(chunk), error: () => this.stop(), @@ -107,15 +115,18 @@ export class MicPipe { mute.gain.value = 0; node.connect(mute).connect(ctx.destination); await ctx.resume(); + if (gen !== this.gen) return drop(); } catch (e) { console.error("punktfunk: microphone", e); - if (this.ctx === ctx) this.stop(); + if (gen === this.gen) this.stop(); + else drop(); return; } - if (this.ctx === ctx) this._state = "on"; + this._state = "on"; } stop(): void { + this.gen++; this._state = MicPipe.supported() ? "off" : "unsupported"; try { this.encoder?.close(); From a9aaeb4e7940ad9598c8253304d8621c349e4a84 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:31:38 +0100 Subject: [PATCH 15/48] fix(stream/mic): capture at the device rate, resampled to 48 kHz The microphone's context was forced to 48 kHz. Firefox refuses to connect a capture to a context at another rate, so on an output device at 44.1 kHz the microphone went back to Off with only a console error. The context now runs at the device's rate, and the capture worklet brings it to the 48 kHz the encoder and the host expect when the two differ. --- packages/stream/src/mic.test.ts | 26 +++++++++++++++-- packages/stream/src/mic.ts | 52 ++++++++++++++++++++++++++------- 2 files changed, 66 insertions(+), 12 deletions(-) diff --git a/packages/stream/src/mic.test.ts b/packages/stream/src/mic.test.ts index 528c3c8..667133d 100644 --- a/packages/stream/src/mic.test.ts +++ b/packages/stream/src/mic.test.ts @@ -1,9 +1,9 @@ // The microphone: a start overtaken by a stop, or by a stop and another start, closes what it -// opened, so exactly one capture is ever live. +// opened, so exactly one capture is ever live. The worklet's resampler brings any rate to 48 kHz. import { test } from "node:test"; import assert from "node:assert/strict"; -import { MicPipe } from "./mic.ts"; +import { MicPipe, RESAMPLER } from "./mic.ts"; /** Browser stand-ins for one test; `prompts` resolves each `getUserMedia` the page asked for. */ function fakeBrowser() { @@ -97,3 +97,25 @@ test("a stop while the worklet loads closes that start's capture", async () => { b.restore(); } }); + +type Resampler = { push(x: Float32Array, emit: (v: number) => void): void }; +const Resampler = new Function(`${RESAMPLER}; return Resampler;`)() as new (from: number) => Resampler; + +/** One second of a ramp at `rate`, pushed in render quanta; what comes out at 48 kHz. */ +const resampled = (rate: number): number[] => { + const r = new Resampler(rate); + const out: number[] = []; + const ramp = Float32Array.from({ length: rate }, (_, i) => i); + for (let at = 0; at < ramp.length; at += 128) r.push(ramp.subarray(at, at + 128), (v) => out.push(v)); + return out; +}; + +test("the resampler brings a ramp to 48 kHz across block edges, up and down", () => { + for (const rate of [44_100, 16_000, 96_000]) { + const out = resampled(rate); + // A second in is a second out, less the samples still waiting on the next block. + assert.ok(out.length <= 48_000 && out.length >= 48_000 - 3, `${rate}: ${out.length}`); + const step = rate / 48_000; + out.forEach((v, j) => assert.ok(Math.abs(v - j * step) < 1e-3, `${rate} sample ${j}: ${v}`)); + } +}); diff --git a/packages/stream/src/mic.ts b/packages/stream/src/mic.ts index efbc07a..39a5956 100644 --- a/packages/stream/src/mic.ts +++ b/packages/stream/src/mic.ts @@ -5,7 +5,7 @@ // cancellation keeps the host's audio, playing from the speakers, out of what goes back. // // Capture is an `AudioWorklet` rather than `MediaStreamTrackProcessor`, which only Chromium has; -// the worklet cuts 20 ms frames at 48 kHz, WebCodecs' `AudioEncoder` makes them Opus, and Rust +// the worklet brings the device's rate to 48 kHz and cuts 20 ms frames, WebCodecs' `AudioEncoder` makes them Opus, and Rust // wraps each as the uplink datagram. The host decodes into its virtual microphone. import type { PunktfunkModule } from "./emscripten.ts"; @@ -16,24 +16,55 @@ const FRAME = 960; export type MicState = "off" | "starting" | "on" | "denied" | "unsupported"; -const WORKLET = ` +/** + * Linear resampling to 48 kHz, as source the worklet runs and a test can. `push` takes one block + * at the context's rate and hands each 48 kHz sample to `emit`; the last sample and the + * fractional position carry into the next block. + */ +// ponytail: linear interpolation with no low-pass, fine for voice. A windowed-sinc filter if +// music, or rates far above 48 kHz, ever go up this path. +export const RESAMPLER = ` +class Resampler { + constructor(from) { + this.step = from / ${SAMPLE_RATE}; + this.pos = 0; + this.prev = 0; + } + push(x, emit) { + const n = x.length; + if (!n) return; + let p = this.pos; + for (; p < n - 1; p += this.step) { + const i = Math.floor(p); + const a = i < 0 ? this.prev : x[i]; + emit(a + (x[i + 1] - a) * (p - i)); + } + this.pos = p - n; + this.prev = x[n - 1]; + } +}`; + +const WORKLET = `${RESAMPLER} class PunktfunkMic extends AudioWorkletProcessor { constructor() { super(); this.buf = new Float32Array(${FRAME}); this.at = 0; - } - process(inputs) { - const ch = inputs[0] && inputs[0][0]; - if (!ch) return true; - for (let i = 0; i < ch.length; i++) { - this.buf[this.at++] = ch[i]; + this.resampler = sampleRate === ${SAMPLE_RATE} ? null : new Resampler(sampleRate); + this.add = (v) => { + this.buf[this.at++] = v; if (this.at === this.buf.length) { this.port.postMessage(this.buf, [this.buf.buffer]); this.buf = new Float32Array(${FRAME}); this.at = 0; } - } + }; + } + process(inputs) { + const ch = inputs[0] && inputs[0][0]; + if (!ch) return true; + if (this.resampler) this.resampler.push(ch, this.add); + else for (let i = 0; i < ch.length; i++) this.add(ch[i]); return true; } } @@ -86,7 +117,8 @@ export class MicPipe { return; } this.stream = stream; - const ctx = new AudioContext({ sampleRate: SAMPLE_RATE }); + // The device's own rate: Firefox refuses a source whose rate differs from its context's. + const ctx = new AudioContext(); this.ctx = ctx; const drop = () => { for (const t of stream.getTracks()) t.stop(); From e2317ac806288f8e98b136f20eccbfd31d4bee07 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:31:45 +0100 Subject: [PATCH 16/48] fix(stream/audio): play through an iPhone's silent switch Safari plays Web Audio in the ambient category unless the page says otherwise, and the ring/silent switch mutes that, so a stream on an iPhone set to silent had no sound. The audio pipe now declares playback where the browser offers an audio session. --- packages/stream/src/audio.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/stream/src/audio.ts b/packages/stream/src/audio.ts index e34276d..5933cf1 100644 --- a/packages/stream/src/audio.ts +++ b/packages/stream/src/audio.ts @@ -197,6 +197,10 @@ export class AudioPipe { } private async open(): Promise { + // Safari plays Web Audio as ambient sound, which an iPhone's silent switch mutes; a stream + // is media. + const session = (navigator as { audioSession?: { type: string } }).audioSession; + if (session) session.type = "playback"; const ctx = new AudioContext({ sampleRate: SAMPLE_RATE, latencyHint: "interactive" }); this.ctx = ctx; const url = URL.createObjectURL(new Blob([WORKLET], { type: "application/javascript" })); From 25c5ee1255358929b05c0811dc1b107cc75d9dfa Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:32:29 +0100 Subject: [PATCH 17/48] fix(stream/video): rebuild the plane after a lost GPU context Nothing claimed a lost WebGL2 context, so the browser never restored it. The canvas then handed the same dead context to every later plane, the plane could not be built, and each stream after it showed black with sound and input until a reload. A lost WebGPU device froze the picture for the rest of its stream. The WebGL2 plane now claims the loss and, once the context is back, the pipe builds a new plane and decoder for the last configuration; a WebGPU device lost other than by destroy does the same. A plane that cannot be made counts as a decode error instead of an unhandled rejection. --- packages/stream/src/video-surface-webgpu.ts | 8 +++++-- packages/stream/src/video-surface.ts | 21 +++++++++++++++-- packages/stream/src/video.ts | 25 ++++++++++++++++++--- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/packages/stream/src/video-surface-webgpu.ts b/packages/stream/src/video-surface-webgpu.ts index 6f7b5a7..4659ce0 100644 --- a/packages/stream/src/video-surface-webgpu.ts +++ b/packages/stream/src/video-surface-webgpu.ts @@ -92,12 +92,16 @@ export class VideoSurfaceWebGPU implements VideoPlane { private toneMappingUnavailable = false; /** Adapter, device and canvas context. Rejects when the engine has no WebGPU, so a caller can - * fall back to the WebGL2 surface rather than showing nothing. */ - static async create(canvas: HTMLCanvasElement): Promise { + * fall back to the WebGL2 surface rather than showing nothing. `onLost` runs when the device + * is lost other than by `destroy`: the caller builds a new plane. */ + static async create(canvas: HTMLCanvasElement, onLost?: () => void): Promise { if (!navigator.gpu) throw new Error("no WebGPU in this engine"); const adapter = await navigator.gpu.requestAdapter({ powerPreference: "high-performance" }); if (!adapter) throw new Error("no WebGPU adapter"); const device = await adapter.requestDevice(); + void device.lost.then((info) => { + if (info.reason !== "destroyed") onLost?.(); + }); try { return new VideoSurfaceWebGPU(canvas, device); } catch (e) { diff --git a/packages/stream/src/video-surface.ts b/packages/stream/src/video-surface.ts index deae00c..fa40484 100644 --- a/packages/stream/src/video-surface.ts +++ b/packages/stream/src/video-surface.ts @@ -73,8 +73,13 @@ export class VideoSurface implements VideoPlane { /** Total milliseconds spent in `present`, so a caller can report the per-frame upload cost * without timing every call itself. */ private uploadMs = 0; + private readonly unwatch: () => void; - constructor(private readonly canvas: HTMLCanvasElement) { + /** `onLost` runs once a lost context is back, blank: the caller builds a new plane on it. */ + constructor( + private readonly canvas: HTMLCanvasElement, + onLost?: () => void, + ) { // `alpha: false` — this is the bottom layer and nothing shows through it; opaque lets the // compositor skip a blend. `preserveDrawingBuffer: false` because every frame is a full // overwrite. `desynchronized` cuts a compositor hop on the plane where latency is the point. @@ -88,6 +93,7 @@ export class VideoSurface implements VideoPlane { powerPreference: "high-performance", }); if (!gl) throw new Error("no WebGL2 context for the video plane"); + if (gl.isContextLost()) throw new Error("the video plane's WebGL2 context is lost"); this.gl = gl; const prog = gl.createProgram(); @@ -118,6 +124,16 @@ export class VideoSurface implements VideoPlane { gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.uniform1i(gl.getUniformLocation(prog, "tex"), 0); + + // A context comes back only if its loss is claimed. + const lost = (e: Event) => e.preventDefault(); + const restored = () => onLost?.(); + canvas.addEventListener("webglcontextlost", lost); + canvas.addEventListener("webglcontextrestored", restored); + this.unwatch = () => { + canvas.removeEventListener("webglcontextlost", lost); + canvas.removeEventListener("webglcontextrestored", restored); + }; } /** Source dimensions and colour. `colorSpace` comes from the frame's own `VideoColorSpace`; @@ -178,8 +194,9 @@ export class VideoSurface implements VideoPlane { } /** Frees the plane's own objects and keeps the context: the canvas outlives this plane, and - * hands the next one this same context, lost or not. */ + * hands the next one this same context. A loss after this is no longer claimed. */ destroy(): void { + this.unwatch(); const gl = this.gl; gl.deleteTexture(this.tex); gl.deleteProgram(this.program); diff --git a/packages/stream/src/video.ts b/packages/stream/src/video.ts index a013087..cb801e7 100644 --- a/packages/stream/src/video.ts +++ b/packages/stream/src/video.ts @@ -180,9 +180,11 @@ const DECODER_KIND: Record = { "prefer-hardware": export async function createPlane( canvas: HTMLCanvasElement, prefer: "auto" | "webgl2" | "webgpu" = "auto", + /** The plane's GPU state is gone and a new plane can be made in its place. */ + onLost?: () => void, ): Promise<{ plane: VideoPlane; backend: "webgpu" | "webgl2" }> { - const webgpu = async () => ({ plane: await VideoSurfaceWebGPU.create(canvas), backend: "webgpu" as const }); - const webgl2 = () => ({ plane: new VideoSurface(canvas), backend: "webgl2" as const }); + const webgpu = async () => ({ plane: await VideoSurfaceWebGPU.create(canvas, onLost), backend: "webgpu" as const }); + const webgl2 = () => ({ plane: new VideoSurface(canvas, onLost), backend: "webgl2" as const }); if (prefer === "webgl2") { try { return webgl2(); @@ -271,7 +273,15 @@ export class VideoPipe { this.stats.width = width; this.stats.height = height; if (!this.plane) { - const made = await createPlane(this.canvas, this.prefer); + let made: Awaited>; + try { + made = await createPlane(this.canvas, this.prefer, () => this.replane()); + } catch (e) { + // No plane this canvas can hold: the picture stays black, and the overlay counts it. + this.stats.errors++; + console.error("punktfunk: video plane", e); + return; + } // Closed while the plane was made: it would outlive the pipe. if (this.closed) return made.plane.destroy(); this.plane = made.plane; @@ -302,6 +312,15 @@ export class VideoPipe { this.decoder = decoder; } + /** The plane's GPU state is gone: a new plane for the last `Welcome`, and a new decoder, which + * waits for a key frame. */ + private replane(): void { + if (this.closed || !this.plane) return; + this.plane.destroy(); + this.plane = null; + void this.configure(this.stats.codec, this.stats.width, this.stats.height); + } + /** The decoder's configuration for the stream `Welcome` last described. No `description`: the * host sends parameter sets inline with every key frame, which lets a browser join mid-stream. */ private config(hardwareAcceleration: HardwareAcceleration): VideoDecoderConfig { From 3715cc51c6103194bba02266f2afff7857b9750a Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:32:48 +0100 Subject: [PATCH 18/48] fix(stream/video): offer HDR only to a canvas that can show it A canvas keeps the first kind of context it hands out. After a stream on WebGL2, switching the backend to Auto or WebGPU without a reload still offered HDR, but the next plane landed on WebGL2 again, which shows the PQ signal unconverted: a washed-out picture for the whole stream. The offer now reads the backend the canvas holds once a plane was made on it, and the preference only before that. --- packages/stream/src/engine.ts | 8 +++++--- packages/stream/src/video.ts | 20 ++++++++++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/packages/stream/src/engine.ts b/packages/stream/src/engine.ts index be30ae4..64e03e6 100644 --- a/packages/stream/src/engine.ts +++ b/packages/stream/src/engine.ts @@ -18,7 +18,7 @@ import { chipText, updateNotice } from "./access.ts"; import type { PunktfunkModule } from "./emscripten.ts"; import { DeviceRefused, Host, type LibraryEntry, VersionSkew } from "./host.ts"; import * as pf from "./pf-connect.ts"; -import { type Decodable, decodableCodecs, decodeSupported, hdrDisplay, VideoPipe, webgpuUsable } from "./video.ts"; +import { type Decodable, decodableCodecs, decodeSupported, hdrDisplay, planeBackend, VideoPipe, webgpuUsable } from "./video.ts"; import { type Chord, InputPipe, playRumble } from "./input.ts"; import { AudioPipe, type AudioSnapshot, opusHead, playableChannels } from "./audio.ts"; import { MicPipe, type MicState } from "./mic.ts"; @@ -619,10 +619,12 @@ export class Engine { /** * What the next `Hello` offers: this browser's codecs less those it refused, the player's pick * first, and HDR unless `sdr`. HDR is asked per stream: the window may have moved to another - * display, and the plane may have been switched to WebGL2, since the last one. + * display, and the plane may have been switched to WebGL2, since the last one. A canvas that + * already holds WebGL2 keeps it, whatever the setting now says. */ private offer(sdr: boolean): void { - const hdr = !sdr && this.tunable.hdr && this.codecs.tenBit && hdrDisplay(this.tunable.videoBackend, this.gpu); + const backend = planeBackend(this.opts.videoCanvas, this.tunable.videoBackend); + const hdr = !sdr && this.tunable.hdr && this.codecs.tenBit && hdrDisplay(backend, this.gpu); this.mod._pf_session_codecs?.(this.codecs.mask & ~this.refused, hdr ? 1 : 0, CODEC_BIT[this.tunable.codec]); } diff --git a/packages/stream/src/video.ts b/packages/stream/src/video.ts index cb801e7..53cffbf 100644 --- a/packages/stream/src/video.ts +++ b/packages/stream/src/video.ts @@ -169,6 +169,14 @@ const ACCELERATIONS: HardwareAcceleration[] = ["prefer-hardware", "no-preference * browser's own pick, which it does not reveal. */ const DECODER_KIND: Record = { "prefer-hardware": 1, "no-preference": 0, "prefer-software": 2 }; +/** The backend each canvas was given a plane on. A canvas keeps that kind of context for life. */ +const held = new WeakMap(); + +/** The backend a plane on `canvas` will have: the one it already holds, else the preference. */ +export function planeBackend(canvas: HTMLCanvasElement, prefer: "auto" | "webgl2" | "webgpu"): "auto" | "webgl2" | "webgpu" { + return held.get(canvas) ?? prefer; +} + /** * Which plane to draw on. * @@ -183,8 +191,16 @@ export async function createPlane( /** The plane's GPU state is gone and a new plane can be made in its place. */ onLost?: () => void, ): Promise<{ plane: VideoPlane; backend: "webgpu" | "webgl2" }> { - const webgpu = async () => ({ plane: await VideoSurfaceWebGPU.create(canvas, onLost), backend: "webgpu" as const }); - const webgl2 = () => ({ plane: new VideoSurface(canvas, onLost), backend: "webgl2" as const }); + const webgpu = async () => { + const plane = await VideoSurfaceWebGPU.create(canvas, onLost); + held.set(canvas, "webgpu"); + return { plane, backend: "webgpu" as const }; + }; + const webgl2 = () => { + const plane = new VideoSurface(canvas, onLost); + held.set(canvas, "webgl2"); + return { plane, backend: "webgl2" as const }; + }; if (prefer === "webgl2") { try { return webgl2(); From 85986367a905e9fd3f0eedd5ed64a917aa3656bd Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:33:16 +0100 Subject: [PATCH 19/48] fix(stream/engine): drop the keyboard lock when a stream ends Fullscreen during a stream locks the keyboard so system shortcuts reach the host. The lock was only released by leaving fullscreen through the engine, so a stream that ended while the player stayed fullscreen left the library holding Alt+Tab and the Windows key, and a short Esc no longer left fullscreen. The lock now ends with the session, and a stream that starts while the page is already fullscreen takes it again. --- packages/stream/src/engine.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/stream/src/engine.ts b/packages/stream/src/engine.ts index 64e03e6..5ecf64d 100644 --- a/packages/stream/src/engine.ts +++ b/packages/stream/src/engine.ts @@ -716,6 +716,8 @@ export class Engine { (codec) => this.undecodable(codec), ); this.video.attach(); + // A page already fullscreen locks the keyboard again for this stream, as `fullscreen` does. + if (globalThis.document?.fullscreenElement) void keyboardLock()?.lock?.()?.catch(() => {}); this.tier = this.tunable.statsTier; this.hud = []; this.lastDropped = 0; @@ -1227,9 +1229,11 @@ export class Engine { } /** The session's pipes and its wasm state, torn down together. Idempotent. The wasm session - * goes too: a refusal leaves it `Offered`, which would read as a forgotten pairing later. */ + * goes too: a refusal leaves it `Offered`, which would read as a forgotten pairing later. The + * keyboard lock goes as well, so a page left fullscreen gives system keys back. */ private stopSession(): void { this.mod._pf_session_reset?.(); + keyboardLock()?.unlock?.(); this.mic?.stop(); this.mic = null; this.input?.detach(); @@ -1468,3 +1472,7 @@ function withStr(mod: PunktfunkModule, strings: string[], f: (...args: number const message = (e: unknown): string => (e instanceof Error ? e.message : String(e)); +/** The Keyboard Lock API, where the browser has it (Chromium). */ +const keyboardLock = (): { lock?: () => Promise; unlock?: () => void } | undefined => + (navigator as { keyboard?: { lock?: () => Promise; unlock?: () => void } }).keyboard; + From a7e343985d06bf16da13e4e8eb86eea77ac0e417 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:33:29 +0100 Subject: [PATCH 20/48] fix(stream/cursor): set the host cursor only when it changes The cursor was compared against the canvas's own style value, which the browser serializes with the URL in quotes, so the host's bitmap never matched and its data URL was parsed and set again on every frame. The compare is now against the last string this set. --- packages/stream/src/cursor.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/stream/src/cursor.ts b/packages/stream/src/cursor.ts index a258170..4595bd5 100644 --- a/packages/stream/src/cursor.ts +++ b/packages/stream/src/cursor.ts @@ -32,6 +32,9 @@ export class HostCursor { private worn: Shape | null = null; /** The shape and size the canvas's cursor was last built from. */ private built: { shape: Shape; w: number; h: number; css: string } | null = null; + /** What this last set the canvas's `cursor` to. Read back, a `url()` comes quoted and never + * matches the string set. */ + private assigned = ""; /** Keep a bitmap the host sent. Called from the glue, possibly before the first frame. */ shape(serial: number, w: number, h: number, hotX: number, hotY: number, rgba: Uint8Array): void { @@ -61,7 +64,10 @@ export class HostCursor { } // Capture mode before the lock takes the arrow: a click is what captures. const css = input && desktop && !captured ? (forwarded ? this.css(mod, canvas, streamWidth) : "none") : ""; - if (css !== canvas.style.cursor) canvas.style.cursor = css; + if (css !== this.assigned) { + this.assigned = css; + canvas.style.cursor = css; + } } /** The CSS `cursor` for the host's pointer at the size the picture is shown. */ @@ -100,6 +106,7 @@ export class HostCursor { this.sent = null; this.worn = null; this.built = null; + this.assigned = ""; if (canvas.style?.cursor) canvas.style.cursor = ""; } } From 6907ab6fe87947947e57e1166f6d3fcb4451a713 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:35:12 +0100 Subject: [PATCH 21/48] fix(stream/video): claim a WebGL2 loss for the canvas's whole life The loss was claimed only while a plane was alive, so a context lost between two streams was never restored, and every later stream on that canvas stayed black until a reload. The first WebGL2 plane on a canvas now claims its losses for good, and a stream that starts after the restore draws again. --- packages/stream/src/video-surface.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/stream/src/video-surface.ts b/packages/stream/src/video-surface.ts index fa40484..da1b87c 100644 --- a/packages/stream/src/video-surface.ts +++ b/packages/stream/src/video-surface.ts @@ -49,6 +49,9 @@ uniform sampler2D tex; out vec4 colour; void main() { colour = texture(tex, uv); }`; +/** Canvases whose WebGL2 loss is claimed for life, so a context lost between streams comes back. */ +const claimed = new WeakSet(); + function compile(gl: WebGL2RenderingContext, type: number, src: string): WebGLShader { const sh = gl.createShader(type); if (!sh) throw new Error("video surface: could not create a shader"); @@ -93,6 +96,10 @@ export class VideoSurface implements VideoPlane { powerPreference: "high-performance", }); if (!gl) throw new Error("no WebGL2 context for the video plane"); + if (!claimed.has(canvas)) { + claimed.add(canvas); + canvas.addEventListener("webglcontextlost", (e) => e.preventDefault()); + } if (gl.isContextLost()) throw new Error("the video plane's WebGL2 context is lost"); this.gl = gl; @@ -125,15 +132,9 @@ export class VideoSurface implements VideoPlane { gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.uniform1i(gl.getUniformLocation(prog, "tex"), 0); - // A context comes back only if its loss is claimed. - const lost = (e: Event) => e.preventDefault(); const restored = () => onLost?.(); - canvas.addEventListener("webglcontextlost", lost); canvas.addEventListener("webglcontextrestored", restored); - this.unwatch = () => { - canvas.removeEventListener("webglcontextlost", lost); - canvas.removeEventListener("webglcontextrestored", restored); - }; + this.unwatch = () => canvas.removeEventListener("webglcontextrestored", restored); } /** Source dimensions and colour. `colorSpace` comes from the frame's own `VideoColorSpace`; @@ -194,7 +195,7 @@ export class VideoSurface implements VideoPlane { } /** Frees the plane's own objects and keeps the context: the canvas outlives this plane, and - * hands the next one this same context. A loss after this is no longer claimed. */ + * hands the next one this same context. */ destroy(): void { this.unwatch(); const gl = this.gl; From 9752d87344701f81b766bf3472429dbd1b4dc619 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:35:34 +0100 Subject: [PATCH 22/48] docs(stream/mic): reflow the header comment to the file's width --- packages/stream/src/mic.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/stream/src/mic.ts b/packages/stream/src/mic.ts index 39a5956..b826261 100644 --- a/packages/stream/src/mic.ts +++ b/packages/stream/src/mic.ts @@ -5,8 +5,9 @@ // cancellation keeps the host's audio, playing from the speakers, out of what goes back. // // Capture is an `AudioWorklet` rather than `MediaStreamTrackProcessor`, which only Chromium has; -// the worklet brings the device's rate to 48 kHz and cuts 20 ms frames, WebCodecs' `AudioEncoder` makes them Opus, and Rust -// wraps each as the uplink datagram. The host decodes into its virtual microphone. +// the worklet brings the device's rate to 48 kHz and cuts 20 ms frames, WebCodecs' +// `AudioEncoder` makes them Opus, and Rust wraps each as the uplink datagram. The host decodes +// into its virtual microphone. import type { PunktfunkModule } from "./emscripten.ts"; From 28d53f048933019005db96da554003b2b1940e67 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:35:34 +0100 Subject: [PATCH 23/48] fix(stream/mic): record in a play-and-record audio session The audio pipe now declares Safari's session as playback, which does not record. While the microphone captures, the session is play-and-record; stopping the microphone gives playback back, unless something else has changed the session since. --- packages/stream/src/mic.test.ts | 6 +++++- packages/stream/src/mic.ts | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/stream/src/mic.test.ts b/packages/stream/src/mic.test.ts index 667133d..b3f0651 100644 --- a/packages/stream/src/mic.test.ts +++ b/packages/stream/src/mic.test.ts @@ -10,8 +10,10 @@ function fakeBrowser() { const tracks: Array<{ stopped: boolean }> = []; const contexts: Array<{ closed: boolean }> = []; const prompts: Array<() => void> = []; + const session = { type: "playback" }; const stubs: Record = { navigator: { + audioSession: session, mediaDevices: { getUserMedia: () => new Promise((resolve) => @@ -56,7 +58,7 @@ function fakeBrowser() { else delete (globalThis as Record)[k]; } }; - return { tracks, contexts, prompts, restore }; + return { tracks, contexts, prompts, session, restore }; } test("on, off and on during the prompt leaves one capture live, and off closes it", async () => { @@ -71,7 +73,9 @@ test("on, off and on during the prompt leaves one capture live, and off closes i assert.equal(mic.state, "on"); assert.deepEqual(b.tracks.map((t) => t.stopped), [true, false], "the overtaken prompt's track is stopped"); assert.equal(b.contexts.length, 1); + assert.equal(b.session.type, "play-and-record"); mic.stop(); + assert.equal(b.session.type, "playback"); assert.deepEqual(b.tracks.map((t) => t.stopped), [true, true]); await Promise.resolve(); assert.ok(b.contexts.every((c) => c.closed)); diff --git a/packages/stream/src/mic.ts b/packages/stream/src/mic.ts index b826261..f182466 100644 --- a/packages/stream/src/mic.ts +++ b/packages/stream/src/mic.ts @@ -72,6 +72,9 @@ class PunktfunkMic extends AudioWorkletProcessor { registerProcessor("punktfunk-mic", PunktfunkMic); `; +/** Safari's audio session, where the browser has one. */ +const audioSession = (): { type: string } | undefined => (navigator as { audioSession?: { type: string } }).audioSession; + export class MicPipe { private _state: MicState = MicPipe.supported() ? "off" : "unsupported"; private stream: MediaStream | null = null; @@ -103,6 +106,9 @@ export class MicPipe { if (this._state === "on" || this._state === "starting" || this._state === "unsupported") return; const gen = ++this.gen; this._state = "starting"; + // Safari captures only in a session that records; `stop` gives playback back. + const session = audioSession(); + if (session) session.type = "play-and-record"; let stream: MediaStream; try { stream = await navigator.mediaDevices.getUserMedia({ @@ -160,6 +166,8 @@ export class MicPipe { stop(): void { this.gen++; + const session = audioSession(); + if (session?.type === "play-and-record") session.type = "playback"; this._state = MicPipe.supported() ? "off" : "unsupported"; try { this.encoder?.close(); From 283c446199feff1972574b5368b100f67f4f9e2a Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:36:23 +0100 Subject: [PATCH 24/48] fix(stream/mic): give playback back when the prompt is denied A denied microphone left Safari's audio session in play-and-record until the stream ended. --- packages/stream/src/mic.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/stream/src/mic.ts b/packages/stream/src/mic.ts index f182466..dcc6aa7 100644 --- a/packages/stream/src/mic.ts +++ b/packages/stream/src/mic.ts @@ -115,7 +115,9 @@ export class MicPipe { audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true }, }); } catch { - if (gen === this.gen) this._state = "denied"; + if (gen !== this.gen) return; + this._state = "denied"; + if (session?.type === "play-and-record") session.type = "playback"; return; } if (gen !== this.gen) { From 434c6902638217448e82cabeeb162e929af8b30a Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:32:01 +0100 Subject: [PATCH 25/48] fix(web): say when the page server turns config.json away A stranger over the server's meter got a 429 for config.json, which did not parse, so the page read it as no server at all: the listed hosts vanished without a word, and a typed address was dialled directly for the rest of the visit. A 429 now means the page's own server is there. The page says the rate-limit sentence, keeps routing typed addresses through the server, and asks for the list again on a later redraw of the hosts. --- apps/web/src/app.ts | 68 +++++++++++++++---------------------- apps/web/src/config.test.ts | 38 +++++++++++++++++++++ apps/web/src/config.ts | 49 ++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 41 deletions(-) create mode 100644 apps/web/src/config.test.ts create mode 100644 apps/web/src/config.ts diff --git a/apps/web/src/app.ts b/apps/web/src/app.ts index 7ce363a..f252ccf 100644 --- a/apps/web/src/app.ts +++ b/apps/web/src/app.ts @@ -45,6 +45,7 @@ import { wakeFailed, wakingLine, } from "@punktfunk/stream"; +import { type Configured, readConfig, type ServerConfig } from "./config.ts"; import { ConsoleUi } from "./ui/console.ts"; import { noop, WebShell } from "./ui/shell.tsx"; import type { HostCard, HostTools, ProfileCard, Screen, Tab, Ui } from "./ui/types.ts"; @@ -156,6 +157,12 @@ class App { private gate: Gate | null = null; /** A Switch profile asked for a host that was not connected: it opens once the host is ready. */ private switchOnReady: string | null = null; + /** The hosts the page's own server proxies, by the API origin the engine keys them under. */ + private readonly configured: Map; + /** Whether that server also proxies a host typed here by its IP address. */ + private viaServer: boolean; + /** When the server's list may be asked for again after it turned the read away; `0`: not owed. */ + private configAgainAt = 0; constructor( private readonly engine: Engine, @@ -163,11 +170,10 @@ class App { /** The web shell, rather than the gamepad console: only it has tabs to keep. */ private readonly web: boolean, private readonly uiCanvas: HTMLCanvasElement, - /** The hosts the page's own server proxies, by the API origin the engine keys them under. */ - private readonly configured: Map, - /** Whether that server also proxies a host typed here by its IP address. */ - private readonly viaServer: boolean, + server: ServerConfig, ) { + this.configured = server.listed; + this.viaServer = server.viaServer; // A login's pass covers another host: the server now tells this browser its address and wake. onPass(() => void this.refreshConfigured()); ui.mount({ @@ -250,6 +256,11 @@ class App { engine.setStatsTier(on ? (start === "off" ? "normal" : start) : "off"); }, }); + // The server turned its list away: say so, and ask again on a later redraw of the hosts. + if (server.wait) { + this.configAgainAt = Date.now() + server.wait * 1000; + ui.notify(sentence(TOO_MANY), "error"); + } this.applyPrefs(); this.watchSize(); if (packaged()) this.watchSleep(); @@ -481,6 +492,7 @@ class App { private home(error?: string): void { // A link waiting for its answer stays in front of the probes that redraw this list. if (this.link) return; + if (this.configAgainAt && Date.now() >= this.configAgainAt) void this.refreshConfigured(); const cards = this.cards(); this.show({ kind: "home", @@ -492,10 +504,17 @@ class App { void this.probe(cards.map((h) => h.origin)); } - /** The server's list again, as a pass holder sees it. */ + /** The server's list again: as a pass holder sees it, or once a turned-away read may retry. */ private async refreshConfigured(): Promise { - const again = await configuredHosts(); + this.configAgainAt = 0; + const again = await readConfig(serverFetch(), location.href); + if (!again) return; + if (again.wait) { + this.configAgainAt = Date.now() + again.wait * 1000; + return; + } for (const [origin, c] of again.listed) this.configured.set(origin, c); + this.viaServer = again.viaServer; this.redrawLists(); } @@ -1330,39 +1349,6 @@ function routeOf(address: string): HostTarget | null { return plane ? { api: address.replace(/\/+$/, ""), plane, bridge: `wa/${ipPort}` } : null; } -/** A host from the page server's `config.json`. `wake` is its wake URL when the server can. */ -interface Configured extends HostTarget { - name: string; - wake?: string; -} - -/** - * The hosts `punktfunk-client-web-server` proxies, from its `config.json`. Any other server has - * none (a dev server answers `index.html`, which does not parse), and the page falls back to - * addresses typed here. - */ -async function configuredHosts(): Promise<{ listed: Map; viaServer: boolean }> { - try { - const r = await serverFetch()("./config.json", { cache: "no-store" }); - const c = (await r.json()) as { - hosts?: Array<{ name: string; api: string; plane?: string; wake?: string; bridge?: string }>; - add?: boolean; - }; - const listed = new Map( - (c.hosts ?? []).map((h): [string, Configured] => { - const api = new URL(h.api, location.href).href.replace(/\/+$/, ""); - const wake = h.wake ? new URL(h.wake, location.href).href : undefined; - const bridge = h.bridge ? { bridge: h.bridge } : {}; - const plane = h.plane ? { plane: h.plane } : {}; - return [api, { api, ...plane, name: h.name, ...(wake ? { wake } : {}), ...bridge }]; - }), - ); - return { listed, viaServer: c.add === true }; - } catch { - return { listed: new Map(), viaServer: false }; - } -} - /** Set by `vite.config.ts` when the dev server proxies a host; absent in a build. */ declare const __PF_TRANSPORT_HOST__: string | undefined; @@ -1378,9 +1364,9 @@ try { uiCanvas, ...(__PF_TRANSPORT_HOST__ ? { transportHost: __PF_TRANSPORT_HOST__ } : {}), }); - const server = await configuredHosts(); + const server = (await readConfig(serverFetch(), location.href)) ?? { listed: new Map(), viaServer: false }; const ui = pickUi(engine, uiCanvas); - new App(engine, ui, ui instanceof WebShell, uiCanvas, server.listed, server.viaServer); + new App(engine, ui, ui instanceof WebShell, uiCanvas, server); } catch (e) { // Before there is an engine there is no interface to say this on; the one sheet the page // carries for exactly this case does. diff --git a/apps/web/src/config.test.ts b/apps/web/src/config.test.ts new file mode 100644 index 0000000..3de1d56 --- /dev/null +++ b/apps/web/src/config.test.ts @@ -0,0 +1,38 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { readConfig } from "./config.ts"; + +const PAGE = "https://pf.example/web/"; +const answer = (r: Response) => (async () => r) as unknown as typeof fetch; + +test("the list resolves against the page, and a host without a plane keeps none", async () => { + const body = { hosts: [{ name: "desk", api: "h/desk", bridge: "w/desk", wake: "wake/desk" }], add: true }; + const c = await readConfig(answer(Response.json(body)), PAGE); + assert.equal(c?.viaServer, true); + assert.equal(c?.wait, undefined); + const desk = c?.listed.get("https://pf.example/web/h/desk"); + assert.deepEqual(desk, { + api: "https://pf.example/web/h/desk", + name: "desk", + wake: "https://pf.example/web/wake/desk", + bridge: "w/desk", + }); + assert.equal(desk !== undefined && "plane" in desk, false); +}); + +test("a 429 is the page's own server: it proxies, and the list is owed after Retry-After", async () => { + const busy = new Response("Too many requests", { status: 429, headers: { "retry-after": "3" } }); + const c = await readConfig(answer(busy), PAGE); + assert.deepEqual({ size: c?.listed.size, viaServer: c?.viaServer, wait: c?.wait }, { size: 0, viaServer: true, wait: 3 }); + const bare = await readConfig(answer(new Response("", { status: 429 })), PAGE); + assert.equal(bare?.wait, 60); +}); + +test("a server without config.json, or no answer, is no page server", async () => { + const html = new Response("", { headers: { "content-type": "text/html" } }); + assert.equal(await readConfig(answer(html), PAGE), null); + const down = (async () => { + throw new TypeError("network"); + }) as unknown as typeof fetch; + assert.equal(await readConfig(down, PAGE), null); +}); diff --git a/apps/web/src/config.ts b/apps/web/src/config.ts new file mode 100644 index 0000000..c43906e --- /dev/null +++ b/apps/web/src/config.ts @@ -0,0 +1,49 @@ +// What the page's own server says in `config.json`: the hosts it proxies, and whether it also +// proxies a host typed here by its IP address. Apart from `app.ts` so a test can read it. + +import type { HostTarget } from "@punktfunk/stream"; + +/** A host from the page server's `config.json`. `wake` is its wake URL when the server can. */ +export interface Configured extends HostTarget { + name: string; + wake?: string; +} + +export interface ServerConfig { + listed: Map; + /** The server also proxies a host typed here by its IP address. */ + viaServer: boolean; + /** The server turned the read away (`429`): seconds until it may be asked again. */ + wait?: number; +} + +/** + * Read `config.json` through `fetch`, resolving its routes against `page`. `null` is a server + * without one: a dev server answers `index.html`, which does not parse. A `429` is the page's + * own server metering a stranger, so it proxies, and its list is owed. + */ +export async function readConfig(fetch: typeof globalThis.fetch, page: string): Promise { + try { + const r = await fetch(new URL("config.json", page).href, { cache: "no-store" }); + if (r.status === 429) { + const wait = Number(r.headers.get("retry-after")); + return { listed: new Map(), viaServer: true, wait: wait > 0 ? wait : 60 }; + } + const c = (await r.json()) as { + hosts?: Array<{ name: string; api: string; plane?: string; wake?: string; bridge?: string }>; + add?: boolean; + }; + const listed = new Map( + (c.hosts ?? []).map((h): [string, Configured] => { + const api = new URL(h.api, page).href.replace(/\/+$/, ""); + const wake = h.wake ? new URL(h.wake, page).href : undefined; + const bridge = h.bridge ? { bridge: h.bridge } : {}; + const plane = h.plane ? { plane: h.plane } : {}; + return [api, { api, ...plane, name: h.name, ...(wake ? { wake } : {}), ...bridge }]; + }), + ); + return { listed, viaServer: c.add === true }; + } catch { + return null; + } +} From d0907d04dc2c061fa5b14229996b401c556fcfce Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:32:39 +0100 Subject: [PATCH 26/48] fix(web): hold a link's host to its fingerprint across retries A link pins its host to a fingerprint, but only its first connect carried the pin. In a fresh browser that connect stops at the certificate, so "I have accepted it" reconnected with no pin and the link's host was trusted on first use. The pin now stays with that host until the page is idle again, and every reconnect to it carries the pin: a retry, the connection after pairing, and forget-and-pair-again. --- apps/web/src/app.ts | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/apps/web/src/app.ts b/apps/web/src/app.ts index f252ccf..c2a5e3e 100644 --- a/apps/web/src/app.ts +++ b/apps/web/src/app.ts @@ -144,6 +144,9 @@ class App { /** A connect link the page opened with, until it is answered, and the title it asked for. */ private link: { target: string | HostTarget; fp?: string; launch?: string } | null = null; private launchOnReady: string | null = null; + /** The fingerprint a followed link pinned its host to. Every reconnect to that host carries it + * until the page is idle again. */ + private pin: { origin: string; fp: string } | null = null; /** The settings page, or its dialog over a live stream. */ private settingsOpen = false; /** The quick menu over a live picture; closed whenever the stream is not. */ @@ -196,7 +199,7 @@ class App { // Otherwise a stream a card asked for is still what the retry is for. if (this.dropped) this.resumeOnReady = this.dropped; this.dropped = null; - if ("origin" in s && s.origin) void engine.connect(this.targetOf(s.origin)); + if ("origin" in s && s.origin) this.reconnect(s.origin); }, // Leaving the profile gate goes back to the host, not away from it. back: () => (this.gate ? this.cancelGate() : engine.disconnect()), @@ -315,6 +318,7 @@ class App { this.launchOnReady = null; this.resumeOnReady = null; this.dropped = null; + this.pin = null; return this.web && this.tab === "library" ? this.showLibrary() : this.home(); case "bad-address": // The field stays in front: what was typed is wrong and this is where it is fixed. @@ -353,7 +357,7 @@ class App { // The host closes after the ceremony, as it does for native clients; streaming is a // fresh connection. const origin = s.origin; - setTimeout(() => void this.engine.connect(this.targetOf(origin)), 300); + setTimeout(() => this.reconnect(origin), 300); return; } case "pair-refused": @@ -650,7 +654,19 @@ class App { this.link = null; if (!yes || !link) return this.home(); this.launchOnReady = link.launch ?? null; - void this.engine.connect(link.target, link.fp ? { expectFingerprint: link.fp } : {}); + const origin = typeof link.target === "string" ? originOf(link.target) : link.target.api.replace(/\/+$/, ""); + this.pin = link.fp ? { origin, fp: link.fp } : null; + void this.engine.connect(link.target, this.pinned(origin)); + } + + /** Reach `origin` again: a retry, or the connection after a pairing. */ + private reconnect(origin: string): void { + void this.engine.connect(this.targetOf(origin), this.pinned(origin)); + } + + /** What a connect to `origin` must check: the fingerprint a link pinned it to, if any. */ + private pinned(origin: string): { expectFingerprint?: string } { + return this.pin?.origin === origin ? { expectFingerprint: this.pin.fp } : {}; } /** A link to this host, pinned to its fingerprint when this browser has paired with it. The @@ -705,14 +721,18 @@ class App { } /** Forget a host. From the trust screen this is "forget and pair again", so the reconnect - * follows — with the stored fingerprint gone, the next connection is a first one. */ + * follows — with the stored fingerprint gone, the next connection is a first one, held to + * the fingerprint of a link that brought it here. */ private forget(origin: string): void { const reconnect = this.screen.kind === "trust" && this.screen.origin === origin; this.reachCache.delete(origin); if (this.libraryFor === origin) this.clearLibrary(); + const pin = this.pin; this.engine.forget(origin); - if (reconnect) void this.engine.connect(this.targetOf(origin)); - else this.redrawLists(); + if (!reconnect) return this.redrawLists(); + // Forgetting disconnects, which drops a link's pin; pairing again is still held to it. + this.pin = pin; + this.reconnect(origin); } // --- the library --------------------------------------------------------------------- From dd21c2f2a074a2d07f0361b3635fb4d9f3445ecc Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:33:04 +0100 Subject: [PATCH 27/48] fix(web): decide a dropped stream from the engine, not the screen Whether an error was a stream that dropped was read off the screen on display. A redraw of the same error found the error screen there, so the exit hint's timer turned "The stream stopped" into "Something went wrong". With Settings open over the stream, the screen was the dialog: the drop stayed hidden behind the Settings page, and leaving it through the sidebar disconnected the error unseen. The drop is now decided on the engine's edge out of a live or starting stream, which also closes a settings dialog over it, and the hint's timer stops when the stream ends. --- apps/web/src/app.ts | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/apps/web/src/app.ts b/apps/web/src/app.ts index c2a5e3e..1fbcf99 100644 --- a/apps/web/src/app.ts +++ b/apps/web/src/app.ts @@ -123,7 +123,8 @@ class App { /** When the live stream began, while its exit hint is up; `0` once the hint is gone. */ private hintSince = 0; private hintTimer = 0; - private wasStreaming = false; + /** The engine state the last render drew: each edge below is against it. */ + private lastKind: EngineState["kind"] = "idle"; private streamNoticeTimer = 0; /** The origin whose running title is being polled. */ private pollingFor: string | null = null; @@ -283,6 +284,8 @@ class App { /** Facts in, words out. */ private render(s: EngineState): void { + const was = this.lastKind; + this.lastKind = s.kind; // The gate waits on its host's `ready`; anything else ends the wait, and a connect that // ended short of it drops the Switch profile it was for. if (this.gate && (s.kind !== "ready" || s.origin !== this.gate.origin)) this.gate = null; @@ -293,7 +296,7 @@ class App { } // The exit hint: armed on the edge into a stream, dropped by its timer or the stream's end. const streaming = s.kind === "streaming"; - if (streaming && !this.wasStreaming && this.prefs.exitHint) { + if (streaming && was !== "streaming" && this.prefs.exitHint) { this.hintSince = Date.now(); clearTimeout(this.hintTimer); this.hintTimer = window.setTimeout(() => { @@ -302,8 +305,14 @@ class App { }, EXIT_HINT_MS); } else if (!streaming) { this.hintSince = 0; + clearTimeout(this.hintTimer); + } + // A stream that ends takes the settings dialog over it along: how it ended is what to show. + // One that drops is what "Try again" brings back. + if (live(was) && !live(s.kind)) { + this.settingsOpen = false; + if (s.kind === "error") this.dropped = this.lastPlay; } - this.wasStreaming = streaming; if (this.settingsOpen) { return this.show({ kind: "settings", values: this.prefs, streaming: s.kind === "streaming" }); } @@ -422,15 +431,11 @@ class App { }); case "error": { this.windowed(); - // Was a stream live, or starting, when this happened? Then trying again resumes it. - const was = this.screen; - const streamed = was.kind === "streaming" || (was.kind === "connecting" && was.phase === "starting"); - if (streamed) this.dropped = this.lastPlay; return this.show({ kind: "error", head: s.skew ? "This host speaks a different version" - : streamed + : this.dropped ? "The stream stopped" : "Something went wrong", text: s.skew @@ -1230,6 +1235,9 @@ class App { } } +/** A stream live or on its way: what a settings dialog sits over, and what a drop ends. */ +const live = (kind: EngineState["kind"]): boolean => kind === "streaming" || kind === "starting"; + /** A stream asked for: the title (unset: the desktop), and the profile the console picked. */ interface Play { entry?: LibraryEntry; From ef85f830667260c1aa00431ff4e553d0b918b6fc Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:33:18 +0100 Subject: [PATCH 28/48] fix(web): keep a mistyped address's sentence through redraws The sentence for an address that is not one went to the host list once. The reachability probe it started redraws the list on every answer, without it, so with known hosts on the page the sentence went within milliseconds. The list now takes it from the engine, which holds the mistyped address until the next connect. --- apps/web/src/app.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/web/src/app.ts b/apps/web/src/app.ts index 1fbcf99..bd43788 100644 --- a/apps/web/src/app.ts +++ b/apps/web/src/app.ts @@ -333,7 +333,7 @@ class App { // The field stays in front: what was typed is wrong and this is where it is fixed. this.adding = true; this.tab = "hosts"; - return this.home(sentence(s.message)); + return this.home(); case "reaching": return this.show({ kind: "connecting", origin: s.origin, phase: "reaching" }); case "blocked": { @@ -498,11 +498,14 @@ class App { } // --- the host list --------------------------------------------------------------------- - private home(error?: string): void { + private home(): void { // A link waiting for its answer stays in front of the probes that redraw this list. if (this.link) return; if (this.configAgainAt && Date.now() >= this.configAgainAt) void this.refreshConfigured(); const cards = this.cards(); + // A mistyped address stays said while the engine holds it, whatever redraws the list. + const s = this.engine.current; + const error = s.kind === "bad-address" ? sentence(s.message) : undefined; this.show({ kind: "home", hosts: cards, From 86c42b44cff8aecc3c69587ed982323fe08236dc Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:33:27 +0100 Subject: [PATCH 29/48] fix(web): clear the End game notice when a stream starts "Ended Hades." stayed on the shelf for the rest of the visit, beside "Running now: Hades" once the game was played again. An answer that landed after another shelf opened wrote its notice there and cleared that shelf's running title. The notice now lasts until the next stream, and an answer for a shelf no longer open is dropped. --- apps/web/src/app.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/web/src/app.ts b/apps/web/src/app.ts index bd43788..563fc34 100644 --- a/apps/web/src/app.ts +++ b/apps/web/src/app.ts @@ -114,7 +114,7 @@ class App { private runningId: string | undefined; /** The running title is this device's launch, which End game may end. */ private runningEndable = false; - /** What the host said when End game last ran from the library. */ + /** What the host said when End game last ran from the library, until the next stream. */ private endNotice: string | undefined; /** The title this device launched that the live stream plays, asked for when the menu opens. */ private streamGame: { appId: string; title: string } | null = null; @@ -939,9 +939,12 @@ class App { return this.render(this.engine.current); } const host = this.shelfHost(); + const shelf = this.libraryFor; const [id, title] = [this.runningId, this.running]; if (!host || !id || !title) return; const outcome = await host.endGame(id); + // Another shelf opened meanwhile: the answer is not about its titles. + if (this.libraryFor !== shelf) return; this.endNotice = gameEndNotice(outcome, title); if (gameGone(outcome)) { this.running = undefined; @@ -1096,6 +1099,7 @@ class App { private dial(entry?: LibraryEntry, profile?: string): void { this.gate = null; + this.endNotice = undefined; this.lastPlay = { ...(entry ? { entry } : {}), ...(profile ? { profile } : {}) }; this.engine.startStream({ ...this.streamOptions(), ...(entry ? { launch: entry } : {}), ...(profile ? { profile } : {}) }); } From fcdfecfd7089b701cea2bd3390a2022fe9428584 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:33:44 +0100 Subject: [PATCH 30/48] fix(web): read the library again after a version mismatch A library read that failed on a version mismatch still stamped the library's clock, so Back then Connect within a minute skipped the read and showed "This host's library is empty" instead of the mismatch. Only a read that succeeds sets the clock now. Any failure clears it, so opening the host again asks again. --- apps/web/src/app.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/apps/web/src/app.ts b/apps/web/src/app.ts index 563fc34..5b5b252 100644 --- a/apps/web/src/app.ts +++ b/apps/web/src/app.ts @@ -754,10 +754,12 @@ class App { if (this.libraryFor !== s.origin) return; this.entries = entries; this.libraryError = undefined; + this.libraryAt = Date.now(); } catch (e) { if (this.libraryFor !== s.origin) return; + // A failure keeps no clock: opening the host again asks again. + this.libraryAt = 0; if (e instanceof VersionSkew) { - this.libraryLoading = false; return this.show({ kind: "error", head: "This host speaks a different version", @@ -768,11 +770,7 @@ class App { // dead end. this.libraryError = e instanceof Error ? e.message : String(e); } finally { - if (this.libraryFor === s.origin) { - this.libraryLoading = false; - // A failure keeps no clock: opening the host again asks again. - if (!this.libraryError) this.libraryAt = Date.now(); - } + if (this.libraryFor === s.origin) this.libraryLoading = false; } this.redrawLibrary(); // Art after the grid, per entry: the grid should appear before its covers do. From 5051c889dd2c6bcef31417f7919c34f44778bd4f Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:33:53 +0100 Subject: [PATCH 31/48] fix(web): let the sidebar leave the profile gate The profile picker, the seat wait and their error sit on a host that is ready, which the sidebar does not cancel. Hosts or Library marked the tab but left the page where it was, and the seat wait still started the stream later. The sidebar now drops the gate as it drops any other flow, which also ends the seat wait's poll. --- apps/web/src/app.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/web/src/app.ts b/apps/web/src/app.ts index 5b5b252..4e88dfc 100644 --- a/apps/web/src/app.ts +++ b/apps/web/src/app.ts @@ -448,8 +448,8 @@ class App { } // --- the tabs -------------------------------------------------------------------------- - /** A sidebar entry. A flow left half-way — a PIN, a certificate, an error — is cancelled; the - * library's own connection is kept, since it is what the Library tab shows. */ + /** A sidebar entry. A flow left half-way — a PIN, a certificate, an error, the profile gate — + * is cancelled; the library's own connection is kept, since it is what the Library tab shows. */ private navigate(tab: Tab): void { const s = this.engine.current; if (tab === "settings") { @@ -459,6 +459,7 @@ class App { this.settingsOpen = false; this.tab = tab; this.link = null; + this.gate = null; if (FLOWS.has(s.kind) && !(this.quiet && tab === "library")) { this.quiet = false; this.engine.disconnect(); From ceb3d0b05ab40083bbfdf686e135a767ad145500 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Fri, 6 Nov 2026 00:35:06 +0100 Subject: [PATCH 32/48] fix(web): keep side-edge controls clear of a notch The page is viewport-fit=cover, so in a phone's landscape it draws under the notch, but only the phone tab bar padded for the safe area. The sidebar rail, the page's right edge, a statistics card in a side corner and the toasts could sit under the camera housing. Each now keeps at least the safe-area inset from its side edge. --- apps/web/src/styles.css | 4 ++++ apps/web/src/ui/frame.tsx | 7 ++++--- apps/web/src/ui/hud.tsx | 10 +++++----- apps/web/src/ui/shell.tsx | 4 ++-- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/apps/web/src/styles.css b/apps/web/src/styles.css index 252f44e..747ac27 100644 --- a/apps/web/src/styles.css +++ b/apps/web/src/styles.css @@ -81,6 +81,10 @@ /* `EDGE_INSET` from pf-console-ui's theme.rs: the distance anything keeps from the screen edge. The one metric the console shell and this one share. */ --pf-inset: 24px; + /* The same inset from a side edge, clear of a phone's notch in landscape: the page is + `viewport-fit=cover`, so it draws under it. */ + --pf-inset-left: max(var(--pf-inset), env(safe-area-inset-left)); + --pf-inset-right: max(var(--pf-inset), env(safe-area-inset-right)); } /* Map the palette to Tailwind colour/util tokens — both the shadcn vocabulary and @unom/ui's, diff --git a/apps/web/src/ui/frame.tsx b/apps/web/src/ui/frame.tsx index c0800b2..dbd47cd 100644 --- a/apps/web/src/ui/frame.tsx +++ b/apps/web/src/ui/frame.tsx @@ -55,8 +55,9 @@ export function Frame({ return (
{/* Pinned at viewport height; the page beside it scrolls. `overflow-y-auto` lets the nav - itself scroll on a very short viewport rather than push Console mode off the bottom. */} -