diff --git a/README.md b/README.md index 1132fd0..3b937d1 100644 --- a/README.md +++ b/README.md @@ -330,8 +330,8 @@ A `v*` tag builds everything above and attaches the app as session overflows just by being constructed — and the symptom is `RuntimeError: Out of bounds memory access` from *every* export, including ones that do nothing. It reads like a corrupt module, not a stack overflow. -- **Automatic bitrate does not run in the browser.** The session streams at the bitrate the - settings name. +- **Automatic bitrate runs on loss and the host's acks alone.** The page measures no clock + offset, so no one-way delay, and asks for no probe burst. - **The console still speaks as a native app** under `?ui=console`: it offers Quit, and says hosts on the network appear by themselves, which a browser cannot discover. - The release payload is **8.2 MB of wasm, 3.5 MB gzipped**. The live heap under a stream is diff --git a/packages/stream/rust/abr.rs b/packages/stream/rust/abr.rs new file mode 100644 index 0000000..0922b0f --- /dev/null +++ b/packages/stream/rust/abr.rs @@ -0,0 +1,276 @@ +//! Automatic bitrate on the page: core's driver, fed through `punktfunk_core::abr::feed` the +//! way the native pump feeds it. The session calls in at four points: the Welcome builds an +//! [`Abr`], each access unit and each control frame it reads goes in, and each pump ticks it. +//! +//! No clock handshake runs in a browser and the page asks for no probe burst, so loss, the +//! session's counters and the host's acks drive it. + +use punktfunk_core::abr::feed::{Facts, Feed, Inputs}; +use punktfunk_core::abr::Action; +use punktfunk_core::audio::{plan_audio_budget, AudioLayout, AudioTier}; +use punktfunk_core::config::Mode; +use punktfunk_core::hud::RateCut; +use punktfunk_core::packet::USER_FLAG_REPEAT; +use punktfunk_core::quic::v2::msg::V2Message; +use punktfunk_core::quic::{ + AckReason, BitrateChanged, HostLink, HostTiming, LinkFacts, Welcome, HOST_CAP2_RAMP, + HOST_CAP2_REPEAT_MARK, HOST_CAP_AUDIO_RED, +}; +use punktfunk_core::session::Session; +use std::time::Instant; + +/// Host timings held between two ticks, the native plane's bound. +const HOST_TIMINGS: usize = 512; + +/// What the session read since the last tick. +#[derive(Default)] +struct Since { + acks: Vec<(u32, Option)>, + mode: Option, + /// `(sum µs, samples)`: the page's arrival → decoded, and the host's encode share. + decode_lat: (u64, u32), + encode_lat: (u64, u32), + host_timings: Vec, + keyframe_asks: u32, +} + +impl Since { + /// One `BitrateChanged` body. One that does not parse is skipped. + fn on_ack(&mut self, body: &[u8]) { + if let Ok(ack) = BitrateChanged::from_body(body) { + self.acks.push((ack.bitrate_kbps, ack.reason)); + } + } +} + +/// One session's driver, and what it has heard since its last tick. +pub struct Abr { + feed: Feed, + since: Since, + /// The ring's drops before this session: the driver counts the session's own. + drops_at_start: u64, + /// What the overlay shows as of the last closed window: the rate, and why it was last cut + /// (a `RateCut` code, `0` none). + pub target_kbps: u32, + pub rate_cut: u8, +} + +impl Abr { + /// `bitrate_kbps` is what `Hello` asked for, `0` Automatic; `ring_drops` the ring's count now. + pub fn new(welcome: &Welcome, bitrate_kbps: u32, ring_drops: u64) -> Abr { + let m = welcome.mode; + let facts = Facts { + bitrate_kbps, + resolved_bitrate_kbps: welcome.bitrate_kbps, + codec: welcome.codec, + bit_depth: welcome.bit_depth, + chroma_format: welcome.chroma_format, + refresh_hz: m.refresh_hz, + stream_cap_kbps: punktfunk_core::abr::stream_ceiling_kbps( + m.width, + m.height, + m.refresh_hz, + welcome.codec, + welcome.bit_depth, + welcome.chroma_format, + ), + // The page decodes no PyroWave. + pyrowave_floor_kbps: None, + // The page takes Opus only, so the reservation is Opus's default tier. + audio_reserved_kbps: plan_audio_budget( + welcome.bitrate_kbps, + welcome.audio_channels, + AudioLayout::from_wire(welcome.audio_layout).unwrap_or_default(), + AudioTier::default(), + welcome.host_caps & HOST_CAP_AUDIO_RED != 0, + ) + .kbps, + marks_repeats: welcome.host_caps2 & HOST_CAP2_REPEAT_MARK != 0, + serves_ramp: welcome.host_caps2 & HOST_CAP2_RAMP != 0, + probe_only: false, + }; + let feed = Feed::new( + facts, + HostLink::default(), + LinkFacts::default(), + Instant::now(), + ); + Abr { + feed, + since: Since::default(), + drops_at_start: ring_drops, + target_kbps: 0, + rate_cut: 0, + } + } + + pub fn on_ack(&mut self, body: &[u8]) { + self.since.on_ack(body); + } + + /// The host accepted a new mode. + pub fn on_mode(&mut self, mode: Mode) { + self.since.mode = Some(mode); + } + + /// One access unit for the decoder. A repeat is the host's idle keepalive, not content. + pub fn on_au(&mut self, flags: u32) { + self.feed.driver.on_au(flags & USER_FLAG_REPEAT != 0); + } + + /// One frame's arrival → decoded time. + pub fn on_decode_us(&mut self, us: u64) { + let d = &mut self.since.decode_lat; + (d.0, d.1) = (d.0 + us, d.1 + 1); + } + + pub fn on_host_timing(&mut self, t: &HostTiming) { + let s = &mut self.since; + if let Some(st) = &t.stages { + s.encode_lat = (s.encode_lat.0 + u64::from(st.encode_us), s.encode_lat.1 + 1); + } + if s.host_timings.len() < HOST_TIMINGS { + s.host_timings.push(*t); + } + } + + /// A recovery keyframe ask went out. + pub fn on_keyframe_ask(&mut self) { + self.since.keyframe_asks += 1; + } + + /// Tick the driver with what was read since the last pump, and send what it asks for. + /// `send` writes a `SetBitrate` and says whether the control stream took it. + pub fn tick(&mut self, session: &mut Session, ring_drops: u64, send: impl FnMut(u32) -> bool) { + let s = std::mem::take(&mut self.since); + let tick = self.feed.tick(Inputs { + // No clock offset: the driver reads no one-way delay. The shard delays are drained + // anyway, to stay bounded. + clock_offset_ns: 0, + stats: session.stats(), + loss_positions: session.take_loss_positions(), + sock_drops: Some(ring_drops.saturating_sub(self.drops_at_start)), + shard_delays: session.take_shard_delays().collect(), + probe_active: false, + mode: s.mode, + acks: s.acks, + decode_lat: s.decode_lat, + encode_lat: s.encode_lat, + host_timings: s.host_timings, + keyframe_asks: s.keyframe_asks, + ..Inputs::default() + }); + let unsent = apply(tick.actions, send); + let d = &mut self.feed.driver; + for kbps in unsent.requests { + d.on_request_dropped(kbps); + } + for _ in 0..unsent.probes { + d.on_probe_dropped(); + } + if tick.window.is_some() { + self.target_kbps = d.target_kbps(); + self.rate_cut = d + .last_cut() + .and_then(RateCut::of_reason) + .map_or(0, |c| c as u8); + } + } +} + +/// What a tick asked for that never went out, for the driver to hear back. +#[derive(Debug, Default, PartialEq)] +struct Unsent { + requests: Vec, + probes: u32, +} + +/// Send what a tick asked for, in order. +fn apply(actions: Vec, mut send: impl FnMut(u32) -> bool) -> Unsent { + let mut unsent = Unsent::default(); + for action in actions { + match action { + Action::SetBitrate(kbps) => { + if !send(kbps) { + unsent.requests.push(kbps); + } + } + // The page asks for no burst: each goes back unsent. + Action::Probe { .. } => unsent.probes += 1, + // The window report, frame shape and link rate ride `Feedback`'s levels, which the + // page does not send. Only a burst raises `Keyframe` and `AbandonProbe`. + Action::Report { .. } + | Action::Shape(_) + | Action::LinkRate(_) + | Action::Keyframe + | Action::AbandonProbe => {} + } + } + unsent +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every ack reaches the next tick in arrival order, its reason with it. + #[test] + fn acks_queue_for_the_next_tick() { + let mut since = Since::default(); + let short = Some(AckReason::EncoderLimit); + for ack in [ + BitrateChanged { + bitrate_kbps: 8_000, + reason: short, + }, + BitrateChanged { + bitrate_kbps: 20_000, + reason: None, + }, + ] { + let frame = ack.encode_v2(); + let (ty, body, _) = punktfunk_core::quic::v2::field::split_frame( + &frame, + punktfunk_core::quic::v2::registry::max_body, + ) + .unwrap() + .unwrap(); + assert_eq!(ty, BitrateChanged::TYPE); + since.on_ack(body); + } + since.on_ack(&[0xff]); + assert_eq!(since.acks, vec![(8_000, short), (20_000, None)]); + } + + /// A rate the control stream took is sent; one it did not goes back, as every probe does. + #[test] + fn an_unsent_rate_goes_back_to_the_driver() { + let probe = Action::Probe { + target_kbps: 50_000, + duration_ms: 300, + ramp: true, + }; + let actions = vec![Action::LinkRate(100_000), Action::SetBitrate(12_000), probe]; + let mut sent = Vec::new(); + let open = apply(actions.clone(), |kbps| { + sent.push(kbps); + true + }); + assert_eq!(sent, vec![12_000]); + assert_eq!( + open, + Unsent { + requests: vec![], + probes: 1 + } + ); + let closed = apply(actions, |_| false); + assert_eq!( + closed, + Unsent { + requests: vec![12_000], + probes: 1 + } + ); + } +} diff --git a/packages/stream/rust/main.rs b/packages/stream/rust/main.rs index f937390..8f000a3 100644 --- a/packages/stream/rust/main.rs +++ b/packages/stream/rust/main.rs @@ -29,6 +29,8 @@ fn main() { // Neither is gated: nothing in them is browser-specific, and off wasm is where their tests run. // The SPAKE2 half in particular is checked against the host's own role B, which a wasm-only // test would never execute. +#[cfg_attr(not(target_family = "wasm"), allow(dead_code))] +mod abr; #[cfg(target_family = "wasm")] mod audio; #[cfg_attr(not(target_family = "wasm"), allow(dead_code))] diff --git a/packages/stream/rust/session.rs b/packages/stream/rust/session.rs index 6d27b58..af025c2 100644 --- a/packages/stream/rust/session.rs +++ b/packages/stream/rust/session.rs @@ -13,20 +13,23 @@ //! R3 holds throughout. An access unit leaves as a `(ptr, len)` view for `VideoDecoder`; no //! decoded pixel ever enters this heap. +use crate::abr::Abr; use crate::credential; use crate::recovery::{self, Ask, Asks, Step}; use crate::transport::{send_feedback, WebTransportDatagrams}; use punktfunk_core::config::{CompositorPref, GamepadPref, Mode, Role}; use punktfunk_core::error::PunktfunkError; use punktfunk_core::hud::{self, StatsSnapshot, StatsVerbosity}; +use punktfunk_core::packet::FLAG_PROBE; use punktfunk_core::quic::v2::features::FeatureSet; use punktfunk_core::quic::v2::field::split_frame; use punktfunk_core::quic::v2::hello::{ClientHello, Ready, ServerHello}; use punktfunk_core::quic::v2::msg::{Redirect, V2Message}; use punktfunk_core::quic::v2::registry as reg; use punktfunk_core::quic::{ - AccessUpdate, AuthChallenge, CursorRenderMode, CursorShape, Hello, LaunchOutcome, LinkFacts, - PairChallenge, PairResult, Reconfigure, Reconfigured, Refused, + AccessUpdate, AuthChallenge, BitrateChanged, CursorRenderMode, CursorShape, Hello, + LaunchOutcome, LinkFacts, PairChallenge, PairResult, Reconfigure, Reconfigured, Refused, + SetBitrate, }; use punktfunk_core::reanchor::{GateVerdict, ReanchorGate}; use punktfunk_core::session::{MediaV2, Session}; @@ -63,6 +66,10 @@ struct Client { /// than emscripten's default stack. Constructing a `Client` that embedded one by value /// overflowed before the function body ran, which surfaces as an out-of-bounds trap. session: Option>, + /// Automatic bitrate, built with the session and dropped with it. + abr: Option>, + /// The rate `Hello` asked for; `0` is Automatic. + bitrate_kbps: u32, /// The negotiated video format, for the page's `VideoDecoder.configure`. codec: u8, width: u32, @@ -129,6 +136,8 @@ impl Client { phase: Phase::Idle, inbox: Vec::new(), session: None, + abr: None, + bitrate_kbps: 0, codec: 0, width: 0, height: 0, @@ -201,6 +210,7 @@ pub extern "C" fn pf_session_reset() { let mut c = c.borrow_mut(); c.phase = Phase::Idle; c.session = None; + c.abr = None; c.inbox.clear(); c.frames = 0; c.codec = 0; @@ -455,7 +465,8 @@ pub unsafe extern "C" fn pf_session_hello( send(&ClientHello { hello, client_label: None, - abr_features: 0, + // Acks name why a rate fell short, which the driver learns host caps from. + abr_features: punktfunk_core::quic::EXT_ABR_ACK_REASON, preset: None, link: LinkFacts::default(), probe_only: false, @@ -471,6 +482,7 @@ pub unsafe extern "C" fn pf_session_hello( screens: Vec::new(), }); c.fps = fps; + c.bitrate_kbps = bitrate_kbps; c.phase = Phase::Offered; 1 }) @@ -645,11 +657,19 @@ fn on_frame(c: &mut Client, ty: u64, body: &[u8]) { if r.accepted { c.width = r.mode.width; c.height = r.mode.height; + if let Some(abr) = c.abr.as_mut() { + abr.on_mode(r.mode); + } // SAFETY: plain integers to a JavaScript function that returns before this does. unsafe { video_config(c) }; } } } + BitrateChanged::TYPE => { + if let Some(abr) = c.abr.as_mut() { + abr.on_ack(body); + } + } AccessUpdate::TYPE => { if let Ok(u) = AccessUpdate::from_body(body) { c.grants = u.grants; @@ -726,6 +746,8 @@ fn on_server_hello(c: &mut Client, server: ServerHello) { match Session::new(cfg, media, Box::new(WebTransportDatagrams)) { Ok(session) => { c.session = Some(Box::new(session)); + let drops = u64::from(crate::transport::pf_rx_dropped()); + c.abr = Some(Box::new(Abr::new(&welcome, c.bitrate_kbps, drops))); c.phase = Phase::Live; send(&Ready {}); // SAFETY: plain integers to a JavaScript function that returns before this does. @@ -764,6 +786,7 @@ pub extern "C" fn pf_session_pump() -> u32 { pending_rfi, last_ask, asks, + abr, .. } = &mut *c; let codec = *codec; @@ -777,6 +800,10 @@ pub extern "C" fn pf_session_pump() -> u32 { if frame.data.is_empty() { break; } + // Probe filler measures the link and is never a picture. + if frame.flags & u32::from(FLAG_PROBE) != 0 { + continue; + } // Frames are numbered consecutively: a jump is a lost frame, and everything after it // predicts from a picture the decoder never got. Hold at once (the decoder still takes // every frame, so a wave can heal it) and ask for the range. @@ -786,6 +813,9 @@ pub extern "C" fn pf_session_pump() -> u32 { Step::Straggler => continue, } asks.on_frame(frame.frame_index, frame.flags); + if let Some(abr) = abr.as_mut() { + abr.on_au(frame.flags); + } hud.note_received(frame.pts_ns, frame.received_ns, frame.data.len(), true); if receipts.len() >= RECEIPTS { receipts.pop_front(); @@ -821,11 +851,22 @@ pub extern "C" fn pf_session_pump() -> u32 { if let Some(ask) = recovery::take_ask(want_keyframe, pending_rfi) { *last_ask = Some(now); send_feedback(&asks.ask(ask, now)); + if let (Ask::Keyframe, Some(abr)) = (ask, abr.as_mut()) { + abr.on_keyframe_ask(); + } } } if let Some(again) = asks.resend(now) { send_feedback(&again); } + if let Some(abr) = abr.as_mut() { + let drops = u64::from(crate::transport::pf_rx_dropped()); + // Live is the control stream's lifetime: its close resets the session. + abr.tick(session, drops, |kbps| { + send(&SetBitrate { bitrate_kbps: kbps }); + true + }); + } delivered }) } @@ -854,6 +895,9 @@ pub extern "C" fn pf_gate_no_output() { if c.gate.on_no_output(now) && c.phase == Phase::Live { c.last_ask = Some(now); send_feedback(&c.asks.ask(Ask::Keyframe, now)); + if let Some(abr) = c.abr.as_mut() { + abr.on_keyframe_ask(); + } } }); } @@ -888,11 +932,14 @@ pub extern "C" fn pf_hud_decoded(pts_us: f64, decoded_ms: f64) { let key = pts_us as u64; let decoded_ns = ms_to_ns(decoded_ms); CLIENT.with(|c| { - let c = c.borrow(); + let mut c = c.borrow_mut(); c.hud.note_decoded(key * 1000, decoded_ns); if let Some(&(_, received_ns)) = c.receipts.iter().rev().find(|(p, _)| *p == key) { - c.hud - .note_decode_us(decoded_ns.saturating_sub(received_ns) / 1000, false); + let us = decoded_ns.saturating_sub(received_ns) / 1000; + c.hud.note_decode_us(us, false); + if let Some(abr) = c.abr.as_mut() { + abr.on_decode_us(us); + } } }); } @@ -914,7 +961,7 @@ pub extern "C" fn pf_hud_drain(skipped: u32) { CLIENT.with(|c| { let mut c = c.borrow_mut(); c.hud.note_skipped(skipped, 0); - let counters = c.session.as_ref().map_or_else(hud::Counters::default, |s| { + let mut counters = c.session.as_ref().map_or_else(hud::Counters::default, |s| { let st = s.stats(); hud::Counters { frames_dropped: st.frames_dropped, @@ -922,7 +969,11 @@ pub extern "C" fn pf_hud_drain(skipped: u32) { ..hud::Counters::default() } }); + if let Some(abr) = &c.abr { + (counters.target_kbps, counters.rate_cut) = (abr.target_kbps, abr.rate_cut); + } let mut snap = c.hud.drain(&counters); + snap.auto_rate = c.bitrate_kbps == 0; (snap.width, snap.height, snap.refresh_hz) = (c.width, c.height, c.fps); snap.codec = hud::codec_label(c.codec).into(); snap.decoder = "WebCodecs".into(); @@ -966,7 +1017,13 @@ pub extern "C" fn pf_hud_text_ptr() -> *const u8 { /// One per-frame host timing datagram (0xCF): the host's own share, for the overlay. pub fn on_host_timing(datagram: &[u8]) { if let Some(t) = punktfunk_core::quic::decode_host_timing_datagram(datagram) { - CLIENT.with(|c| c.borrow().hud.note_host_timing(&t)); + CLIENT.with(|c| { + let mut c = c.borrow_mut(); + c.hud.note_host_timing(&t); + if let Some(abr) = c.abr.as_mut() { + abr.on_host_timing(&t); + } + }); } }