diff --git a/crates/pf-nx-app/src/core/model.rs b/crates/pf-nx-app/src/core/model.rs index c515fc5..513abef 100644 --- a/crates/pf-nx-app/src/core/model.rs +++ b/crates/pf-nx-app/src/core/model.rs @@ -141,6 +141,47 @@ pub fn upsert_known_host(hosts: &mut Vec, mut new: KnownHost) -> Opti None } +/// Follows a paired host to the address it now advertises from, matched by its pinned +/// fingerprint, so a DHCP move doesn't leave the record offline and a re-add doesn't save +/// it twice. `advertised` says whether some advert currently comes from an address: a record +/// whose own address still advertises is a second interface, not a move, and stays put. +/// +/// The advert is unauthenticated, but the pin is what every connection checks, so a forged +/// one can only point the record at an address whose handshake then fails. +/// +/// With no record at the advert's address, the record that pins `fp_hex` moves there. With +/// one there that pins the same certificate, the others pinning it are dropped as +/// duplicates. Returns whether anything changed. +pub fn follow_moved_host( + hosts: &mut Vec, + fp_hex: &str, + addr: &str, + port: u16, + advertised: impl Fn(&str, u16) -> bool, +) -> bool { + if fp_hex.is_empty() { + return false; + } + let stale = + |h: &KnownHost| h.fp_hex == fp_hex && (h.addr.as_str(), h.port) != (addr, port) && !advertised(&h.addr, h.port); + match hosts.iter().find(|h| (h.addr.as_str(), h.port) == (addr, port)) { + Some(here) if here.fp_hex == fp_hex => { + let before = hosts.len(); + hosts.retain(|h| !stale(h)); + hosts.len() != before + } + Some(_) => false, + None => { + let Some(moved) = hosts.iter_mut().find(|h| stale(h)) else { + return false; + }; + moved.addr = addr.to_string(); + moved.port = port; + true + } + } +} + fn unique_profile_name(catalog: &[StreamPreset], wanted: &str) -> String { let taken = |name: &str| catalog.iter().any(|p| p.name == name); if !taken(wanted) { @@ -273,12 +314,26 @@ impl GamepadType { } } -/// Automatic bitrate's ceiling, handed to `punktfunk_core::abr` through `run` -/// (`PUNKTFUNK_ABR_MAX_MBPS`). It binds every codec, so it sits above Automatic `PyroWave`'s +/// Automatic bitrate's ceiling where `PyroWave` decodes, handed to `punktfunk_core::abr` through +/// `run` (`PUNKTFUNK_ABR_MAX_MBPS`). It binds every codec, so it sits above Automatic `PyroWave`'s /// quality floor (0.5 bit/pixel), below which core clamps a `PyroWave` session to that floor. -/// HEVC and H.264 stop well below it, at the link's and the decoder's headroom. pub const BITRATE_MAX_KBPS: u32 = 650_000; +/// Automatic bitrate's ceiling where only NVDEC decodes: the rate a pinned stream ran clean at, +/// handheld at stock clocks. Core finds the link's edge by crossing it, and its climbs to +/// 22-25 Mb/s preceded every 60-110 ms delay spike (2026-10-31). Under the ceiling it still cuts +/// for a dock or a weak Wi-Fi. A raised CPU clock is what would carry more. +pub const NVDEC_BITRATE_MAX_KBPS: u32 = 20_000; + +/// Automatic bitrate's ceiling for a title whose video lane can or cannot decode `PyroWave`. +pub const fn bitrate_max_kbps(pyrowave: bool) -> u32 { + if pyrowave { + BITRATE_MAX_KBPS + } else { + NVDEC_BITRATE_MAX_KBPS + } +} + /// 70% of measured goodput (headroom for FEC, loss). /// Integer arithmetic order matters (`/ 10 * 7`, not `* 0.7`) — all clients must agree exactly. pub fn recommended_bitrate_kbps(throughput_kbps: u32) -> u32 { @@ -430,4 +485,48 @@ mod tests { assert_eq!(hosts[0].id, h.id); assert_eq!(hosts[0].game_profile("doom"), Some("p1")); } + + fn paired_at(addr: &str, fp: u8) -> KnownHost { + let mut h = KnownHost { + shared: pf_client_core::trust::KnownHost { + name: "desk".into(), + addr: addr.into(), + port: 9777, + ..Default::default() + }, + ..Default::default() + }; + h.set_fingerprint([fp; 32]); + h + } + + /// The 2026-11-02 settings: the router moved the host from .137 to .10, the record stayed + /// on .137 and showed offline, and re-adding it saved the same pin a second time. + #[test] + fn a_paired_host_follows_its_advert_to_a_new_address() { + let fp = "5a".repeat(32); + let quiet = |_: &str, _: u16| false; + + let mut hosts = vec![paired_at("192.168.0.137", 0x5a)]; + assert!(follow_moved_host(&mut hosts, &fp, "192.168.0.10", 9777, quiet)); + assert_eq!(hosts[0].addr, "192.168.0.10"); + + let mut hosts = vec![paired_at("192.168.0.137", 0x5a), paired_at("192.168.0.10", 0x5a)]; + assert!(follow_moved_host(&mut hosts, &fp, "192.168.0.10", 9777, quiet)); + assert_eq!(hosts.len(), 1, "the stale duplicate is dropped"); + assert_eq!(hosts[0].addr, "192.168.0.10"); + + // A second interface that still advertises is not a move. + let mut hosts = vec![paired_at("192.168.0.137", 0x5a)]; + let both = |a: &str, _: u16| a == "192.168.0.137"; + assert!(!follow_moved_host(&mut hosts, &fp, "192.168.0.10", 9777, both)); + assert_eq!(hosts[0].addr, "192.168.0.137"); + + // Another certificate, or another host already at the address, moves nothing. + let mut hosts = vec![paired_at("192.168.0.137", 0x11)]; + assert!(!follow_moved_host(&mut hosts, &fp, "192.168.0.10", 9777, quiet)); + let mut hosts = vec![paired_at("192.168.0.137", 0x5a), paired_at("192.168.0.10", 0x11)]; + assert!(!follow_moved_host(&mut hosts, &fp, "192.168.0.10", 9777, quiet)); + assert_eq!(hosts.len(), 2); + } } diff --git a/crates/pf-nx-app/src/platform/mod.rs b/crates/pf-nx-app/src/platform/mod.rs index d9432f6..1401bfa 100644 --- a/crates/pf-nx-app/src/platform/mod.rs +++ b/crates/pf-nx-app/src/platform/mod.rs @@ -9,6 +9,7 @@ pub mod fake; use std::io::Write; use std::path::Path; +use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; use std::sync::Arc; use anyhow::Result; @@ -122,6 +123,9 @@ pub struct FrameReport { pub replaced: u32, /// Decoded pictures dropped unseen since the last frame, pushed out of a full smoothing queue. pub overflow: u32, + /// How long an earlier frame's composition ran on the GPU (the one whose slot this frame + /// reused), where the GPU can time it. + pub gpu_us: Option, } /// A read-back frame. @@ -159,6 +163,42 @@ pub struct StreamSpec { pub smoothing: usize, /// The session's overlay figures: lanes note decode, the presenter notes display. pub hud: Arc, + /// Where the presenter leaves each picture's capture-to-glass time for the audio's A/V + /// sync measure. + pub video_e2e: VideoE2e, +} + +/// The video's capture-to-glass time in core's cells: the presenter writes it, the audio route +/// compares its own against it. `0` until a picture reaches the glass. +#[derive(Clone, Debug, Default)] +pub struct VideoE2e { + e2e_ns: Arc, + /// Host clock minus this one, kept current by core. + clock_offset_ns: Arc, +} + +/// A capture-to-glass time past this is a clock step, not a picture. +const E2E_SANE_NS: u64 = 10_000_000_000; + +impl VideoE2e { + pub fn new(e2e_ns: Arc, clock_offset_ns: Arc) -> Self { + Self { + e2e_ns, + clock_offset_ns, + } + } + + /// A picture captured at host time `pts_ns` reached the glass at `displayed_ns`, this + /// clock's realtime. + pub fn note(&self, pts_ns: u64, displayed_ns: u64) { + let offset = self.clock_offset_ns.load(Ordering::Relaxed); + let e2e = i128::from(displayed_ns) + i128::from(offset) - i128::from(pts_ns); + if let Ok(ns) = u64::try_from(e2e) { + if ns > 0 && ns < E2E_SANE_NS { + self.e2e_ns.store(ns, Ordering::Relaxed); + } + } + } } /// The console's audio output. Opening a port is cheap; a session holds one. @@ -173,6 +213,11 @@ pub trait AudioPort: Send { /// Plays exactly one grain, blocking until the device takes it. fn write(&mut self, pcm: &[f32]) -> Result<()>; + + /// Frames handed to the device and not yet played, which the next grain queues behind. + fn queued_frames(&self) -> usize { + 0 + } } /// One pad's state at one instant, already in punktfunk's wire vocabulary. @@ -277,4 +322,37 @@ pub trait Lifecycle { fn hide_splash(&self); fn poll_events(&mut self, out: &mut Vec); fn exit(&self) -> !; + + /// The clocks the system runs the title at now, for the logs. `None` where they cannot be + /// read: a dev box, or a console before firmware 8.0.0. + fn clocks(&self) -> Option { + None + } + + /// Shows the system's keyboard over the title and blocks until the player closes it: the + /// text they confirmed, or `None` when they cancelled or there is no such keyboard. + fn keyboard(&mut self) -> Option { + None + } +} + +/// The CPU, GPU and memory clocks, in Hz. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Clocks { + pub cpu_hz: u32, + pub gpu_hz: u32, + pub emc_hz: u32, +} + +impl std::fmt::Display for Clocks { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mhz = |hz: u32| f64::from(hz) / 1e6; + write!( + f, + "gpu {:.1} cpu {:.1} emc {:.1} MHz", + mhz(self.gpu_hz), + mhz(self.cpu_hz), + mhz(self.emc_hz) + ) + } } diff --git a/crates/pf-nx-app/src/runtime/menu.rs b/crates/pf-nx-app/src/runtime/menu.rs index 672c4fe..8127425 100644 --- a/crates/pf-nx-app/src/runtime/menu.rs +++ b/crates/pf-nx-app/src/runtime/menu.rs @@ -222,7 +222,7 @@ pub(super) fn run(app: &mut App, notice: Option) -> Result { })?; frames += 1; if let Some(report) = perf.frame(before_draw + drew, art_snapshot()) { - tracing::info!("{}", report.line()); + tracing::info!("{}{}", report.line(), app.clocks_suffix()); } if let Some(done) = testrun::menu_step(app, frames) { break 'ui done; diff --git a/crates/pf-nx-app/src/runtime/mod.rs b/crates/pf-nx-app/src/runtime/mod.rs index 20828ce..c7dccac 100644 --- a/crates/pf-nx-app/src/runtime/mod.rs +++ b/crates/pf-nx-app/src/runtime/mod.rs @@ -14,7 +14,7 @@ mod touch; use std::sync::atomic::{AtomicBool, Ordering}; -use crate::core::model::BITRATE_MAX_KBPS; +use crate::core::model::bitrate_max_kbps; use crate::platform::{Display, Lifecycle, Pads, Platform, SystemEvent}; use crate::services::power::ExitPlan; use crate::services::store; @@ -48,6 +48,13 @@ impl App { } events } + + /// `" | clocks gpu … MHz"` for a 10 s summary line, empty where the clocks cannot be read. + fn clocks_suffix(&self) -> String { + self.lifecycle + .clocks() + .map_or_else(String::new, |c| format!(" | clocks {c}")) + } } /// Sleep is on its way: no new session may start. @@ -71,14 +78,17 @@ const ABR_PROBE_KBPS: u32 = 320_000; /// The two automatic-bitrate knobs punktfunk-core reads from the environment. Set before any /// thread exists: `set_var` is not thread-safe. -fn set_abr_env() { +fn set_abr_env(pyrowave: bool) { std::env::set_var("PUNKTFUNK_ABR_PROBE_KBPS", ABR_PROBE_KBPS.to_string()); - std::env::set_var("PUNKTFUNK_ABR_MAX_MBPS", (BITRATE_MAX_KBPS / 1_000).to_string()); + std::env::set_var( + "PUNKTFUNK_ABR_MAX_MBPS", + (bitrate_max_kbps(pyrowave) / 1_000).to_string(), + ); } /// Runs the title until the player leaves it. Never returns. pub fn run(platform: Platform) -> ! { - set_abr_env(); + set_abr_env(platform.video.caps().pyrowave); let Platform { display, video, diff --git a/crates/pf-nx-app/src/runtime/stream.rs b/crates/pf-nx-app/src/runtime/stream.rs index 8231fde..92f20e5 100644 --- a/crates/pf-nx-app/src/runtime/stream.rs +++ b/crates/pf-nx-app/src/runtime/stream.rs @@ -391,7 +391,8 @@ fn stream( ring.set_facts(&facts); } ring.tick(); - if let Some(n) = take_ring_commands(&mut ring, &mut hud, connected.client(), &mut pad_type) { + let mut keyboard = false; + if let Some(n) = take_ring_commands(&mut ring, &mut hud, connected.client(), &mut pad_type, &mut keyboard) { break n; } for cmd in ring.take_cmds() { @@ -410,6 +411,10 @@ fn stream( .to .send(if open { ToInput::DialOpened } else { ToInput::DialClosed }); } + // The dial closed itself before asking: its own close went out just above. + if keyboard { + type_text(app, connected.client(), &input); + } if connected.is_session_ended() { tracing::info!("session ended: {:?}", connected.end_reason()); break if connected.end_reason() == PunktfunkEndReason::Lost { @@ -420,7 +425,7 @@ fn stream( } if let Some(core) = hud.refresh(connected, decoder) { - tracing::info!("stats: {} | {core}", perf.take_window()); + tracing::info!("stats: {} | {core}{}", perf.take_window(), app.clocks_suffix()); } hud.local = vec![format!( "present: {} frames, {} missed{}", @@ -516,6 +521,7 @@ fn take_ring_commands( hud: &mut StatsOverlay, client: &NativeClient, pad_type: &mut GamepadType, + keyboard: &mut bool, ) -> Option { while let Some(cmd) = ring.take_command() { match cmd { @@ -528,12 +534,76 @@ fn take_ring_commands( RingCommand::CyclePadType => *pad_type = pad_type.next_on_dial(), // Touch is the screen's one model so far: nothing to step to. RingCommand::CycleTouchMode => {} + // Shown once the dial's close has gone out to the input thread. + RingCommand::Keyboard => *keyboard = true, + RingCommand::Shortcut(keys) => send_chord(client, &keys), other => tracing::info!("dial: {other:?} does nothing on this console"), } } None } +/// A dial shortcut, typed on the host: every key down in order, then up in reverse, as the +/// desktop client does. A name this build does not know fires nothing. +fn send_chord(client: &NativeClient, keys: &[String]) { + use punktfunk_core::input::{key_vk, InputEvent, InputKind}; + let Some(vks) = keys.iter().map(|k| key_vk(k)).collect::>>() else { + tracing::info!("dial: shortcut {keys:?} names a key this build does not know"); + return; + }; + let key = |kind, vk: u8| InputEvent { + kind, + _pad: [0; 3], + code: u32::from(vk), + x: 0, + y: 0, + flags: 0, + }; + for &vk in &vks { + let _ = client.send_input(&key(InputKind::KeyDown, vk)); + } + for &vk in vks.iter().rev() { + let _ = client.send_input(&key(InputKind::KeyUp, vk)); + } + tracing::info!("dial: shortcut {} sent", keys.join("+")); +} + +/// The dial's Keyboard: the system keyboard over the stream, then its text typed on the host as +/// committed characters. The main thread waits on the keyboard, so no frame is presented +/// meanwhile; the stream and the audio go on. +fn type_text(app: &mut App, client: &NativeClient, input: &InputThread) { + use punktfunk_core::input::{InputEvent, InputKind}; + use punktfunk_core::quic::{GRANT_KEYBOARD, HOST_CAP_TEXT_INPUT}; + if client.host_caps() & HOST_CAP_TEXT_INPUT == 0 { + tracing::info!("keyboard: this host cannot type text"); + return; + } + if client.access_grants() & GRANT_KEYBOARD == 0 { + tracing::info!("keyboard: this host does not grant the keyboard"); + return; + } + // The pads stay off the wire while the keyboard has them, as under the dial. + let _ = input.to.send(ToInput::DialOpened); + let text = app.lifecycle.keyboard(); + let _ = input.to.send(ToInput::DialClosed); + // What the pads did meanwhile was typing, not the dial's. + while input.from.try_recv().is_ok() {} + let Some(text) = text.filter(|t| !t.is_empty()) else { + return; + }; + for c in text.chars() { + let _ = client.send_input(&InputEvent { + kind: InputKind::TextInput, + _pad: [0; 3], + code: u32::from(c), + x: 0, + y: 0, + flags: 0, + }); + } + tracing::info!("keyboard: typed {} characters on the host", text.chars().count()); +} + /// Copies the controller-mouse facts the dial shows from the session; whether they changed. fn follow_pad_mouse(client: &NativeClient, facts: &mut RingFacts) -> bool { let target = client.live_pads(); @@ -582,6 +652,8 @@ struct PerfWindow { replaced: u32, overflow: u32, decode_to_present_us: Vec, + /// The composition's GPU time per frame, where the presenter can time it. + gpu_us: Vec, } impl StreamPerf { @@ -604,6 +676,9 @@ impl StreamPerf { } w.decode_to_present_us.push(us); } + if let Some(us) = report.gpu_us { + w.gpu_us.push(us); + } } /// The presenter's figures since the last call, and a new window. @@ -611,10 +686,13 @@ impl StreamPerf { let mut w = std::mem::take(&mut self.window); let secs = w.since.map_or(0.0, |t| t.elapsed().as_secs_f64()); w.decode_to_present_us.sort_unstable(); + w.gpu_us.sort_unstable(); let ms = |us: Option<&u32>| us.map_or("-".to_string(), |&us| format!("{:.1}", f64::from(us) / 1000.0)); + let fine = |us: Option<&u32>| us.map_or("-".to_string(), |&us| format!("{:.2}", f64::from(us) / 1000.0)); let d = &w.decode_to_present_us; + let g = &w.gpu_us; format!( - "present {secs:.1}s: {} frames, {} missed vblanks, {} pictures, {} unseen ({} replaced, {} queue full), decode to present p50 {} p95 {} max {} ms", + "present {secs:.1}s: {} frames, {} missed vblanks, {} pictures, {} unseen ({} replaced, {} queue full), decode to present p50 {} p95 {} max {} ms, compose on the gpu p50 {} p95 {} max {} ms", w.frames, w.missed, w.pictures, @@ -624,6 +702,9 @@ impl StreamPerf { ms(d.get(d.len() / 2)), ms(d.get(d.len() * 95 / 100)), ms(d.last()), + fine(g.get(g.len() / 2)), + fine(g.get(g.len() * 95 / 100)), + fine(g.last()), ) } diff --git a/crates/pf-nx-app/src/services/store/mod.rs b/crates/pf-nx-app/src/services/store/mod.rs index 28d5e54..6ce2c7f 100644 --- a/crates/pf-nx-app/src/services/store/mod.rs +++ b/crates/pf-nx-app/src/services/store/mod.rs @@ -15,7 +15,7 @@ use anyhow::{Context, Result}; use crate::core::VERSION; pub use crate::core::model::{ - seed_new_host_profiles, upsert_known_host, CodecPref, ExitAction, GamepadType, Persisted, + follow_moved_host, seed_new_host_profiles, upsert_known_host, CodecPref, ExitAction, GamepadType, Persisted, }; pub use crate::core::settings::NxSettings; pub use crate::services::paths::app_dir; diff --git a/crates/pf-nx-app/src/session/audio_out.rs b/crates/pf-nx-app/src/session/audio_out.rs index 6398986..5b33dae 100644 --- a/crates/pf-nx-app/src/session/audio_out.rs +++ b/crates/pf-nx-app/src/session/audio_out.rs @@ -3,14 +3,21 @@ //! //! The port blocks for a grain and nothing rate-matches it to the host, so depth is steered by //! punktfunk-core's shared [`JitterPolicy`], the same one every client's ring runs. +//! +//! Every packet is also placed against the picture on the glass with core's [`AvSync`]: where +//! the audio will play in the host's capture clock, next to where the video is. That only +//! measures (plan WP5): the offset goes to the stats line and the `audio[10s]` line, and the +//! ring is not steered by it until a long session shows it drift. The same line splits the +//! underruns by whether the audio before them was silent, and counts the host's pauses. use std::collections::VecDeque; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU32, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Condvar, Mutex}; -use std::time::Duration; +use std::time::{Duration, Instant}; use anyhow::{Context, Result}; -use punktfunk_core::audio::{crossfade_drop, crossfade_insert, JitterPolicy, JitterTuning}; +use punktfunk_core::audio::{crossfade_drop, crossfade_insert, AvSync, AvSyncObservation, JitterPolicy, JitterTuning}; +use punktfunk_core::client::NativeClient; use crate::core::media::{AudioFormat, AudioSink, Samples}; use crate::platform::AudioPort; @@ -38,10 +45,66 @@ fn playout(channels: u8) -> punktfunk_core::audio::AudioPlayout { } } +/// How often the route logs its window. +const WINDOW: Duration = Duration::from_secs(10); +/// A packet this far below full scale is silence, so an underrun after it cannot be heard: +/// −50 dBFS. +const SILENT_PEAK: f32 = 0.003_16; +/// No packet for this long is the host pausing, not jitter. +const PAUSE: Duration = Duration::from_millis(100); + +/// Core's cells the route reads and writes: the clock skew and the video's capture-to-glass +/// time in, the smoothed A/V offset and the ring's depth out (the stats line shows both). +#[derive(Clone, Default)] +pub struct SyncCells { + pub clock_offset_ns: Arc, + pub video_e2e_ns: Arc, + pub av_offset_ms: Arc, + pub buffer_ms: Arc, +} + +impl SyncCells { + pub fn of(client: &NativeClient) -> Self { + Self { + clock_offset_ns: client.clock_offset_shared(), + video_e2e_ns: client.video_e2e_shared(), + av_offset_ms: client.audio_av_offset_shared(), + buffer_ms: client.audio_buffer_ms_shared(), + } + } +} + struct Ring { samples: VecDeque, policy: JitterPolicy, depth_ms: u32, + sync: AvSync, + /// The peak of the last packet fed, which tells a silent underrun from an audible one. + last_peak: f32, + last_fed: Option, + window: Window, +} + +/// One [`WINDOW`] of what the route did, filled by both threads under the ring's lock. +#[derive(Default)] +struct Window { + grains: u32, + /// Grains of silence while the ring primed or re-primed: not underruns. + priming: u32, + silent_underruns: u32, + audible_underruns: u32, + sheds: u32, + inserts: u32, + depth_min_ms: Option, + depth_max_ms: u32, + packets: u32, + pauses: u32, + longest_gap: Duration, + /// Unsmoothed A/V offsets, ms: positive is audio late. + av_sum_ms: i64, + av_n: u32, + av_min_ms: Option, + av_max_ms: Option, } struct Shared { @@ -49,6 +112,9 @@ struct Shared { fed: Condvar, stop: AtomicBool, underruns: AtomicU64, + /// The port's queue after the last write, in frames: audio that plays before anything fed now. + port_queued: AtomicUsize, + cells: SyncCells, } /// An [`AudioSink`] over one open port, plus the thread that plays it. @@ -59,17 +125,23 @@ pub struct AudioOutSink { } impl AudioOutSink { - pub fn start(port: Box, channels: u8) -> Result { + pub fn start(port: Box, channels: u8, cells: SyncCells) -> Result { let rate = crate::session::audio::SAMPLE_RATE; let shared = Arc::new(Shared { ring: Mutex::new(Ring { samples: VecDeque::with_capacity(rate as usize * usize::from(channels) / 5), policy: playout(channels).jitter(), depth_ms: 0, + sync: playout(channels).sync(), + last_peak: 0.0, + last_fed: None, + window: Window::default(), }), fed: Condvar::new(), stop: AtomicBool::new(false), underruns: AtomicU64::new(0), + port_queued: AtomicUsize::new(0), + cells, }); let thread_shared = shared.clone(); let writer = std::thread::Builder::new() @@ -115,12 +187,47 @@ impl AudioSink for AudioOutSink { } } - fn feed(&self, samples: Samples<'_>, _host_pts_ns: u64) -> Result<()> { + fn feed(&self, samples: Samples<'_>, host_pts_ns: u64) -> Result<()> { let Samples::F32(pcm) = samples; - let mut ring = self.shared.ring.lock().expect("audio ring poisoned"); - ring.samples.extend(pcm.iter().copied()); + let peak = pcm.iter().fold(0f32, |m, s| m.max(s.abs())); + let now = Instant::now(); + let shared = &self.shared; + let mut ring = shared.ring.lock().expect("audio ring poisoned"); + let r = &mut *ring; + if let Some(gap) = r.last_fed.map(|t| now - t) { + r.window.pauses += u32::from(gap >= PAUSE); + r.window.longest_gap = r.window.longest_gap.max(gap); + } + r.last_fed = Some(now); + r.last_peak = peak; + r.window.packets += 1; + // Everything that plays before this packet: the ring, and the port's queue. + let ahead = r.samples.len() + shared.port_queued.load(Ordering::Relaxed) * usize::from(self.channels); + let cells = &shared.cells; + let video_e2e_ns = cells.video_e2e_ns.load(Ordering::Relaxed); + let o = AvSyncObservation { + pts_ns: host_pts_ns, + now_local_ns: punktfunk_core::client::now_realtime_ns(), + clock_offset_ns: cells.clock_offset_ns.load(Ordering::Relaxed), + buffered_ahead: ahead, + // What audout adds past its queue is not known here: a constant, so drift still shows. + output_latency_ns: 0, + video_e2e_ns: (video_e2e_ns > 0).then_some(video_e2e_ns), + }; + if let Some(ms) = raw_offset_ms(&o, self.channels) { + let w = &mut r.window; + w.av_sum_ms += ms; + w.av_n += 1; + w.av_min_ms = Some(w.av_min_ms.map_or(ms, |m| m.min(ms))); + w.av_max_ms = Some(w.av_max_ms.map_or(ms, |m| m.max(ms))); + } + r.sync.observe(o); + cells + .av_offset_ms + .store(i64::from(r.sync.offset_ms()), Ordering::Relaxed); + r.samples.extend(pcm.iter().copied()); drop(ring); - self.shared.fed.notify_one(); + shared.fed.notify_one(); Ok(()) } @@ -140,8 +247,9 @@ impl AudioSink for AudioOutSink { fn write_loop(shared: &Shared, mut port: Box, channels: usize) { let want = port.grain_frames() * channels; let mut grain = vec![0f32; want]; + let mut since = Instant::now(); while !shared.stop.load(Ordering::Relaxed) { - { + let window = { let mut ring = shared.ring.lock().expect("audio ring poisoned"); if ring.samples.is_empty() && !ring.policy.is_primed() { // Nothing to play yet: wait for the decoder rather than spinning on silence. @@ -152,18 +260,71 @@ fn write_loop(shared: &Shared, mut port: Box, channels: usize) { .0; } fill(&mut ring, &mut grain, &shared.underruns); - } + shared.cells.buffer_ms.store(ring.depth_ms, Ordering::Relaxed); + (since.elapsed() >= WINDOW).then(|| { + let (target_ms, offset_ms) = (ring.policy.target_ms(), ring.sync.offset_ms()); + (std::mem::take(&mut ring.window), target_ms, offset_ms) + }) + }; if let Err(e) = port.write(&grain) { tracing::warn!("audio out: {e:#}"); std::thread::sleep(Duration::from_millis(5)); } + shared.port_queued.store(port.queued_frames(), Ordering::Relaxed); + // Off the lock and after the grain is queued, so the port has its queue to play. + if let Some((w, target_ms, offset_ms)) = window { + log_window(&w, since.elapsed(), target_ms, offset_ms, port.queued_frames()); + since = Instant::now(); + } } } +/// Where the audio in `o` will play against its picture, in ms (positive is audio late): the +/// arithmetic of [`AvSync::observe`] without the smoothing, for the window's spread. `None` +/// before a picture reached the glass. +fn raw_offset_ms(o: &AvSyncObservation, channels: u8) -> Option { + let video = i128::from(o.video_e2e_ns?); + let per_ms = i128::from(crate::session::audio::SAMPLE_RATE / 1000) * i128::from(channels.max(1)); + let buffered_ns = o.buffered_ahead as i128 * 1_000_000 / per_ms; + let audio = o.now_local_ns + buffered_ns + i128::from(o.clock_offset_ns) - i128::from(o.pts_ns); + i64::try_from((audio - video) / 1_000_000).ok() +} + +fn log_window(w: &Window, took: Duration, target_ms: u32, offset_ms: i32, queued_frames: usize) { + let av = match (w.av_min_ms, w.av_max_ms) { + (Some(min), Some(max)) => format!( + "a/v {offset_ms:+} ms smoothed, {:+} mean, {min:+} to {max:+} over {} packets", + w.av_sum_ms / i64::from(w.av_n.max(1)), + w.av_n + ), + _ => "a/v not measured, no picture on the glass".to_string(), + }; + tracing::info!( + "audio[{:.0}s]: {} grains, {} priming; underruns {} silent, {} audible; {} sheds, {} inserts; depth {}-{} ms, target {target_ms} ms, device queue {} ms; {} packets, {} pauses over {} ms, longest gap {} ms; {av}", + took.as_secs_f64(), + w.grains, + w.priming, + w.silent_underruns, + w.audible_underruns, + w.sheds, + w.inserts, + w.depth_min_ms.unwrap_or(0), + w.depth_max_ms, + queued_frames / (crate::session::audio::SAMPLE_RATE / 1000) as usize, + w.packets, + w.pauses, + PAUSE.as_millis(), + w.longest_gap.as_millis(), + ); +} + /// One policy step and one read into `grain`. fn fill(ring: &mut Ring, grain: &mut [f32], underruns: &AtomicU64) { let want = grain.len(); let step = ring.policy.step(ring.samples.len(), want); + ring.window.grains += 1; + ring.window.sheds += u32::from(step.drop_front > 0); + ring.window.inserts += u32::from(step.insert_front > 0); if step.drop_front > 0 { crossfade_drop(&mut ring.samples, step.drop_front, step.crossfade); } @@ -172,6 +333,7 @@ fn fill(ring: &mut Ring, grain: &mut [f32], underruns: &AtomicU64) { } if step.silence { grain.fill(0.0); + ring.window.priming += 1; ring.policy.note_read(false); } else { let take = want.min(ring.samples.len()); @@ -182,10 +344,18 @@ fn fill(ring: &mut Ring, grain: &mut [f32], underruns: &AtomicU64) { let short = take < want; if short { underruns.fetch_add(1, Ordering::Relaxed); + if ring.last_peak < SILENT_PEAK { + ring.window.silent_underruns += 1; + } else { + ring.window.audible_underruns += 1; + } } ring.policy.note_read(short); } ring.depth_ms = ring.policy.depth_ms(ring.samples.len()); + let (depth, w) = (ring.depth_ms, &mut ring.window); + w.depth_min_ms = Some(w.depth_min_ms.map_or(depth, |m| m.min(depth))); + w.depth_max_ms = w.depth_max_ms.max(depth); } #[cfg(test)] @@ -198,7 +368,7 @@ mod tests { #[test] fn fed_audio_reaches_the_port_in_grains() { let audio = FakeAudio::default(); - let sink = AudioOutSink::start(audio.open(2, 48_000).unwrap(), 2).unwrap(); + let sink = AudioOutSink::start(audio.open(2, 48_000).unwrap(), 2, SyncCells::default()).unwrap(); let tone = vec![0.5f32; 240 * 2]; for _ in 0..40 { sink.feed(Samples::F32(&tone), 0).unwrap(); diff --git a/crates/pf-nx-app/src/session/pipeline.rs b/crates/pf-nx-app/src/session/pipeline.rs index 32e0c94..4045cef 100644 --- a/crates/pf-nx-app/src/session/pipeline.rs +++ b/crates/pf-nx-app/src/session/pipeline.rs @@ -14,7 +14,7 @@ use crate::core::media::{AudioSink, VideoSink}; use crate::platform::{AudioOut, StreamSpec, VideoDecoders}; use crate::services::join::{join_with_timeout, SHUTDOWN_JOIN_TIMEOUT}; use crate::session::audio::AudioStage; -use crate::session::audio_out::AudioOutSink; +use crate::session::audio_out::{AudioOutSink, SyncCells}; use crate::session::connect::ConnectParams; use crate::session::pump::{spawn_audio_feed, video_pump}; use crate::session::stage::{SinkConfig, VideoStage}; @@ -54,7 +54,8 @@ impl MediaPipeline { .audio .open(client.audio_channels, crate::session::audio::SAMPLE_RATE) .context("open the audio port")?; - let audio_sink: Arc = Arc::new(AudioOutSink::start(port, client.audio_channels)?); + let audio_sink: Arc = + Arc::new(AudioOutSink::start(port, client.audio_channels, SyncCells::of(client))?); let stage = AudioStage::new(audio_sink, client.audio_channels, client.audio_layout)?; let video_thread = spawn_video_thread(client, sink, first, opener, stop, stats, params.present_priority)?; let audio_thread = match spawn_audio_feed(client.clone(), stage, stop.clone()) { @@ -143,6 +144,7 @@ fn spec_of(client: &NativeClient, smoothing: usize) -> StreamSpec { shard_payload: client.shard_payload as usize, smoothing, hud: client.hud_shared(), + video_e2e: crate::platform::VideoE2e::new(client.video_e2e_shared(), client.clock_offset_shared()), } } diff --git a/crates/pf-nx-app/src/session/stage/mod.rs b/crates/pf-nx-app/src/session/stage/mod.rs index 506c177..feca0db 100644 --- a/crates/pf-nx-app/src/session/stage/mod.rs +++ b/crates/pf-nx-app/src/session/stage/mod.rs @@ -872,7 +872,22 @@ mod tests { }; let (bare, paced) = (cushion(bare), cushion(paced)); assert!(paced > bare, "the clocked stage reserved the budget: {paced} vs {bare}"); - assert!(paced - bare >= 2 * 16_666_666, "two stream periods of extra lead"); + assert!( + paced - bare >= 16_666_666, + "the second frame is a stream period of extra lead" + ); + } + + /// A one-frame budget is the adaptive cushion alone: no whole period on top of it. + #[test] + fn a_one_frame_budget_adds_no_period() { + let cushion = |budget| { + let mut h = harness_with(FakeSink::default(), budget); + h.stage.submit(&frame(1, None, true, false)); + h.stage.pacing_health().cushion_ns + }; + let latency = cushion(PresentPriority::Latency); + assert_eq!(cushion(PresentPriority::Smooth { buffer: 1 }), latency); } /// An intra-only sink never holds for a re-anchor: the frame after a loss decodes alone. diff --git a/crates/pf-nx-app/src/session/timeline.rs b/crates/pf-nx-app/src/session/timeline.rs index 11d1be5..ed1fef5 100644 --- a/crates/pf-nx-app/src/session/timeline.rs +++ b/crates/pf-nx-app/src/session/timeline.rs @@ -3,14 +3,14 @@ use pf_client_core::trust::PresentPriority; /// Correct clock offset and drift without replacing source cadence with arrival cadence. -/// Smoothness reserves the selected number of stream periods on TOP of core's adaptive -/// cushion; the default keeps the adaptive cushion alone. Neither mode delays compressed -/// input in a Rust queue. +/// Smoothness counts core's adaptive cushion as its first frame and reserves each further one +/// as a whole stream period on top. Neither mode delays compressed input in a Rust queue. pub struct Pacing { clock: punktfunk_core::phase::CadenceClock, /// Source period, and the ceiling core clamps its own adaptive cushion to. source_interval_ns: i64, - /// Extra lead Smoothness adds over the adaptive cushion; `0` under the latency intent. + /// Extra lead Smoothness adds over the adaptive cushion; `0` under the latency intent and + /// for a one-frame budget. smooth_cushion_ns: i64, /// Stamps never go backwards, across recovery too. last_base_ns: u64, @@ -32,13 +32,15 @@ pub struct Pacing { impl Pacing { /// Use source period, not panel period. Priority includes Automatic from settings. /// - /// ⚠ This client reads frame count as EXTRA (core clamps adaptive to 1 period, making - /// one-frame budget unreachable); the honest fix needs additive semantics in core. + /// Core clamps its adaptive cushion to one period, so that cushion is the budget's first + /// frame. Reading the count as extra periods on top held every picture a whole refresh past + /// what the jitter asked for: handheld at one frame the display stage ran 21-30 ms against + /// 13 ms under the latency intent (log of 2026-11-02). pub fn new(source_interval_ns: u64, priority: PresentPriority) -> Self { let interval = i64::try_from(source_interval_ns).unwrap_or(i64::MAX).max(1); // `fifo_capacity` is core's own reading of the enum: 0 means the latency intent, and // `PresentPriority::resolve` has already clamped the buffer to 1–3. - let frames = priority.fifo_capacity(); + let frames = priority.fifo_capacity().saturating_sub(1); let smooth_cushion_ns = interval.saturating_mul(i64::from(frames)); Self { // Default tuning assumes about half a refresh of latch slack; the first pacing record diff --git a/crates/pf-nx-app/src/ui/service.rs b/crates/pf-nx-app/src/ui/service.rs index ed0f095..f5453e4 100644 --- a/crates/pf-nx-app/src/ui/service.rs +++ b/crates/pf-nx-app/src/ui/service.rs @@ -161,13 +161,13 @@ impl Service { /// The background's answers, on the service cadence: adverts, sweeps, games, art, pairing. fn service(&mut self) { - if let Some(discovery) = &mut self.discovery { - for host in discovery.poll() { - self.rows_dirty = true; - self.discovered - .retain(|d| !(d.addr == host.addr && d.port == host.port)); - self.discovered.push(host); - } + let arrived = self.discovery.as_mut().map(Discovery::poll).unwrap_or_default(); + for host in arrived { + self.rows_dirty = true; + self.discovered + .retain(|d| !(d.addr == host.addr && d.port == host.port)); + self.follow_moved_host(&host); + self.discovered.push(host); } self.drain_sweep(); self.drain_games(); @@ -775,6 +775,42 @@ impl Service { } } + /// Moves a saved host to the address its advert now comes from, by its pinned fingerprint. + fn follow_moved_host(&mut self, advert: &DiscoveredHost) { + let discovered = &self.discovered; + let advertised = |addr: &str, port: u16| discovered.iter().any(|d| (d.addr.as_str(), d.port) == (addr, port)); + let moved = self.store.edit(|state| { + let moved = store::follow_moved_host( + &mut state.known_hosts, + &advert.fp_hex, + &advert.addr, + advert.port, + advertised, + ); + // A selection on the old address names a record that is no longer there. + if moved + && state + .selected_host + .as_ref() + .is_some_and(|(h, p)| !state.known_hosts.iter().any(|k| k.addr == *h && k.port == *p)) + { + state.selected_host = None; + } + moved + }); + if moved { + tracing::info!( + "console: {} now answers at {}:{}", + advert.name, + advert.addr, + advert.port + ); + self.last_sweep = None; + self.store + .with(|s| crate::services::art::reconcile_host_caches(&s.known_hosts)); + } + } + fn forget_host(&mut self, key: &str) { let mut gone: Option = None; self.store.edit(|state| { diff --git a/crates/pf-nx-platform/src/console/audio.rs b/crates/pf-nx-platform/src/console/audio.rs index 08b1bee..edaa1f4 100644 --- a/crates/pf-nx-platform/src/console/audio.rs +++ b/crates/pf-nx-platform/src/console/audio.rs @@ -51,4 +51,8 @@ impl AudioPort for Port { } self.out.push(&self.pcm) } + + fn queued_frames(&self) -> usize { + self.out.queued_frames() + } } diff --git a/crates/pf-nx-platform/src/console/lifecycle.rs b/crates/pf-nx-platform/src/console/lifecycle.rs index 4f4e0d1..cad21be 100644 --- a/crates/pf-nx-platform/src/console/lifecycle.rs +++ b/crates/pf-nx-platform/src/console/lifecycle.rs @@ -7,9 +7,9 @@ use std::sync::mpsc; use std::time::{Duration, Instant}; -use pf_nx_app::platform::{Lifecycle, SystemEvent}; +use pf_nx_app::platform::{Clocks, Lifecycle, SystemEvent}; use pf_nx_sys::applet::{self, FocusState, HookEvent}; -use pf_nx_sys::{mem, rt, socket}; +use pf_nx_sys::{clk, mem, rt, socket, swkbd}; pub struct NxLifecycle { events: mpsc::Receiver, @@ -46,7 +46,13 @@ impl Lifecycle for NxLifecycle { self.exit(); } if !std::mem::replace(&mut self.reported, true) { - tracing::info!("lifecycle: {:?}, {:?}", applet::applet_type(), applet::operation_mode()); + tracing::info!( + "lifecycle: {:?}, {:?}, clocks {}", + applet::applet_type(), + applet::operation_mode(), + self.clocks() + .map_or_else(|| "unreadable".to_string(), |c| c.to_string()) + ); } // `appletMainLoop` in its halves, so every message is logged: no hook event and no // operation-mode change has reached a log yet, the dock change included. @@ -68,7 +74,31 @@ impl Lifecycle for NxLifecycle { }), HookEvent::Resume => out.push(SystemEvent::Resume), HookEvent::OperationMode => out.push(SystemEvent::OperationMode(super::operation_mode())), - HookEvent::Exit | HookEvent::PerformanceMode => {} + // The system moved the title to another clock configuration (WP5's question). + HookEvent::PerformanceMode => tracing::info!( + "lifecycle: performance mode changed, clocks {}", + self.clocks() + .map_or_else(|| "unreadable".to_string(), |c| c.to_string()) + ), + HookEvent::Exit => {} + } + } + } + + fn clocks(&self) -> Option { + clk::clocks().map(|c| Clocks { + cpu_hz: c.cpu_hz, + gpu_hz: c.gpu_hz, + emc_hz: c.emc_hz, + }) + } + + fn keyboard(&mut self) -> Option { + match swkbd::ask("Text to type on the PC", "Send") { + Ok(text) => Some(text), + Err(rc) => { + tracing::info!("keyboard: closed without text ({})", pf_nx_sys::result_code(rc)); + None } } } diff --git a/crates/pf-nx-platform/src/console/mod.rs b/crates/pf-nx-platform/src/console/mod.rs index 9e64f75..066728a 100644 --- a/crates/pf-nx-platform/src/console/mod.rs +++ b/crates/pf-nx-platform/src/console/mod.rs @@ -18,6 +18,10 @@ use pf_nx_vk::{GpuDevice, Presenter, Target, VideoSlot}; /// Identity, settings, logs and bundles; the NRO itself lives here too (plan D18). const DATA_DIR: &str = "sdmc:/switch/punktfunk"; +/// A file of this name in [`DATA_DIR`], read at start, turns the straight VIC route on, so one +/// build A/Bs both routes. +const STRAIGHT_MARKER: &str = "vic-direto"; + /// The Mesa pin NVK was built from, for the log and the About screen. const MESA_PIN: &str = match option_env!("PF_MESA_PIN") { Some(p) => p, @@ -94,7 +98,9 @@ pub fn platform(kind: DisplayKind) -> Result { let ui = presenter.ui(); ( Box::new(display_nvk::NvkDisplay::new(presenter, mode)), - Arc::new(crate::video::NxVideo::gpu(dev, slot)), + Arc::new( + crate::video::NxVideo::gpu(dev, slot).straight(Path::new(DATA_DIR).join(STRAIGHT_MARKER).exists()), + ), gpu, ui, ) diff --git a/crates/pf-nx-platform/src/video.rs b/crates/pf-nx-platform/src/video.rs index 8588899..73748ec 100644 --- a/crates/pf-nx-platform/src/video.rs +++ b/crates/pf-nx-platform/src/video.rs @@ -4,7 +4,10 @@ //! //! `feed` decodes on the session's video thread: send the access unit, then receive every //! picture straight into the presenter's next staging buffer, which `commit` copies into a -//! plane set and publishes (D7). On the framebuffer display the pictures land in scratch planes +//! plane set and publishes (D7). On the console the picture goes through libavutil's own frame +//! and the shim copies it; with [`NxVideo::straight`] the VIC writes into that staging buffer +//! itself where it can map it (WP5). The `nvdec[10s]` line names which route ran. On the +//! framebuffer display the pictures land in scratch planes //! and are dropped, so the emulator still runs the hwaccel. A decoder error fails this feed, //! which has the session ask for a keyframe, and the next feed starts from a flushed decoder. @@ -32,15 +35,30 @@ const DECODER: &str = "libavcodec (software)"; pub struct NxVideo { /// The presenter's device and slot; `None` on the framebuffer display. gpu: Option<(Arc, Arc)>, + /// The VIC writes into the staging buffer itself. Off by default: on the console it neither + /// lowered the transfer plus copy nor the end-to-end time (WP5, 2026-11-01). + straight: bool, } impl NxVideo { pub fn gpu(dev: Arc, slot: Arc) -> Self { - Self { gpu: Some((dev, slot)) } + Self { + gpu: Some((dev, slot)), + straight: false, + } } pub fn cpu() -> Self { - Self { gpu: None } + Self { + gpu: None, + straight: false, + } + } + + /// Has the VIC write straight into the staging buffer where it can. + pub fn straight(mut self, on: bool) -> Self { + self.straight = on; + self } } @@ -61,12 +79,21 @@ impl VideoDecoders for NxVideo { spec.bit_depth, if spec.chroma444 { " 4:4:4" } else { "" } ); - let decoder = av::Decoder::open(codec, spec.width, spec.height)?; + let decoder = av::Decoder::open(codec, spec.width, spec.height, self.straight)?; let out = match &self.gpu { Some((dev, slot)) => Out::Gpu(Nv12Upload::new(dev.clone(), slot.clone(), spec)?), None => Out::scratch(spec), }; - tracing::info!("{DECODER}: {codec:?} {}x{}", spec.width, spec.height); + tracing::info!( + "{DECODER}: {codec:?} {}x{}, {}", + spec.width, + spec.height, + if self.straight { + "straight VIC route asked for" + } else { + "VIC then copy" + } + ); let started = Instant::now(); Ok(Box::new(NxSink { lane: Mutex::new(Lane { @@ -120,6 +147,14 @@ impl Out { } } + /// The GPU time of the last staging copy into a plane set, where the GPU can time it. + fn copy_gpu_us(&self) -> Option { + match self { + Self::Gpu(upload) => upload.copy_gpu_us(), + Self::Scratch { .. } => None, + } + } + /// `false` when the picture was dropped because every plane set is on screen. fn commit(&mut self, meta: av::PictureMeta, due: Option) -> Result { match self { @@ -180,9 +215,12 @@ impl Lane { .receive_into(planes.y, planes.y_stride, planes.uv, planes.uv_stride)?; let Some(meta) = got else { return Ok(false) }; let due = self.due_of(meta.pts); + self.timing.note_due(due); let started = Instant::now(); let shown = self.out.commit(meta, due)?; - self.timing.note(&self.decoder.stats(), wait, started.elapsed(), shown); + let upload = started.elapsed(); + self.timing + .note(&self.decoder.stats(), wait, upload, self.out.copy_gpu_us(), shown); Ok(true) } @@ -199,7 +237,9 @@ impl Lane { } /// Where the lane's time went over one [`TIMING_WINDOW`]: the wait for a free staging buffer, -/// the decoder's own time, the VIC transfer into the staging planes, and the upload's submit. +/// the decoder's own time, the transfer to NV12, the shim's copy out of libavutil's frame (none +/// when the transfer wrote the staging planes itself), the upload's submit, and the staging +/// copy's time on the GPU. struct Timing { since: Instant, pictures: u32, @@ -208,10 +248,23 @@ struct Timing { decode_max_us: u32, transfer_us: u64, transfer_max_us: u32, + copy_us: u64, + copy_max_us: u32, wait: Duration, wait_max: Duration, upload: Duration, + gpu_copy_us: u64, + gpu_copies: u32, in_flight_max: u32, + /// How the last picture reached the staging planes; `None` before the first. + route: Option, + /// Pictures whose host PTS found no due time: they are shown at once, whatever the budget. + undue: u32, + /// The due time less the moment the picture came out of the decoder, µs, over the pictures + /// that had one. Negative is already late. + slack_us: i64, + slack_min_us: i64, + slack_max_us: i64, } impl Timing { @@ -224,14 +277,74 @@ impl Timing { decode_max_us: 0, transfer_us: 0, transfer_max_us: 0, + copy_us: 0, + copy_max_us: 0, wait: Duration::ZERO, wait_max: Duration::ZERO, upload: Duration::ZERO, + gpu_copy_us: 0, + gpu_copies: 0, in_flight_max: 0, + route: None, + undue: 0, + slack_us: 0, + slack_min_us: i64::MAX, + slack_max_us: i64::MIN, } } - fn note(&mut self, s: &av::Stats, wait: Duration, upload: Duration, shown: bool) { + /// Whether the picture out now kept its due time, and how far ahead of it. + fn note_due(&mut self, due: Option) { + let Some(due) = due else { + self.undue += 1; + return; + }; + let now = Instant::now(); + let us = |d: Duration| i64::try_from(d.as_micros()).unwrap_or(i64::MAX); + let slack = if due >= now { us(due - now) } else { -us(now - due) }; + self.slack_us += slack; + self.slack_min_us = self.slack_min_us.min(slack); + self.slack_max_us = self.slack_max_us.max(slack); + } + + /// `"due slack mean … min … max … ms, N without a due"` over this window's pictures. + fn due_summary(&self) -> String { + let timed = i64::from(self.pictures.saturating_sub(self.undue)); + let ms = |us: i64| us as f64 / 1000.0; + if timed == 0 { + return format!("no due times, {} without", self.undue); + } + format!( + "due slack mean {:.1}ms min {:.1}ms max {:.1}ms, {} without a due", + ms(self.slack_us / timed), + ms(self.slack_min_us), + ms(self.slack_max_us), + self.undue + ) + } + + fn note(&mut self, s: &av::Stats, wait: Duration, upload: Duration, gpu_copy_us: Option, shown: bool) { + // The first route, and any change, at once: a change mid-stream is the straight route + // falling back. + if self.route != Some(s.route) { + tracing::info!( + "nvdec: pictures reach the staging planes by {}{}{}", + s.route.name(), + self.route.map_or_else(String::new, |r| format!(" (was {})", r.name())), + if s.direct_err == 0 { + String::new() + } else { + format!("; the straight route failed with {}", s.direct_err) + } + ); + } + self.route = Some(s.route); + self.copy_us += u64::from(s.last_copy_us); + self.copy_max_us = self.copy_max_us.max(s.last_copy_us); + if let Some(us) = gpu_copy_us { + self.gpu_copy_us += u64::from(us); + self.gpu_copies += 1; + } self.pictures += 1; self.dropped += u32::from(!shown); self.decode_us += u64::from(s.last_decode_us); @@ -248,8 +361,12 @@ impl Timing { let n = u64::from(self.pictures); let ms = |us: u64| us as f64 / 1000.0; let per = |d: Duration| d.as_secs_f64() * 1000.0 / n as f64; + let gpu_copy = self + .gpu_copy_us + .checked_div(u64::from(self.gpu_copies)) + .map_or_else(|| "-".to_string(), |us| format!("{:.2}", us as f64 / 1000.0)); tracing::info!( - "nvdec[{}s]: {} pictures ({} dropped, {} decoded in all); staging wait mean {:.1}ms max {:.1}ms; decode mean {:.1}ms max {:.1}ms; transfer mean {:.1}ms max {:.1}ms; upload mean {:.2}ms; in flight max {}", + "nvdec[{}s]: {} pictures ({} dropped, {} decoded in all); staging wait mean {:.1}ms max {:.1}ms; decode mean {:.1}ms max {:.1}ms; transfer mean {:.1}ms max {:.1}ms; copy mean {:.1}ms max {:.1}ms; upload mean {:.2}ms; gpu copy mean {gpu_copy}ms; in flight max {}; route {}; {}", TIMING_WINDOW.as_secs(), self.pictures, self.dropped, @@ -260,10 +377,16 @@ impl Timing { ms(u64::from(self.decode_max_us)), ms(self.transfer_us / n), ms(u64::from(self.transfer_max_us)), + ms(self.copy_us / n), + ms(u64::from(self.copy_max_us)), per(self.upload), self.in_flight_max, + self.route.map_or("-", av::Route::name), + self.due_summary(), ); + let route = self.route; *self = Self::new(); + self.route = route; } } diff --git a/crates/pf-nx-probe/src/nvdec.rs b/crates/pf-nx-probe/src/nvdec.rs index 667eef3..85d9734 100644 --- a/crates/pf-nx-probe/src/nvdec.rs +++ b/crates/pf-nx-probe/src/nvdec.rs @@ -90,7 +90,7 @@ fn decode(step: &str, codec: Codec, data: &[u8], out: &mut Out) -> Result @@ -14,9 +16,23 @@ #define PF_AV_NOT_NV12 (-0x70660001) #define PF_AV_TOO_SMALL (-0x70660002) +// How the last picture reached the caller's planes (src/av.rs `Route`). +enum { + PF_AV_ROUTE_SOFTWARE = 0, + // The hwcontext's transfer into libavutil's frame, then the shim's copy. + PF_AV_ROUTE_VIC_THEN_COPY = 1, + PF_AV_ROUTE_CPU_THEN_COPY = 2, + // The hwcontext's transfer straight into the caller's planes. + PF_AV_ROUTE_VIC = 3, + PF_AV_ROUTE_CPU = 4, +}; + typedef struct PfAvStats { uint64_t decoded; - uint32_t in_flight, last_decode_us, last_transfer_us, reserved; + uint32_t in_flight, last_decode_us, last_transfer_us, last_copy_us; + uint32_t route; + // Why transfers stopped going straight into the caller's planes; 0 while they do. + int32_t direct_err; } PfAvStats; typedef struct PfAvPictureMeta { @@ -30,6 +46,8 @@ typedef struct PfAvDecoder { AVPacket *pkt; AVFrame *hw; AVFrame *sw; + // The caller's planes as the transfer's destination; NULL off the console. + AVFrame *direct; int64_t sent_at; uint64_t taken; PfAvStats stats; @@ -42,12 +60,15 @@ void pf_av_close(PfAvDecoder *d) { av_packet_free(&d->pkt); av_frame_free(&d->hw); av_frame_free(&d->sw); + av_frame_free(&d->direct); av_free(d); } // codec: 0 H.264, 1 HEVC. The hwaccel is found by name, so no header needs its enum. +// `direct` non-zero tries the transfer straight into the caller's planes first; zero always +// goes through libavutil's frame and a copy. // TODO(verify): the device type's name is "nvtegra" in the pinned tree (WP0.6). -int pf_av_open(int codec, int width, int height, PfAvDecoder **out) { +int pf_av_open(int codec, int width, int height, int direct, PfAvDecoder **out) { *out = NULL; const AVCodec *c = avcodec_find_decoder(codec == 1 ? AV_CODEC_ID_HEVC : AV_CODEC_ID_H264); if (!c) return AVERROR_DECODER_NOT_FOUND; @@ -70,8 +91,10 @@ int pf_av_open(int codec, int width, int height, PfAvDecoder **out) { if (!d->ctx || !d->pkt || !d->hw || !d->sw) goto fail; #ifdef __SWITCH__ - if (!(d->ctx->hw_device_ctx = av_buffer_ref(d->device))) + if (!(d->ctx->hw_device_ctx = av_buffer_ref(d->device)) || (direct && !(d->direct = av_frame_alloc()))) goto fail; +#else + (void)direct; #endif // The default get_format picks the device's hw format when hw_device_ctx is set. d->ctx->width = width; @@ -103,6 +126,46 @@ int pf_av_send(PfAvDecoder *d, const uint8_t *data, int len, uint64_t pts) { return 0; } +// The caller's memory as a buffer the frame refers to; the caller owns it. +static void keep(void *opaque, uint8_t *data) { + (void)opaque; + (void)data; +} + +// The hwcontext maps the destination's buffers into the VIC per transfer and writes the +// picture there: the copy out of libavutil's frame goes away. It needs 256-byte aligned +// addresses and pitches, else it detiles on the CPU, still into these planes. `rows` is the +// picture's height rounded up to even, which is what either path writes. +static int transfer_into(PfAvDecoder *d, uint8_t *y, size_t y_stride, size_t y_len, uint8_t *uv, + size_t uv_stride, size_t uv_len) { + AVFrame *f = d->direct; + f->format = AV_PIX_FMT_NV12; + f->width = d->hw->width; + f->height = d->hw->height; + f->data[0] = y; + f->data[1] = uv; + f->linesize[0] = (int)y_stride; + f->linesize[1] = (int)uv_stride; + // One mapping when the planes are one block, as the staging buffer lays them out. + int err = AVERROR(ENOMEM); + if (uv == y + y_len) { + f->buf[0] = av_buffer_create(y, y_len + uv_len, keep, NULL, 0); + if (!f->buf[0]) goto out; + } else { + f->buf[0] = av_buffer_create(y, y_len, keep, NULL, 0); + f->buf[1] = av_buffer_create(uv, uv_len, keep, NULL, 0); + if (!f->buf[0] || !f->buf[1]) goto out; + } + err = av_hwframe_transfer_data(f, d->hw, 0); +out: + av_frame_unref(f); + return err; +} + +static int aligned_256(const void *p, size_t stride) { + return (((uintptr_t)p | stride) & 0xff) == 0; +} + static void copy_plane(uint8_t *dst, size_t dst_stride, const uint8_t *src, int src_stride, size_t row, size_t rows) { for (size_t i = 0; i < rows; i++) @@ -136,13 +199,36 @@ int pf_av_receive(PfAvDecoder *d, uint8_t *y, size_t y_stride, size_t y_len, uin meta->height = (uint32_t)d->hw->height; meta->pts = (uint64_t)d->hw->pts; AVFrame *src = d->hw; + d->stats.last_copy_us = 0; + d->stats.route = PF_AV_ROUTE_SOFTWARE; if (d->hw->hw_frames_ctx) { + size_t rows = ((size_t)d->hw->height + 1) & ~(size_t)1; + size_t w = ((size_t)d->hw->width + 1) & ~(size_t)1; + if (d->direct && d->stats.direct_err == 0 && y_stride >= w && uv_stride >= w && + y_len >= y_stride * rows && uv_len >= uv_stride * rows / 2) { + int64_t t = av_gettime_relative(); + err = transfer_into(d, y, y_stride, y_len, uv, uv_stride, uv_len); + d->stats.last_transfer_us = (uint32_t)(av_gettime_relative() - t); + if (err >= 0) { + d->stats.route = aligned_256(y, y_stride) && aligned_256(uv, uv_stride) + ? PF_AV_ROUTE_VIC + : PF_AV_ROUTE_CPU; + err = 1; + goto out; + } + // Never again for this decoder: the route through libavutil's frame is proven. + d->stats.direct_err = err < 0 ? err : -1; + } int64_t t = av_gettime_relative(); d->sw->format = AV_PIX_FMT_NV12; // TODO(verify): the VIC transfer offers NV12 (WP0.6). err = av_hwframe_transfer_data(d->sw, d->hw, 0); d->stats.last_transfer_us = (uint32_t)(av_gettime_relative() - t); if (err < 0) goto out; src = d->sw; + d->stats.route = aligned_256(src->data[0], (size_t)src->linesize[0]) && + aligned_256(src->data[1], (size_t)src->linesize[1]) + ? PF_AV_ROUTE_VIC_THEN_COPY + : PF_AV_ROUTE_CPU_THEN_COPY; } // On the console a software frame means the hwaccel did not take the stream. #ifdef __SWITCH__ @@ -165,6 +251,7 @@ int pf_av_receive(PfAvDecoder *d, uint8_t *y, size_t y_stride, size_t y_len, uin d->stats.last_transfer_us = (uint32_t)(av_gettime_relative() - t); } else { copy_plane(uv, uv_stride, src->data[1], src->linesize[1], uw, uh); + d->stats.last_copy_us = (uint32_t)(av_gettime_relative() - t); } err = 1; out: diff --git a/crates/pf-nx-sys/csrc/pf_swkbd.c b/crates/pf-nx-sys/csrc/pf_swkbd.c new file mode 100644 index 0000000..ab0ef59 --- /dev/null +++ b/crates/pf-nx-sys/csrc/pf_swkbd.c @@ -0,0 +1,21 @@ +// The system keyboard applet as one call the Rust side binds (src/swkbd.rs). libnx's +// SwkbdConfig is a versioned argument block it sizes per firmware, so it stays on this side. +#include + +// Shows the keyboard over the title and blocks until the player closes it. 0 with `out` +// holding the committed UTF-8 text, NUL-terminated; libnx's Result otherwise, cancel included. +uint32_t pf_swkbd_ask(const char *guide, const char *ok, uint32_t max_chars, char *out, size_t out_len) { + SwkbdConfig kbd; + out[0] = '\0'; + Result rc = swkbdCreate(&kbd, 0); + if (R_FAILED(rc)) { + return rc; + } + swkbdConfigMakePresetDefault(&kbd); + swkbdConfigSetGuideText(&kbd, guide); + swkbdConfigSetOkButtonText(&kbd, ok); + swkbdConfigSetStringLenMax(&kbd, max_chars); + rc = swkbdShow(&kbd, out, out_len); + swkbdClose(&kbd); + return rc; +} diff --git a/crates/pf-nx-sys/src/av.rs b/crates/pf-nx-sys/src/av.rs index bcf5941..5e9ac96 100644 --- a/crates/pf-nx-sys/src/av.rs +++ b/crates/pf-nx-sys/src/av.rs @@ -36,8 +36,54 @@ pub struct Stats { pub in_flight: u32, /// Microseconds from the last send to the last picture out. pub last_decode_us: u32, - /// Microseconds of the last VIC transfer to NV12. + /// Microseconds of the last transfer to NV12 (the VIC's, or the CPU's detiling). pub last_transfer_us: u32, + /// Microseconds of the last copy out of libavutil's frame; 0 when there was none. + pub last_copy_us: u32, + pub route: Route, + /// Why transfers stopped going straight into the caller's planes, an `AVERROR`; 0 while + /// they do. + pub direct_err: i32, +} + +/// How the last picture reached the caller's planes (`PF_AV_ROUTE_*`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Route { + /// A software decoder's picture, copied (the dev box). + #[default] + Software, + /// The VIC into libavutil's frame, then copied. + VicThenCopy, + /// libavutil's frame was not 256-byte aligned, so the hwcontext detiled on the CPU, then + /// copied. + CpuThenCopy, + /// The VIC straight into the caller's planes. + Vic, + /// Detiled on the CPU straight into the caller's planes. + Cpu, +} + +impl Route { + fn from_raw(raw: u32) -> Self { + match raw { + 1 => Self::VicThenCopy, + 2 => Self::CpuThenCopy, + 3 => Self::Vic, + 4 => Self::Cpu, + _ => Self::Software, + } + } + + /// For the log. + pub fn name(self) -> &'static str { + match self { + Self::Software => "software, copied", + Self::VicThenCopy => "VIC, copied", + Self::CpuThenCopy => "CPU detile, copied", + Self::Vic => "VIC straight", + Self::Cpu => "CPU detile straight", + } + } } /// `PfAvStats`. @@ -48,7 +94,9 @@ pub struct RawStats { pub in_flight: u32, pub last_decode_us: u32, pub last_transfer_us: u32, - pub reserved: u32, + pub last_copy_us: u32, + pub route: u32, + pub direct_err: i32, } /// The shim's own codes, outside libavcodec's `AVERROR` space (`csrc/pf_av.c`). @@ -67,7 +115,13 @@ mod ffi { } extern "C" { - pub fn pf_av_open(codec: c_int, width: c_int, height: c_int, out: *mut *mut PfAvDecoder) -> c_int; + pub fn pf_av_open( + codec: c_int, + width: c_int, + height: c_int, + direct: c_int, + out: *mut *mut PfAvDecoder, + ) -> c_int; pub fn pf_av_send(d: *mut PfAvDecoder, data: *const u8, len: c_int, pts: u64) -> c_int; pub fn pf_av_receive( d: *mut PfAvDecoder, @@ -116,12 +170,14 @@ unsafe impl Send for Decoder {} #[cfg(pf_av)] impl Decoder { - /// Opens `codec` for a `width`×`height` stream. - pub fn open(codec: Codec, width: u32, height: u32) -> anyhow::Result { + /// Opens `codec` for a `width`×`height` stream. `straight` has the VIC write into the + /// caller's planes ([`Route::Vic`]) where it can; otherwise every picture goes through + /// libavutil's frame and a copy ([`Route::VicThenCopy`]). + pub fn open(codec: Codec, width: u32, height: u32, straight: bool) -> anyhow::Result { let (w, h) = (i32::try_from(width)?, i32::try_from(height)?); let mut raw = core::ptr::null_mut(); // SAFETY: `raw` is a valid out-pointer; the shim sets it only on success. - let rc = unsafe { ffi::pf_av_open(codec as i32, w, h, &mut raw) }; + let rc = unsafe { ffi::pf_av_open(codec as i32, w, h, i32::from(straight), &mut raw) }; if rc < 0 { return Err(av_error(&format!("open the {codec:?} decoder"), rc)); } @@ -196,6 +252,9 @@ impl Decoder { in_flight: s.in_flight, last_decode_us: s.last_decode_us, last_transfer_us: s.last_transfer_us, + last_copy_us: s.last_copy_us, + route: Route::from_raw(s.route), + direct_err: s.direct_err, } } } @@ -210,7 +269,7 @@ impl Drop for Decoder { #[cfg(not(pf_av))] impl Decoder { - pub fn open(codec: Codec, _width: u32, _height: u32) -> anyhow::Result { + pub fn open(codec: Codec, _width: u32, _height: u32, _straight: bool) -> anyhow::Result { anyhow::bail!("open the {codec:?} decoder: no decoder on this box (NVDEC on the console, libavcodec on Linux)") } @@ -244,7 +303,7 @@ mod tests { #[cfg(not(pf_av))] #[test] fn open_fails_off_the_console() { - let err = Decoder::open(Codec::Hevc, 1920, 1080).err().unwrap(); + let err = Decoder::open(Codec::Hevc, 1920, 1080, false).err().unwrap(); assert_eq!( err.to_string(), "open the Hevc decoder: no decoder on this box (NVDEC on the console, libavcodec on Linux)" @@ -255,7 +314,7 @@ mod tests { #[test] fn a_dev_box_opens_both_codecs_in_software() { for codec in [Codec::H264, Codec::Hevc] { - let mut d = Decoder::open(codec, 1920, 1080).unwrap(); + let mut d = Decoder::open(codec, 1920, 1080, false).unwrap(); assert_eq!(d.stats(), Stats::default()); d.flush(); } @@ -266,10 +325,14 @@ mod tests { use core::mem::{offset_of, size_of}; // PfAvPictureMeta: u32 width, u32 height, u64 pts → 16. assert_eq!((size_of::(), offset_of!(PictureMeta, pts)), (16, 8)); - // PfAvStats: u64 decoded, four u32 → 24. + // PfAvStats: u64 decoded, five u32 and an i32 → 32. assert_eq!( - (size_of::(), offset_of!(RawStats, last_transfer_us)), - (24, 16) + ( + size_of::(), + offset_of!(RawStats, last_transfer_us), + offset_of!(RawStats, direct_err) + ), + (32, 16, 28) ); assert_eq!(Codec::Hevc as i32, 1); } diff --git a/crates/pf-nx-sys/src/lib.rs b/crates/pf-nx-sys/src/lib.rs index 262ba74..78da52b 100644 --- a/crates/pf-nx-sys/src/lib.rs +++ b/crates/pf-nx-sys/src/lib.rs @@ -29,6 +29,7 @@ pub mod random; pub mod rt; pub mod setsys; pub mod socket; +pub mod swkbd; pub mod thread; pub mod vi; diff --git a/crates/pf-nx-sys/src/swkbd.rs b/crates/pf-nx-sys/src/swkbd.rs new file mode 100644 index 0000000..487e709 --- /dev/null +++ b/crates/pf-nx-sys/src/swkbd.rs @@ -0,0 +1,46 @@ +//! The system keyboard: libnx's `swkbd` library applet, through `csrc/pf_swkbd.c` (libnx 4.12.0 +//! `applets/swkbd.h`), which keeps the applet's versioned config block on the C side. + +/// The longest text one [`ask`] takes, in characters. +pub const MAX_CHARS: u32 = 256; + +#[cfg(target_os = "horizon")] +mod ffi { + use core::ffi::c_char; + + extern "C" { + pub fn pf_swkbd_ask( + guide: *const c_char, + ok: *const c_char, + max_chars: u32, + out: *mut c_char, + out_len: usize, + ) -> u32; + } +} + +/// Shows the keyboard with `guide` in its empty field and `ok` on its confirm button, and blocks +/// until the player closes it. The committed text, or the applet's `Result` (a cancel included). +#[cfg(target_os = "horizon")] +pub fn ask(guide: &str, ok: &str) -> Result { + let guide = std::ffi::CString::new(guide).unwrap_or_default(); + let ok = std::ffi::CString::new(ok).unwrap_or_default(); + // UTF-8 takes up to four bytes a character, and the shim NUL-terminates. + let mut out = vec![0u8; MAX_CHARS as usize * 4 + 1]; + // SAFETY: both strings are NUL-terminated and outlive the call; `out` is writable for its + // length, which the applet's reply is bounded by. + let rc = unsafe { + ffi::pf_swkbd_ask( + guide.as_ptr(), + ok.as_ptr(), + MAX_CHARS, + out.as_mut_ptr().cast(), + out.len(), + ) + }; + if rc != 0 { + return Err(rc); + } + let end = out.iter().position(|&b| b == 0).unwrap_or(out.len()); + Ok(String::from_utf8_lossy(&out[..end]).into_owned()) +} diff --git a/crates/pf-nx-sys/tests/layout.rs b/crates/pf-nx-sys/tests/layout.rs index 2dd9d12..703e94f 100644 --- a/crates/pf-nx-sys/tests/layout.rs +++ b/crates/pf-nx-sys/tests/layout.rs @@ -117,8 +117,8 @@ fn firmware_version() { #[test] fn shim_structs() { - // PfAvPictureMeta: u32, u32, u64 → 16. PfAvStats: u64, 4 × u32 → 24. + // PfAvPictureMeta: u32, u32, u64 → 16. PfAvStats: u64, 6 × 32-bit → 32. assert_eq!(layout::(), (16, 8)); - assert_eq!(layout::(), (24, 8)); + assert_eq!(layout::(), (32, 8)); assert_eq!(offset_of!(RawStats, in_flight), 8); } diff --git a/crates/pf-nx-vk/src/lib.rs b/crates/pf-nx-vk/src/lib.rs index b77a6e7..59a77eb 100644 --- a/crates/pf-nx-vk/src/lib.rs +++ b/crates/pf-nx-vk/src/lib.rs @@ -13,6 +13,7 @@ pub mod pipelines; pub mod presenter; #[cfg(feature = "pyrowave")] pub mod pyro; +mod timer; pub mod ui; pub mod util; diff --git a/crates/pf-nx-vk/src/picture.rs b/crates/pf-nx-vk/src/picture.rs index 4496b26..4935734 100644 --- a/crates/pf-nx-vk/src/picture.rs +++ b/crates/pf-nx-vk/src/picture.rs @@ -17,6 +17,7 @@ use pf_client_video::video_color::ColorDesc; use pf_nx_app::platform::StreamSpec; use crate::device::GpuDevice; +use crate::timer::GpuTimer; /// Plane views in the layout the lane leaves them (`layout`). #[derive(Clone, Copy, Debug)] @@ -50,6 +51,8 @@ pub struct Picture { pub turn: Option, /// The session's overlay figures, where the display stage is noted. pub hud: Option>, + /// Where the presenter leaves the capture-to-glass time for the audio's A/V sync. + pub e2e: Option, /// Keeps the planes' images and memory alive. pub hold: Box, } @@ -74,7 +77,14 @@ impl VideoSlot { /// The picture for the vblank at `latch`, if a new one is due by then. pub fn take(&self, latch: Instant) -> Option> { - self.lock().take(latch) + let mut q = self.lock(); + let picture = q.take(latch); + if q.smoothing > 0 { + if let Some(w) = q.window.close(latch) { + tracing::info!("{}", w.line()); + } + } + picture } /// Drops the waiting pictures: the stream ended. @@ -97,6 +107,11 @@ const MAX_SMOOTHING: usize = 3; /// Pictures a smoothing queue holds past its budget: the pacing's own cushion, up to a period, /// and one picture of slack before the oldest is dropped. const SMOOTHING_SLACK: usize = 2; +/// Takes in a row with a second due picture waiting before the slot skips one. A burst on the +/// wire leaves that picture standing for the rest of the session, a refresh of latency at every +/// budget (log of 2026-11-02: one left behind on 571 of 598 takes, no take ever held for a due). +/// Due times straddling a vblank leave one now and then, never half a second running. +const STANDING_TAKES: u32 = 30; /// The most pictures the slot ever holds. const MAX_QUEUED: usize = MAX_SMOOTHING + SMOOTHING_SLACK; @@ -110,12 +125,91 @@ pub struct Drops { pub overflow: u32, } +/// How often the slot reports how its pictures met their due times. +const WINDOW: std::time::Duration = std::time::Duration::from_secs(10); + +/// One window of the smoothing slot's takes: whether a due time ever held a picture back, how +/// late the ones taken were against it, and how many waited behind each. A standing picture +/// behind every take is a refresh of latency the budget did not ask for. +#[derive(Default)] +struct Window { + since: Option, + taken: u32, + /// Vblanks with a picture waiting that was not due yet. + held: u32, + /// Taken with no due time. + undue: u32, + /// Taken before their due time, to keep the queue inside the budget. + early: u32, + /// Standing backlogs skipped, one picture each. + caught_up: u32, + /// Latch less due over the pictures taken at or past it, µs. + late_us: u64, + late_max_us: u64, + late_n: u32, + /// Pictures left waiting after each take: none, one, two or more. + left: [u32; 3], +} + +impl Window { + fn note_take(&mut self, latch: Instant, due: Option, left: usize) { + self.taken += 1; + self.left[left.min(2)] += 1; + match due { + None => self.undue += 1, + Some(d) if d > latch => self.early += 1, + Some(d) => { + let us = u64::try_from((latch - d).as_micros()).unwrap_or(u64::MAX); + self.late_us += us; + self.late_max_us = self.late_max_us.max(us); + self.late_n += 1; + } + } + } + + /// The finished window at `now`, arming the next; `None` until one has run its length. + fn close(&mut self, now: Instant) -> Option { + let since = *self.since.get_or_insert(now); + (now.duration_since(since) >= WINDOW).then(|| { + std::mem::replace( + self, + Self { + since: Some(now), + ..Self::default() + }, + ) + }) + } + + fn line(&self) -> String { + let ms = |us: u64| us as f64 / 1000.0; + format!( + "slot[{}s]: {} taken, {} vblanks held for a due, {} without a due, {} early, {} caught up; late mean {:.1}ms max {:.1}ms; left behind 0:{} 1:{} 2+:{}", + WINDOW.as_secs(), + self.taken, + self.held, + self.undue, + self.early, + self.caught_up, + ms(self.late_us.checked_div(u64::from(self.late_n)).unwrap_or(0)), + ms(self.late_max_us), + self.left[0], + self.left[1], + self.left[2], + ) + } +} + /// The slot's rule, apart from the pictures, so the tests drive it with plain values. struct Queue { items: VecDeque<(Option, T)>, /// `0`: newest wins. smoothing: usize, drops: Drops, + /// Smoothing only: takes in a row that left a due picture waiting. + standing: u32, + /// Smoothing only: the takes since the last report. + window: Window, } impl Default for Queue { @@ -124,6 +218,8 @@ impl Default for Queue { items: VecDeque::with_capacity(MAX_QUEUED + 1), smoothing: 0, drops: Drops::default(), + standing: 0, + window: Window::default(), } } } @@ -146,6 +242,7 @@ impl Queue { /// Newest wins ignores `due`. Smoothing shows the oldest picture due by `latch` and lets one /// more due wait for the next vblank: due times that straddle a vblank then take one refresh /// each. A third due is a backlog, and its oldest are dropped so latency stays the budget. + /// A second due left waiting [`STANDING_TAKES`] takes running is one too, and is skipped. /// More waiting than the budget holds means the due times run ahead of the arrivals, and the /// oldest goes out early. fn take(&mut self, latch: Instant) -> Option { @@ -162,16 +259,26 @@ impl Queue { // still for seconds (smoothing log of 2026-10-31, 151 trimmed in one window). let due = due.max(self.items.len().saturating_sub(self.smoothing + 1)); if due == 0 { + self.window.held += u32::from(!self.items.is_empty()); return None; } - let behind = due.saturating_sub(2); + let mut behind = due.saturating_sub(2); + self.standing = if due >= 2 { self.standing + 1 } else { 0 }; + if self.standing >= STANDING_TAKES { + behind = due - 1; + self.standing = 0; + self.window.caught_up += 1; + } self.drops.replaced += behind as u32; self.items.drain(..behind); - self.items.pop_front().map(|(_, item)| item) + let (at, item) = self.items.pop_front()?; + self.window.note_take(latch, at, self.items.len()); + Some(item) } fn clear(&mut self) { self.items.clear(); + self.standing = 0; } fn trim(&mut self) { @@ -250,10 +357,15 @@ pub struct Nv12Upload { stride: usize, color: ColorDesc, hud: Arc, + e2e: pf_nx_app::platform::VideoE2e, planes: Vec>, pool: vk::CommandPool, staging: Vec, next: usize, + /// Each staging buffer's copy on the GPU; `None` where the queue has no timestamps. + timer: Option, + /// The last copy's GPU time that [`Self::stage`] read back. + copy_gpu_us: Option, } // SAFETY: the mapped pointers address memory this upload owns for its whole life and only @@ -290,10 +402,13 @@ impl Nv12Upload { full_range: spec.color.full_range != 0, }, hud: spec.hud.clone(), + e2e: spec.video_e2e.clone(), planes: Vec::new(), pool, staging: Vec::with_capacity(COPIES), next: 0, + timer: None, + copy_gpu_us: None, }; let dev = &upload.dev; // SAFETY: the pool is live; the buffers are freed with it. @@ -325,10 +440,12 @@ impl Nv12Upload { in_flight: false, }); } + upload.timer = GpuTimer::new(&upload.dev, COPIES); tracing::info!( - "upload: NV12 {}x{} through {COPIES} staging buffers of {bytes} bytes, row {stride}", + "upload: NV12 {}x{} through {COPIES} staging buffers of {bytes} bytes, row {stride}, at {:p}", extent.width, - extent.height + extent.height, + upload.staging[0].ptr, ); upload.slot.set_smoothing(spec.smoothing); if spec.smoothing == 0 { @@ -409,6 +526,9 @@ impl Nv12Upload { s.cmd, &vk::CommandBufferBeginInfo::default().flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), )?; + if let Some(t) = &mut self.timer { + t.begin(dev, s.cmd, i); + } for image in planes.images { // After the presenter's last read and any earlier copy into this set, which a // replaced picture can leave in flight. @@ -450,6 +570,9 @@ impl Nv12Upload { (vk::PipelineStageFlags2::FRAGMENT_SHADER, vk::AccessFlags2::SHADER_READ), ); } + if let Some(t) = &self.timer { + t.end(dev, s.cmd, i); + } dev.device.end_command_buffer(s.cmd)?; let cmds = [s.cmd]; dev.vkd.queue_lock.yield_to_present(); @@ -476,6 +599,7 @@ impl Nv12Upload { decoded_ns: u64::try_from(punktfunk_core::client::now_realtime_ns()).unwrap_or(0), turn: None, hud: Some(self.hud.clone()), + e2e: Some(self.e2e.clone()), hold: Box::new(planes), }); Ok(true) @@ -493,10 +617,16 @@ impl Nv12Upload { .context("staging copy")?; self.dev.device.reset_fences(&[s.fence])?; } + self.copy_gpu_us = self.timer.as_ref().and_then(|t| t.read(&self.dev, i)); } Ok(()) } + /// The GPU time of the last staging copy that has finished, where the GPU can time it. + pub fn copy_gpu_us(&self) -> Option { + self.copy_gpu_us + } + /// A plane set nothing else holds, or a new one while there are fewer than [`PLANE_SETS`]. fn free_planes(&mut self) -> Result>> { if let Some(p) = self.planes.iter().find(|p| Arc::strong_count(p) == 1) { @@ -544,6 +674,9 @@ impl Drop for Nv12Upload { } // SAFETY: the device is idle and every handle below is this upload's. unsafe { + if let Some(t) = &self.timer { + t.destroy(&self.dev); + } for s in self.staging.drain(..) { self.dev.device.destroy_fence(s.fence, None); self.dev.device.destroy_buffer(s.buffer, None); @@ -648,4 +781,66 @@ mod tests { q.push(None, 1); assert_eq!(q.take(t0), Some(1)); } + + #[test] + fn a_standing_backlog_is_skipped() { + let t0 = Instant::now(); + let mut q = Queue::default(); + q.set_smoothing(1); + q.push(Some(t0), 0); + let mut shown = Vec::new(); + for i in 1..=u64::from(STANDING_TAKES) + 2 { + q.push(Some(ms(t0, i * 16)), i); + shown.extend(q.take(ms(t0, i * 16 + 1))); + } + let skip = STANDING_TAKES as usize - 1; + assert_eq!(shown[skip - 1], skip as u64 - 1); + assert_eq!(shown[skip], skip as u64 + 1, "one picture skipped"); + assert_eq!(q.items.len(), 0, "nothing stands behind the take after it"); + assert_eq!((q.drops.replaced, q.window.caught_up), (1, 1)); + } + + #[test] + fn a_backlog_that_drains_is_not_skipped() { + let t0 = Instant::now(); + let mut q = Queue::default(); + q.set_smoothing(1); + for i in 0..200_u64 { + match i % 10 { + // One picture arrives late with the next; the host later skips one, which drains it. + 2 | 6 => {} + 3 => { + q.push(Some(ms(t0, (i - 1) * 16)), i - 1); + q.push(Some(ms(t0, i * 16)), i); + } + _ => q.push(Some(ms(t0, i * 16)), i), + } + q.take(ms(t0, i * 16 + 1)); + } + assert_eq!(q.drops, Drops::default()); + assert_eq!(q.window.caught_up, 0); + } + + /// The window says whether a due time held anything, how late the takes ran and what + /// stood behind each: the figures that tell a binding budget from a plain queue. + #[test] + fn the_window_counts_holds_lateness_and_what_waits() { + let t0 = Instant::now(); + let mut q = Queue::default(); + q.set_smoothing(1); + q.push(Some(ms(t0, 40)), 1); + assert_eq!(q.take(ms(t0, 16)), None, "not due: held"); + q.push(Some(ms(t0, 45)), 2); + assert_eq!(q.take(ms(t0, 50)), Some(1), "10 ms late, one behind it"); + q.push(None, 3); + assert_eq!(q.take(ms(t0, 66)), Some(2)); + assert_eq!(q.take(ms(t0, 83)), Some(3)); + let w = &q.window; + assert_eq!((w.taken, w.held, w.undue, w.early), (3, 1, 1, 0)); + assert_eq!((w.late_n, w.late_max_us), (2, 21_000)); + assert_eq!(w.left, [1, 2, 0]); + assert!(q.window.close(ms(t0, 0)).is_none(), "the first close arms the window"); + assert!(q.window.close(ms(t0, 10_000)).is_some()); + assert_eq!(q.window.taken, 0, "the next window starts empty"); + } } diff --git a/crates/pf-nx-vk/src/presenter.rs b/crates/pf-nx-vk/src/presenter.rs index c4af318..f2a4f4b 100644 --- a/crates/pf-nx-vk/src/presenter.rs +++ b/crates/pf-nx-vk/src/presenter.rs @@ -19,6 +19,7 @@ use crate::device::{GpuDevice, Target}; use crate::output::Output; use crate::picture::{Picture, Planes, VideoSlot}; use crate::pipelines::Pipelines; +use crate::timer::GpuTimer; use crate::ui::{UiLayer, FRAMES}; /// Tonemap source peak for a PQ stream on this SDR output, in 203-nit units (~1000 nits). @@ -68,6 +69,8 @@ pub struct Presenter { want: vk::Extent2D, /// The swapchain no longer matches the window: rebuild before the next acquire. stale: bool, + /// Each frame's command buffer on the GPU; `None` where the queue has no timestamps. + timer: Option, } // SAFETY: the read-back pointer maps memory this struct owns; the presenter lives on the main @@ -143,6 +146,10 @@ impl Presenter { } let readback_sets = alloc_sets(&dev, desc_pool, &pipes)?; let ui = UiLayer::new(&dev, output.extent)?; + let timer = GpuTimer::new(&dev, FRAMES); + if timer.is_none() { + tracing::info!("presenter: the queue has no timestamps, so no GPU time per frame"); + } // The presenter runs on the main thread, decode on the pump: take present turns so a // picture handed over is presented ahead of the next decode on the shared queue. dev.vkd.queue_lock.take_turns(); @@ -166,6 +173,7 @@ impl Presenter { readback_sets, want, stale: false, + timer, }) } @@ -456,6 +464,7 @@ impl Display for Presenter { unsafe { self.dev.device.wait_for_fences(&[fence], true, u64::MAX) } .map_err(|e| anyhow!("wait_for_fences: {e}"))?; self.frames[f].in_use = None; + let gpu_us = self.timer.as_ref().and_then(|t| t.read(&self.dev, f)); if self.stale { self.rebuild()?; } @@ -464,7 +473,10 @@ impl Display for Presenter { // The acquire blocks until the panel frees an image, just after a vblank: the picture is // taken after it, so one that lands during the wait still makes this frame. let index = self.acquire(ready)?; - let mut report = FrameReport::default(); + let mut report = FrameReport { + gpu_us, + ..FrameReport::default() + }; if self.stream.is_some() { // What is presented now latches at the next vblank, a refresh from here. let latch = Instant::now() + Duration::from_nanos(self.refresh().period_ns()); @@ -493,8 +505,14 @@ impl Display for Presenter { cmd, &vk::CommandBufferBeginInfo::default().flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), )?; + if let Some(t) = &mut self.timer { + t.begin(&self.dev, cmd, f); + } self.ui.finish(&self.dev, cmd, f); self.compose(cmd, self.pipes.pass_out, self.framebuffers[index as usize], sets); + if let Some(t) = &self.timer { + t.end(&self.dev, cmd, f); + } self.dev.device.end_command_buffer(cmd)?; } let presents = self.output.presents(); @@ -622,12 +640,16 @@ impl Display for Presenter { } } -/// The overlay's display stage: decoded → handed to scan-out, on the session's HUD. +/// The overlay's display stage: decoded → handed to scan-out, on the session's HUD, and the +/// capture-to-glass time the audio's A/V sync compares against. fn note_displayed(picture: &Picture) { + let now = u64::try_from(punktfunk_core::client::now_realtime_ns()).unwrap_or(0); if let Some(hud) = &picture.hud { - let now = u64::try_from(punktfunk_core::client::now_realtime_ns()).unwrap_or(0); hud.note_displayed(picture.pts_ns, picture.decoded_ns, now, now); } + if let Some(e2e) = &picture.e2e { + e2e.note(picture.pts_ns, now); + } } impl Drop for Presenter { @@ -644,6 +666,9 @@ impl Drop for Presenter { destroy_readback(dev, &rb); } self.ui.destroy(dev); + if let Some(t) = &self.timer { + t.destroy(dev); + } for fr in self.frames.drain(..) { dev.device.destroy_fence(fr.fence, None); dev.device.destroy_semaphore(fr.ready, None); diff --git a/crates/pf-nx-vk/src/pyro.rs b/crates/pf-nx-vk/src/pyro.rs index 92742e3..77e8bb6 100644 --- a/crates/pf-nx-vk/src/pyro.rs +++ b/crates/pf-nx-vk/src/pyro.rs @@ -70,6 +70,7 @@ impl PyroLane { decoder: Mutex::new(decoder), slot: self.slot.clone(), hud: spec.hud.clone(), + e2e: spec.video_e2e.clone(), started: Instant::now(), })) } @@ -79,6 +80,7 @@ struct PyroSink { decoder: Mutex, slot: Arc, hud: Arc, + e2e: pf_nx_app::platform::VideoE2e, started: Instant, } @@ -118,6 +120,7 @@ impl VideoSink for PyroSink { decoded_ns: u64::try_from(punktfunk_core::client::now_realtime_ns()).unwrap_or(0), turn: Some(frame.turn), hud: Some(self.hud.clone()), + e2e: Some(self.e2e.clone()), hold: Box::new(frame.hold), }); Ok(()) diff --git a/crates/pf-nx-vk/src/timer.rs b/crates/pf-nx-vk/src/timer.rs new file mode 100644 index 0000000..f447f2b --- /dev/null +++ b/crates/pf-nx-vk/src/timer.rs @@ -0,0 +1,102 @@ +//! How long a command buffer ran on the GPU, from a timestamp at its start and one at its end: +//! the margin the GPU clock leaves (plan WP5). One pair per command buffer slot, read once that +//! slot's fence has been waited, so a read never stalls. A queue without timestamps reads +//! nothing and records nothing. + +use ash::vk; + +use crate::device::GpuDevice; + +pub(crate) struct GpuTimer { + pool: vk::QueryPool, + /// Nanoseconds per tick. + period_ns: f64, + /// The bits a timestamp carries on this queue. + mask: u64, + /// Whether each slot's pair has been recorded since the pool was made. + recorded: Vec, +} + +impl GpuTimer { + /// Timestamps for `slots` command buffers, or `None` where the queue has none. + pub fn new(dev: &GpuDevice, slots: usize) -> Option { + // SAFETY: read-only queries on the live physical device. + let (limits, families) = unsafe { + ( + dev.instance.get_physical_device_properties(dev.pdev).limits, + dev.instance.get_physical_device_queue_family_properties(dev.pdev), + ) + }; + let bits = families.get(dev.qf as usize)?.timestamp_valid_bits; + if bits == 0 || limits.timestamp_period <= 0.0 { + return None; + } + // SAFETY: the create-info is a local; the pool is destroyed in `destroy`. + let pool = unsafe { + dev.device.create_query_pool( + &vk::QueryPoolCreateInfo::default() + .query_type(vk::QueryType::TIMESTAMP) + .query_count(slots as u32 * 2), + None, + ) + } + .ok()?; + Some(Self { + pool, + period_ns: f64::from(limits.timestamp_period), + mask: if bits >= 64 { u64::MAX } else { (1 << bits) - 1 }, + recorded: vec![false; slots], + }) + } + + /// Slot `i`'s last run in µs. Call only after its fence was waited. + pub fn read(&self, dev: &GpuDevice, i: usize) -> Option { + if !self.recorded[i] { + return None; + } + let mut ticks = [0u64; 2]; + // SAFETY: both queries were written by a submission whose fence the caller waited, so + // the results are available without waiting; `ticks` holds exactly two. + unsafe { + dev.device + .get_query_pool_results(self.pool, i as u32 * 2, &mut ticks, vk::QueryResultFlags::TYPE_64) + } + .ok()?; + let ticks = ticks[1].wrapping_sub(ticks[0]) & self.mask; + u32::try_from((ticks as f64 * self.period_ns / 1000.0) as u64).ok() + } + + /// Records slot `i`'s start. Outside any render pass, first in the command buffer. + /// + /// # Safety + /// `cmd` is recording and is slot `i`'s, whose last submission has finished. + pub unsafe fn begin(&mut self, dev: &GpuDevice, cmd: vk::CommandBuffer, i: usize) { + let q = i as u32 * 2; + // SAFETY: per the contract above. + unsafe { + dev.device.cmd_reset_query_pool(cmd, self.pool, q, 2); + dev.device + .cmd_write_timestamp(cmd, vk::PipelineStageFlags::TOP_OF_PIPE, self.pool, q); + } + self.recorded[i] = true; + } + + /// Records slot `i`'s end, after everything else the command buffer does. + /// + /// # Safety + /// As [`Self::begin`], which was recorded into the same `cmd`. + pub unsafe fn end(&self, dev: &GpuDevice, cmd: vk::CommandBuffer, i: usize) { + // SAFETY: per the contract above. + unsafe { + dev.device + .cmd_write_timestamp(cmd, vk::PipelineStageFlags::BOTTOM_OF_PIPE, self.pool, i as u32 * 2 + 1); + } + } + + /// # Safety + /// No submission that writes the pool is in flight. + pub unsafe fn destroy(&self, dev: &GpuDevice) { + // SAFETY: per the contract above. + unsafe { dev.device.destroy_query_pool(self.pool, None) }; + } +} diff --git a/docs/NOTES.md b/docs/NOTES.md index 9413796..12fe79f 100644 --- a/docs/NOTES.md +++ b/docs/NOTES.md @@ -13,7 +13,9 @@ What builds, and how; nothing here has run on a console. - Check 3 (`quic`) needs punktfunk-core's Horizon arm for `if-addrs` (monorepo #2180, in the pin). - aws-lc builds with its CC builder: no BIO sockets, CPU features from `-march`, getentropy for entropy and no CPU jitter entropy (`scripts/cargo-nx.sh`). rustls's provider is aws-lc-rs; ring is not linked. -- quinn-udp runs its fallback (no ECN, one datagram per call) until the console shows `sendmsg`. +- quinn-udp runs its fallback (no ECN) until the console shows `sendmsg`. Its receive takes up + to 32 datagrams per bsd call through libnx's `recvmmsg`, and drops to one per call if the first + batch is refused: unproven on the console. - tokio's I/O driver wakes through a loopback UDP socket (the mio leg): unproven on the console. - devkitPro's `switch-ffmpeg` in the image carries the `nvtegra` hwaccel (`libavutil/ hwcontext_nvtegra.h`; the symbols are in `libavcodec.a`), so plan D6 needs no FFmpeg fork. diff --git a/docs/improvements.md b/docs/improvements.md index 63f120c..89b7baf 100644 --- a/docs/improvements.md +++ b/docs/improvements.md @@ -6,12 +6,13 @@ the first changes what a player sees by default, the others live in punktfunk's ## The smoothness buffer's Automatic on the Switch The shared setting resolves Automatic to two frames on every client -(`pf-client-core::trust::PresentPriority::resolve`). This client adds the frames on top of -core's adaptive cushion (`session/timeline.rs`), so one frame here already holds up to about -two refreshes. On the console, one frame presented 59–60 pictures a second at 15–23 ms decode to -present; two frames presented the same at 31–40 ms (NOTES, 2026-10-31). The proposal is that -Automatic picks one frame on the Switch, through a per-platform default in the shared crate or -an override in this client. +(`pf-client-core::trust::PresentPriority::resolve`). This client counts core's adaptive cushion +as the first frame and adds a stream period for each further one (`session/timeline.rs`). Until +2f4002a it added every frame on top of the cushion, so one frame held up to about two +refreshes. On the console, one frame presented 59–60 pictures a second at 15–23 ms decode to +present; two frames presented the same at 31–40 ms (NOTES, 2026-10-31). Both runs predate +2f4002a. The proposal is that Automatic picks one frame on the Switch, through a per-platform +default in the shared crate or an override in this client. Open: one tester's Wi-Fi so far. A busier network may want the second frame back. @@ -50,3 +51,63 @@ the Switch's own step goes. The Windows host built a Joy-Con pair in every conso (NOTES, 2026-10-31); a Switch Pro there is untested. Open: whether every host can build a Switch Pro, or the lists should offer it per host. + +## A Switch platform in the shared shell + +The shell has no Switch, so this client fronts it as `Platform::Ps5` (`runtime/menu.rs`), the +nearest: a console title that quits, with no window. That borrows what is the PS5's alone. The +tester found the first on 2026-11-02: + +- **Stream controls.** About > Stream controls draws a DualSense card that lists "Touchpad + + Cross" and "L1 + R1 + Options + Touchpad". On the Switch the dial is Minus + A + (`core/dial.rs`), and holding L + R + Plus + Minus for a second ends the stream + (`runtime/pads.rs`, `CHORD_HOLD`). The page tells a player to press buttons that do nothing. +- **Confirm side.** The shell confirms on south. Here A, on east, confirms, so `legend_pref` + (`runtime/menu.rs`) swaps the Nintendo legend for the letters one. Without that, the hints + read "B Select" over the button that selects. + +The contract this client builds against, in the monorepo: + +1. `pf-console-ui::platform::Platform::Switch`, in `Platform::ALL`, with `can_quit` true. +2. `pf-client-core::shortcuts::Client::Switch`, mapped from it in `screens/controls.rs`. Its pad + group is titled "Joy-Con" and holds: + - `item("Minus + A", "Open the quick actions dial")`; + - `item("L + R + Plus + Minus", "Hold to disconnect")`. + + The group's icon and badges come from the first pad's pref, as now. The pad art for + `joyconpair` and `switchpro` already exists (`pad_art.rs`). +3. On `Platform::Switch`, the Nintendo glyph style confirms on east and backs on south. Every + hint that says Select or Back follows that. +4. The rest of the PS5's branches, decided for the Switch: + + | Where | PS5 now | Switch | + | --- | --- | --- | + | `codecs` | `CODECS_PS5`: no AV1, PyroWave behind `pyrowave_ok` | the same. NVDEC has no AV1, and PyroWave runs on NVK's Vulkan 1.3, untested on the console (`pyrowave: false` until then) | + | `pad_types` | drops Xbox 360, Steam Deck, Steam Controller 2 | the same, plus the Nintendo kinds from the section above | + | `own_stats_corner` | top left | top left | + | `has_window` | false | false | + | `bitrate_ceiling_kbps` | the custom maximum | the custom maximum | + | `ring_editor` platform | Desktop | Desktop | + | `GlyphStyle::keys` | Remote | Remote | + | help-text branches (mouse mode, stats verbosity, gamepad UI) | the no-chord wording | the same | + + `players.rs` opens a pad's menu (Test controller, Rumble) on Android and Apple only + (`can_test`, `can_rumble`). On every other platform, OK on a Controllers card does nothing, so + the Switch's Joy-Con card is display-only. The Switch can do both: its pads rumble in every + stream. The contract lets `Platform::Switch` offer the two rows. This client then answers + `ConsoleCmd::PadTest` and the `rumble` `PadAction`, which `ui/service.rs` logs and drops + today. + + The handheld screen is a touchscreen, and this client sends its touches to the host as a + direct pointer. That is a later row, not part of this contract. + +On this side, once the client builds on a release that has it: + +- `runtime/menu.rs` sets `platform: Platform::Switch`; +- `legend_pref` goes, and the pads report their own pref; +- both `TODO(core)` lines go. + +Until then, the guide names the two chords, because the Stream controls page does not. + +Open: whether the shell should take the dial and escape chords from the client, not from a +table, so a client that rebinds them cannot drift from the page. diff --git a/docs/plan.md b/docs/plan.md index f90ec75..ea0db79 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -1,7 +1,7 @@ # Nintendo Switch client — a Rust homebrew title on NVDEC > **Status: designed 2026-10-28; WP0's probe passed on the console and WP1 closed on 2026-10-29 -> (`docs/NOTES.md`, §4); WP2–WP4 closed by 2026-11-01. WP5 is next.** Target: the Switch 1 (Erista and Mariko, Tegra +> (`docs/NOTES.md`, §4); WP2–WP4 closed by 2026-11-01, WP5 on 2026-11-02. WP6 is next.** Target: the Switch 1 (Erista and Mariko, Tegra > X1), on Atmosphère, as an `.nro`. The Switch 2 is a later leg behind the same seams (§9); no > modded one exists to test on. The maintainer's fellow developer holds the only test device, so > every work package is written for a developer who never holds the console — the PS5 client's @@ -28,7 +28,8 @@ answer for anyone who will not install a CFW. settings are the console UI's shared catalogue (`clients/shared/settings-catalog.json`). - **Contract twins are pinned with the shared vectors**, not FFI: `gamepad-kind-vectors.json`, `touch-vectors.json`, `device-gyro-vectors.json`, `jitter-vectors.json`. -- **Automatic means automatic**: no bitrate limit, no decoder pick, no clock boost in v1. +- **Automatic picks the bitrate under a ceiling**: 20 Mb/s where only NVDEC decodes, the rate + the console receives clean at stock clocks. No decoder pick. Clock control is its own task (§10). - **Prior art is pinned, not forked loosely**: every vendored crate leg names its upstream version and the project it came from (§3.1) in its README. @@ -48,7 +49,7 @@ answer for anyone who will not install a CFW. | D10 | **Face buttons follow the OS: A confirms, B backs**, in the console UI on this platform. Every Switch app does; a client that swaps them is a bug to the player. (The Android Nintendo-menu question stays the maintainer's; this is the native case of it.) | | D11 | **Stream mode follows the operation mode**: 1280×720@60 handheld, 1920×1080@60 docked, read at connect and on `AppletHookType_OnOperationMode`; a dock change mid-stream asks the host for the new size (host owns the delivered mode). Refresh is 60. | | D12 | **Sleep ends the stream** (`OnFocusState`/sleep hook → clean close); wake shows the host list. Reconnect is the player's press. No session survives suspend. | -| D13 | **Full-memory mode first.** The title is validated launched by title override (hold R over an installed title); applet mode (album) is detected (`appletGetAppletType`) and warned about on the start screen, and WP5 measures whether the client fits in it. | +| D13 | **Full-memory mode first.** The title is validated launched by title override (hold R over an installed title); applet mode (album) is detected (`appletGetAppletType`) and warned about on the start screen. A launch from the Album crashed before the first menu, so applet mode is not supported: the manual says to hold R over a game (WP6). | | D14 | **Codecs advertised**: `CODEC_H264 | CODEC_HEVC`, 8-bit, SDR. No AV1 on the X1 (accepted). Main10 only if WP0's probe decodes it through the VIC path. PyroWave is out of v1: the lane lifts with `pf-ps5-vk`, but a 256-core Maxwell at 307–460 MHz has to be measured against the wavelet decode before it is advertised (§10). | | D15 | **Dev loop without the console**: `pf-nx-host` runs the app on a Linux box through the real presenter on lavapipe and on the Mac on CPU raster, as `pf-ps5-host` does; **Ryujinx (Ryubing) on the Mac runs the real NRO on the framebuffer `Display`** — libnx, sockets, the nvservices the hwaccel talks to are emulated, so everything but the GPU path is proven here. Stock emulators do not run NVK builds (`bevy-horizon`'s finding). Timing from the emulator is never cited. `nxlink -s` is the deploy and the stdout on the device. | | D16 | **CI**: GitHub Actions like `client-ps5`: a host lane (test, clippy, the host client on lavapipe) and a console lane in a Docker image from `devkitpro/devkita64` plus the nightly, rust-src, and three archives built once per pin (`task nvk`, `task ffmpeg`, `task skia`). The NRO is a CI artifact per PR; releases are tags. | @@ -223,7 +224,8 @@ bundle to `sdmc:/switch/punktfunk/verdict/` (`docs/NOTES.md`, title at `927533b` is gone. Ryubing recognised the NRO and then closed itself, untraced; the framebuffer fallback waits for a console where NVK fails. - The menu holds 60 fps in most ten-second windows, not all: some lose up to 33 frames to - single frames of 65–213 ms. WP5 traces them. + single frames of 65–213 ms. WP5's clock sweep put them on the CPU clock: at 1785 MHz the menu + lagged less. They go with clock control (§10). - The bundle is a folder of PNGs and JSON on the SD card, not a zip, fetched by hand. ### WP3 — The video plane (three weeks) @@ -275,13 +277,26 @@ package there. Departures: - The Nintendo controller types in the shared Settings list and labels wait for a monorepo PR (`docs/improvements.md`). Audio underruns climb in silence with nothing audible; left for WP5. -### WP5 — Reliability and latency (two weeks) +### WP5 — Reliability and latency (closed 2026-11-02) -Measured, not guessed: the picture copy (D7) and nvmap import if it shows; the audio sync cell -if the presentation clock drifts from it; Wi-Fi loss behaviour against the host's recovery; -memory in applet mode; the GPU clock the title gets and whether the official handheld -configuration is reachable without an opt-in; PyroWave on NVK against the GPU budget (D14). -Ten-hour soak. +Measured on the console: + +- The picture copy (D7) costs what the straight VIC route saves, so the copy stays and nvmap + import is not needed. +- Audio plays 40–90 ms behind the picture, steady, with no drift. +- The receive path reads 32 datagrams a call. It holds 20 Mb/s handheld with no loss; docked + 1080p wants 10–12 Mb/s. +- The smoothness budget adds no whole period, and the slot no longer keeps a picture standing. +- The stats line carries the CPU, GPU and memory clocks. A handheld sweep found that the CPU + clock is the one that cuts latency. + +Taken out: + +- The ten-hour soak: public use answers it better than one tester can. +- Memory in applet mode: the title does not launch there (D13). +- A trace of the menu's long frames: the CPU clock explains them. + +Clock control and PyroWave on NVK are their own tasks after the baseline (§10). ### WP6 — Package and release (one week) @@ -343,7 +358,7 @@ client-switch/ | Skia archive | §3.3 | raster fallback | | `nvtegra` is not in devkitPro's `switch-ffmpeg` at the pinned version | WP0.6 | build the pinned fork; `scripts/build-ffmpeg.sh` exists for it either way | | The crate legs drift | every nightly or pin bump | `scripts/pins.env` pins them; a bump is a PR with the probe re-run | -| Applet mode out of memory | WP0.9, WP5 | title-override documented as required; applet mode best effort | +| Applet mode out of memory | WP0.9 | title override documented as required; applet mode unsupported (D13) | | The emulator hides a device-only fault | every round | the tester's bundle decides; Ryujinx is never cited | | One device, one tester | throughput | rounds are batched; the probe lists what to run | @@ -358,10 +373,12 @@ is a second `pf-nx-platform` cfg, not a fork. ## 10. After the baseline -PyroWave on NVK if WP5's budget allows; nvmap import for the stream image (if WP5 leaves it); +Clock control from the title (CPU, GPU, memory): a probe first, for whether clkrst sets them +under hbloader, against sys-clk, and how they come back on exit, sleep and a dock change. Then an +opt-in performance mode, the menu's long frames, and Automatic's ceiling raised to about 25 Mb/s. +PyroWave on NVK, measured against the GPU budget (D14), as a task of its own. USB keyboard and mouse on the dock (`hidKeyboard*`/`hidMouse*` → the keymap contract); -`hwopus`; the GPU-clock question if WP5 shows a case; the Homebrew App Store listing; the -Switch 2. +`hwopus`; the Homebrew App Store listing; the Switch 2. ## 11. Open before WP0 starts (the maintainer's) diff --git a/toolchain/patches/quinn-udp/LEG.md b/toolchain/patches/quinn-udp/LEG.md index 7ea82e8..065f609 100644 --- a/toolchain/patches/quinn-udp/LEG.md +++ b/toolchain/patches/quinn-udp/LEG.md @@ -4,7 +4,9 @@ - **Origin:** punktfunk. - **Changes:** `src/lib.rs` builds `fallback.rs` on Horizon instead of `unix.rs`, whose `cmsg` path names Linux and BSD socket options libnx does not define. `fallback.rs` (dead upstream on - every target quinn ships) receives one datagram per call with `recv_from` (socket2 has no - vectored receive on Horizon) and discards `send_to`'s byte count. -- **Cost:** no ECN, no destination address, one datagram per syscall. The `unix.rs` path comes - back behind a cfg once probe check 2 shows `sendmsg`/`recvmsg` on the console. + every target quinn ships) discards `send_to`'s byte count and, on Horizon, receives up to 32 + datagrams per call through libnx's `recvmmsg` (`BATCH_SIZE` 32). A first refusal other than + `WouldBlock` drops that socket to one datagram per call with `recv_from` (socket2 has no + vectored receive on Horizon). +- **Cost:** no ECN, no destination address. The `unix.rs` path comes back behind a cfg once probe + check 2 shows `sendmsg`/`recvmsg` on the console. diff --git a/toolchain/patches/quinn-udp/src/fallback.rs b/toolchain/patches/quinn-udp/src/fallback.rs index 932942b..2428181 100644 --- a/toolchain/patches/quinn-udp/src/fallback.rs +++ b/toolchain/patches/quinn-udp/src/fallback.rs @@ -1,6 +1,9 @@ use std::{ io::{self, IoSliceMut}, - sync::Mutex, + sync::{ + Mutex, + atomic::{AtomicU8, Ordering}, + }, time::Instant, }; @@ -13,14 +16,22 @@ use super::{IO_ERROR_LOG_INTERVAL, RecvMeta, Transmit, UdpSockRef, log_sendmsg_e #[derive(Debug)] pub struct UdpSocketState { last_send_error: Mutex, + /// Horizon (punktfunk's leg): whether `recvmmsg` works here, [`BATCH_UNKNOWN`] until its + /// first answer. + batched: AtomicU8, } +const BATCH_UNKNOWN: u8 = 0; +const BATCH_ON: u8 = 1; +const BATCH_OFF: u8 = 2; + impl UdpSocketState { pub fn new(socket: UdpSockRef<'_>) -> io::Result { socket.0.set_nonblocking(true)?; let now = Instant::now(); Ok(Self { last_send_error: Mutex::new(now.checked_sub(2 * IO_ERROR_LOG_INTERVAL).unwrap_or(now)), + batched: AtomicU8::new(BATCH_UNKNOWN), }) } @@ -52,28 +63,33 @@ impl UdpSocketState { send(socket, transmit) } + /// Horizon (punktfunk's leg): up to [`BATCH_SIZE`] datagrams in one `recvmmsg`, one bsd + /// call for the lot. A first refusal other than an empty socket turns batching off for + /// this socket and reads one datagram a call, as before. pub fn recv( &self, socket: UdpSockRef<'_>, bufs: &mut [IoSliceMut<'_>], meta: &mut [RecvMeta], ) -> io::Result { - // Horizon (punktfunk's leg): socket2 has no vectored receive there, so this reads one - // datagram into the first buffer, which quinn sizes for a whole datagram. - // Safety: `u8` and `MaybeUninit` share a layout, and `recv_from` writes only - // initialised bytes into the buffer. - let buf = unsafe { - &mut *(&mut *bufs[0] as *mut [u8] as *mut [std::mem::MaybeUninit]) - }; - let (len, addr) = socket.0.recv_from(buf)?; - meta[0] = RecvMeta { - len, - stride: len, - addr: addr.as_socket().unwrap(), - ecn: None, - dst_ip: None, - }; - Ok(1) + #[cfg(target_os = "horizon")] + if self.batched.load(Ordering::Relaxed) != BATCH_OFF { + match recv_batch(&socket, bufs, meta) { + Ok(n) => { + if self.batched.swap(BATCH_ON, Ordering::Relaxed) == BATCH_UNKNOWN { + crate::log::info!("recvmmsg: up to {BATCH_SIZE} datagrams a call, {n} in the first"); + } + return Ok(n); + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => return Err(e), + Err(e) if self.batched.load(Ordering::Relaxed) == BATCH_ON => return Err(e), + Err(e) => { + crate::log::warn!("recvmmsg refused ({e}); one datagram a call from here"); + self.batched.store(BATCH_OFF, Ordering::Relaxed); + } + } + } + recv_one(&socket, bufs, meta) } #[inline] @@ -126,4 +142,113 @@ fn send(socket: UdpSockRef<'_>, transmit: &Transmit<'_>) -> io::Result<()> { .map(|_| ()) } +/// One datagram into the first buffer, which quinn sizes for a whole datagram: socket2 has no +/// vectored receive on Horizon. +fn recv_one( + socket: &UdpSockRef<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], +) -> io::Result { + // Safety: `u8` and `MaybeUninit` share a layout, and `recv_from` writes only + // initialised bytes into the buffer. + let buf = unsafe { &mut *(&mut *bufs[0] as *mut [u8] as *mut [std::mem::MaybeUninit]) }; + let (len, addr) = socket.0.recv_from(buf)?; + meta[0] = RecvMeta { + len, + stride: len, + addr: addr.as_socket().unwrap(), + ecn: None, + dst_ip: None, + }; + Ok(1) +} + +/// Up to [`BATCH_SIZE`] datagrams, one per buffer, through libnx's `recvmmsg` (bsd's +/// RecvMMsg, firmware 7.0.0). `MSG_DONTWAIT`: an empty socket is `WouldBlock`, and the call +/// returns what is queued rather than waiting for a full batch. +#[cfg(target_os = "horizon")] +fn recv_batch( + socket: &UdpSockRef<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], +) -> io::Result { + use std::os::unix::io::AsRawFd; + let n = bufs.len().min(meta.len()).min(BATCH_SIZE); + // Safety: all-zero is a valid value of these plain C structs. + let mut names: [libc::sockaddr_storage; BATCH_SIZE] = unsafe { std::mem::zeroed() }; + let mut iovs: [libc::iovec; BATCH_SIZE] = unsafe { std::mem::zeroed() }; + let mut hdrs: [libc::mmsghdr; BATCH_SIZE] = unsafe { std::mem::zeroed() }; + for i in 0..n { + iovs[i] = libc::iovec { + iov_base: bufs[i].as_mut_ptr().cast(), + iov_len: bufs[i].len(), + }; + let h = &mut hdrs[i].msg_hdr; + h.msg_name = (&mut names[i] as *mut libc::sockaddr_storage).cast(); + h.msg_namelen = std::mem::size_of::() as libc::socklen_t; + h.msg_iov = &mut iovs[i]; + h.msg_iovlen = 1; + } + // Safety: the first `n` headers point at `n` live buffers, iovecs and address slots, all + // outliving the call; bsd writes within their stated lengths. + let got = unsafe { + libc::recvmmsg( + socket.0.as_raw_fd(), + hdrs.as_mut_ptr(), + n as libc::c_uint, + libc::MSG_DONTWAIT, + std::ptr::null_mut(), + ) + }; + if got < 0 { + return Err(io::Error::last_os_error()); + } + let got = got as usize; + for i in 0..got { + let len = hdrs[i].msg_len as usize; + let addr = socket_addr(&names[i]).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "recvmmsg: an address of no known family") + })?; + meta[i] = RecvMeta { + len, + stride: len, + addr, + ecn: None, + dst_ip: None, + }; + } + Ok(got) +} + +/// The sender's address as `recvmmsg` wrote it. +#[cfg(target_os = "horizon")] +fn socket_addr(s: &libc::sockaddr_storage) -> Option { + use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; + match libc::c_int::from(s.ss_family) { + libc::AF_INET => { + // Safety: an AF_INET address is a `sockaddr_in`, which the storage outsizes. + let a = unsafe { &*(s as *const libc::sockaddr_storage).cast::() }; + Some(SocketAddr::V4(SocketAddrV4::new( + Ipv4Addr::from(u32::from_be(a.sin_addr.s_addr)), + u16::from_be(a.sin_port), + ))) + } + libc::AF_INET6 => { + // Safety: an AF_INET6 address is a `sockaddr_in6`, which the storage outsizes. + let a = unsafe { &*(s as *const libc::sockaddr_storage).cast::() }; + Some(SocketAddr::V6(SocketAddrV6::new( + Ipv6Addr::from(a.sin6_addr.s6_addr), + u16::from_be(a.sin6_port), + a.sin6_flowinfo, + a.sin6_scope_id, + ))) + } + _ => None, + } +} + +/// Datagrams one [`UdpSocketState::recv`] takes: one `recvmmsg` batch on Horizon. +#[cfg(target_os = "horizon")] +pub(crate) const BATCH_SIZE: usize = 32; +#[cfg(not(target_os = "horizon"))] pub(crate) const BATCH_SIZE: usize = 1;