From b90bc515206157f97eb02c1eb805ee526703d33c Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:17:36 -0300 Subject: [PATCH 01/14] feat(wp5): measure the picture copy, the GPU and the audio sync The VIC now writes each picture straight into the presenter's staging buffer: the shim hands av_hwframe_transfer_data the caller's planes, which the nvtegra hwcontext maps per transfer. Through libavutil's own frame the picture was copied again, and that frame's 16-byte alignment likely sent the hwcontext to its CPU detiling instead of the VIC. A failed straight transfer falls back to the old route for the rest of the decoder's life. The nvdec line names the route and adds the shim's copy and the staging copy's time on the GPU. The stats and menu lines carry the CPU, GPU and memory clocks, read from clkrst every ten seconds, and the stats line the composition's GPU time per frame from timestamp queries, which is the margin the clock leaves. The audio route places every packet against the picture on the glass with core's AvSync and logs the offset, smoothed and its spread, every ten seconds; it only measures and does not steer the ring. The same line splits underruns into silent and audible ones and counts the host's pauses. Co-Authored-By: Claude Opus 5.5 --- crates/pf-nx-app/src/platform/mod.rs | 72 +++++++ crates/pf-nx-app/src/runtime/menu.rs | 2 +- crates/pf-nx-app/src/runtime/mod.rs | 7 + crates/pf-nx-app/src/runtime/stream.rs | 15 +- crates/pf-nx-app/src/session/audio_out.rs | 192 +++++++++++++++++- crates/pf-nx-app/src/session/pipeline.rs | 6 +- crates/pf-nx-platform/src/console/audio.rs | 4 + .../pf-nx-platform/src/console/lifecycle.rs | 28 ++- crates/pf-nx-platform/src/video.rs | 65 +++++- crates/pf-nx-sys/csrc/pf_av.c | 89 +++++++- crates/pf-nx-sys/src/av.rs | 65 +++++- crates/pf-nx-sys/tests/layout.rs | 4 +- crates/pf-nx-vk/src/lib.rs | 1 + crates/pf-nx-vk/src/picture.rs | 33 ++- crates/pf-nx-vk/src/presenter.rs | 31 ++- crates/pf-nx-vk/src/pyro.rs | 3 + crates/pf-nx-vk/src/timer.rs | 102 ++++++++++ 17 files changed, 679 insertions(+), 40 deletions(-) create mode 100644 crates/pf-nx-vk/src/timer.rs diff --git a/crates/pf-nx-app/src/platform/mod.rs b/crates/pf-nx-app/src/platform/mod.rs index d9432f6..b105659 100644 --- a/crates/pf-nx-app/src/platform/mod.rs +++ b/crates/pf-nx-app/src/platform/mod.rs @@ -9,6 +9,7 @@ pub mod fake; use std::io::Write; use std::path::Path; +use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; use std::sync::Arc; use anyhow::Result; @@ -122,6 +123,9 @@ pub struct FrameReport { pub replaced: u32, /// Decoded pictures dropped unseen since the last frame, pushed out of a full smoothing queue. pub overflow: u32, + /// How long an earlier frame's composition ran on the GPU (the one whose slot this frame + /// reused), where the GPU can time it. + pub gpu_us: Option, } /// A read-back frame. @@ -159,6 +163,42 @@ pub struct StreamSpec { pub smoothing: usize, /// The session's overlay figures: lanes note decode, the presenter notes display. pub hud: Arc, + /// Where the presenter leaves each picture's capture-to-glass time for the audio's A/V + /// sync measure. + pub video_e2e: VideoE2e, +} + +/// The video's capture-to-glass time in core's cells: the presenter writes it, the audio route +/// compares its own against it. `0` until a picture reaches the glass. +#[derive(Clone, Debug, Default)] +pub struct VideoE2e { + e2e_ns: Arc, + /// Host clock minus this one, kept current by core. + clock_offset_ns: Arc, +} + +/// A capture-to-glass time past this is a clock step, not a picture. +const E2E_SANE_NS: u64 = 10_000_000_000; + +impl VideoE2e { + pub fn new(e2e_ns: Arc, clock_offset_ns: Arc) -> Self { + Self { + e2e_ns, + clock_offset_ns, + } + } + + /// A picture captured at host time `pts_ns` reached the glass at `displayed_ns`, this + /// clock's realtime. + pub fn note(&self, pts_ns: u64, displayed_ns: u64) { + let offset = self.clock_offset_ns.load(Ordering::Relaxed); + let e2e = i128::from(displayed_ns) + i128::from(offset) - i128::from(pts_ns); + if let Ok(ns) = u64::try_from(e2e) { + if ns > 0 && ns < E2E_SANE_NS { + self.e2e_ns.store(ns, Ordering::Relaxed); + } + } + } } /// The console's audio output. Opening a port is cheap; a session holds one. @@ -173,6 +213,11 @@ pub trait AudioPort: Send { /// Plays exactly one grain, blocking until the device takes it. fn write(&mut self, pcm: &[f32]) -> Result<()>; + + /// Frames handed to the device and not yet played, which the next grain queues behind. + fn queued_frames(&self) -> usize { + 0 + } } /// One pad's state at one instant, already in punktfunk's wire vocabulary. @@ -277,4 +322,31 @@ pub trait Lifecycle { fn hide_splash(&self); fn poll_events(&mut self, out: &mut Vec); fn exit(&self) -> !; + + /// The clocks the system runs the title at now, for the logs. `None` where they cannot be + /// read: a dev box, or a console before firmware 8.0.0. + fn clocks(&self) -> Option { + None + } +} + +/// The CPU, GPU and memory clocks, in Hz. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Clocks { + pub cpu_hz: u32, + pub gpu_hz: u32, + pub emc_hz: u32, +} + +impl std::fmt::Display for Clocks { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mhz = |hz: u32| f64::from(hz) / 1e6; + write!( + f, + "gpu {:.1} cpu {:.1} emc {:.1} MHz", + mhz(self.gpu_hz), + mhz(self.cpu_hz), + mhz(self.emc_hz) + ) + } } diff --git a/crates/pf-nx-app/src/runtime/menu.rs b/crates/pf-nx-app/src/runtime/menu.rs index 672c4fe..8127425 100644 --- a/crates/pf-nx-app/src/runtime/menu.rs +++ b/crates/pf-nx-app/src/runtime/menu.rs @@ -222,7 +222,7 @@ pub(super) fn run(app: &mut App, notice: Option) -> Result { })?; frames += 1; if let Some(report) = perf.frame(before_draw + drew, art_snapshot()) { - tracing::info!("{}", report.line()); + tracing::info!("{}{}", report.line(), app.clocks_suffix()); } if let Some(done) = testrun::menu_step(app, frames) { break 'ui done; diff --git a/crates/pf-nx-app/src/runtime/mod.rs b/crates/pf-nx-app/src/runtime/mod.rs index 20828ce..28ebf55 100644 --- a/crates/pf-nx-app/src/runtime/mod.rs +++ b/crates/pf-nx-app/src/runtime/mod.rs @@ -48,6 +48,13 @@ impl App { } events } + + /// `" | clocks gpu … MHz"` for a 10 s summary line, empty where the clocks cannot be read. + fn clocks_suffix(&self) -> String { + self.lifecycle + .clocks() + .map_or_else(String::new, |c| format!(" | clocks {c}")) + } } /// Sleep is on its way: no new session may start. diff --git a/crates/pf-nx-app/src/runtime/stream.rs b/crates/pf-nx-app/src/runtime/stream.rs index 8231fde..9cab900 100644 --- a/crates/pf-nx-app/src/runtime/stream.rs +++ b/crates/pf-nx-app/src/runtime/stream.rs @@ -420,7 +420,7 @@ fn stream( } if let Some(core) = hud.refresh(connected, decoder) { - tracing::info!("stats: {} | {core}", perf.take_window()); + tracing::info!("stats: {} | {core}{}", perf.take_window(), app.clocks_suffix()); } hud.local = vec![format!( "present: {} frames, {} missed{}", @@ -582,6 +582,8 @@ struct PerfWindow { replaced: u32, overflow: u32, decode_to_present_us: Vec, + /// The composition's GPU time per frame, where the presenter can time it. + gpu_us: Vec, } impl StreamPerf { @@ -604,6 +606,9 @@ impl StreamPerf { } w.decode_to_present_us.push(us); } + if let Some(us) = report.gpu_us { + w.gpu_us.push(us); + } } /// The presenter's figures since the last call, and a new window. @@ -611,10 +616,13 @@ impl StreamPerf { let mut w = std::mem::take(&mut self.window); let secs = w.since.map_or(0.0, |t| t.elapsed().as_secs_f64()); w.decode_to_present_us.sort_unstable(); + w.gpu_us.sort_unstable(); let ms = |us: Option<&u32>| us.map_or("-".to_string(), |&us| format!("{:.1}", f64::from(us) / 1000.0)); + let fine = |us: Option<&u32>| us.map_or("-".to_string(), |&us| format!("{:.2}", f64::from(us) / 1000.0)); let d = &w.decode_to_present_us; + let g = &w.gpu_us; format!( - "present {secs:.1}s: {} frames, {} missed vblanks, {} pictures, {} unseen ({} replaced, {} queue full), decode to present p50 {} p95 {} max {} ms", + "present {secs:.1}s: {} frames, {} missed vblanks, {} pictures, {} unseen ({} replaced, {} queue full), decode to present p50 {} p95 {} max {} ms, compose on the gpu p50 {} p95 {} max {} ms", w.frames, w.missed, w.pictures, @@ -624,6 +632,9 @@ impl StreamPerf { ms(d.get(d.len() / 2)), ms(d.get(d.len() * 95 / 100)), ms(d.last()), + fine(g.get(g.len() / 2)), + fine(g.get(g.len() * 95 / 100)), + fine(g.last()), ) } diff --git a/crates/pf-nx-app/src/session/audio_out.rs b/crates/pf-nx-app/src/session/audio_out.rs index 6398986..5b33dae 100644 --- a/crates/pf-nx-app/src/session/audio_out.rs +++ b/crates/pf-nx-app/src/session/audio_out.rs @@ -3,14 +3,21 @@ //! //! The port blocks for a grain and nothing rate-matches it to the host, so depth is steered by //! punktfunk-core's shared [`JitterPolicy`], the same one every client's ring runs. +//! +//! Every packet is also placed against the picture on the glass with core's [`AvSync`]: where +//! the audio will play in the host's capture clock, next to where the video is. That only +//! measures (plan WP5): the offset goes to the stats line and the `audio[10s]` line, and the +//! ring is not steered by it until a long session shows it drift. The same line splits the +//! underruns by whether the audio before them was silent, and counts the host's pauses. use std::collections::VecDeque; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU32, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Condvar, Mutex}; -use std::time::Duration; +use std::time::{Duration, Instant}; use anyhow::{Context, Result}; -use punktfunk_core::audio::{crossfade_drop, crossfade_insert, JitterPolicy, JitterTuning}; +use punktfunk_core::audio::{crossfade_drop, crossfade_insert, AvSync, AvSyncObservation, JitterPolicy, JitterTuning}; +use punktfunk_core::client::NativeClient; use crate::core::media::{AudioFormat, AudioSink, Samples}; use crate::platform::AudioPort; @@ -38,10 +45,66 @@ fn playout(channels: u8) -> punktfunk_core::audio::AudioPlayout { } } +/// How often the route logs its window. +const WINDOW: Duration = Duration::from_secs(10); +/// A packet this far below full scale is silence, so an underrun after it cannot be heard: +/// −50 dBFS. +const SILENT_PEAK: f32 = 0.003_16; +/// No packet for this long is the host pausing, not jitter. +const PAUSE: Duration = Duration::from_millis(100); + +/// Core's cells the route reads and writes: the clock skew and the video's capture-to-glass +/// time in, the smoothed A/V offset and the ring's depth out (the stats line shows both). +#[derive(Clone, Default)] +pub struct SyncCells { + pub clock_offset_ns: Arc, + pub video_e2e_ns: Arc, + pub av_offset_ms: Arc, + pub buffer_ms: Arc, +} + +impl SyncCells { + pub fn of(client: &NativeClient) -> Self { + Self { + clock_offset_ns: client.clock_offset_shared(), + video_e2e_ns: client.video_e2e_shared(), + av_offset_ms: client.audio_av_offset_shared(), + buffer_ms: client.audio_buffer_ms_shared(), + } + } +} + struct Ring { samples: VecDeque, policy: JitterPolicy, depth_ms: u32, + sync: AvSync, + /// The peak of the last packet fed, which tells a silent underrun from an audible one. + last_peak: f32, + last_fed: Option, + window: Window, +} + +/// One [`WINDOW`] of what the route did, filled by both threads under the ring's lock. +#[derive(Default)] +struct Window { + grains: u32, + /// Grains of silence while the ring primed or re-primed: not underruns. + priming: u32, + silent_underruns: u32, + audible_underruns: u32, + sheds: u32, + inserts: u32, + depth_min_ms: Option, + depth_max_ms: u32, + packets: u32, + pauses: u32, + longest_gap: Duration, + /// Unsmoothed A/V offsets, ms: positive is audio late. + av_sum_ms: i64, + av_n: u32, + av_min_ms: Option, + av_max_ms: Option, } struct Shared { @@ -49,6 +112,9 @@ struct Shared { fed: Condvar, stop: AtomicBool, underruns: AtomicU64, + /// The port's queue after the last write, in frames: audio that plays before anything fed now. + port_queued: AtomicUsize, + cells: SyncCells, } /// An [`AudioSink`] over one open port, plus the thread that plays it. @@ -59,17 +125,23 @@ pub struct AudioOutSink { } impl AudioOutSink { - pub fn start(port: Box, channels: u8) -> Result { + pub fn start(port: Box, channels: u8, cells: SyncCells) -> Result { let rate = crate::session::audio::SAMPLE_RATE; let shared = Arc::new(Shared { ring: Mutex::new(Ring { samples: VecDeque::with_capacity(rate as usize * usize::from(channels) / 5), policy: playout(channels).jitter(), depth_ms: 0, + sync: playout(channels).sync(), + last_peak: 0.0, + last_fed: None, + window: Window::default(), }), fed: Condvar::new(), stop: AtomicBool::new(false), underruns: AtomicU64::new(0), + port_queued: AtomicUsize::new(0), + cells, }); let thread_shared = shared.clone(); let writer = std::thread::Builder::new() @@ -115,12 +187,47 @@ impl AudioSink for AudioOutSink { } } - fn feed(&self, samples: Samples<'_>, _host_pts_ns: u64) -> Result<()> { + fn feed(&self, samples: Samples<'_>, host_pts_ns: u64) -> Result<()> { let Samples::F32(pcm) = samples; - let mut ring = self.shared.ring.lock().expect("audio ring poisoned"); - ring.samples.extend(pcm.iter().copied()); + let peak = pcm.iter().fold(0f32, |m, s| m.max(s.abs())); + let now = Instant::now(); + let shared = &self.shared; + let mut ring = shared.ring.lock().expect("audio ring poisoned"); + let r = &mut *ring; + if let Some(gap) = r.last_fed.map(|t| now - t) { + r.window.pauses += u32::from(gap >= PAUSE); + r.window.longest_gap = r.window.longest_gap.max(gap); + } + r.last_fed = Some(now); + r.last_peak = peak; + r.window.packets += 1; + // Everything that plays before this packet: the ring, and the port's queue. + let ahead = r.samples.len() + shared.port_queued.load(Ordering::Relaxed) * usize::from(self.channels); + let cells = &shared.cells; + let video_e2e_ns = cells.video_e2e_ns.load(Ordering::Relaxed); + let o = AvSyncObservation { + pts_ns: host_pts_ns, + now_local_ns: punktfunk_core::client::now_realtime_ns(), + clock_offset_ns: cells.clock_offset_ns.load(Ordering::Relaxed), + buffered_ahead: ahead, + // What audout adds past its queue is not known here: a constant, so drift still shows. + output_latency_ns: 0, + video_e2e_ns: (video_e2e_ns > 0).then_some(video_e2e_ns), + }; + if let Some(ms) = raw_offset_ms(&o, self.channels) { + let w = &mut r.window; + w.av_sum_ms += ms; + w.av_n += 1; + w.av_min_ms = Some(w.av_min_ms.map_or(ms, |m| m.min(ms))); + w.av_max_ms = Some(w.av_max_ms.map_or(ms, |m| m.max(ms))); + } + r.sync.observe(o); + cells + .av_offset_ms + .store(i64::from(r.sync.offset_ms()), Ordering::Relaxed); + r.samples.extend(pcm.iter().copied()); drop(ring); - self.shared.fed.notify_one(); + shared.fed.notify_one(); Ok(()) } @@ -140,8 +247,9 @@ impl AudioSink for AudioOutSink { fn write_loop(shared: &Shared, mut port: Box, channels: usize) { let want = port.grain_frames() * channels; let mut grain = vec![0f32; want]; + let mut since = Instant::now(); while !shared.stop.load(Ordering::Relaxed) { - { + let window = { let mut ring = shared.ring.lock().expect("audio ring poisoned"); if ring.samples.is_empty() && !ring.policy.is_primed() { // Nothing to play yet: wait for the decoder rather than spinning on silence. @@ -152,18 +260,71 @@ fn write_loop(shared: &Shared, mut port: Box, channels: usize) { .0; } fill(&mut ring, &mut grain, &shared.underruns); - } + shared.cells.buffer_ms.store(ring.depth_ms, Ordering::Relaxed); + (since.elapsed() >= WINDOW).then(|| { + let (target_ms, offset_ms) = (ring.policy.target_ms(), ring.sync.offset_ms()); + (std::mem::take(&mut ring.window), target_ms, offset_ms) + }) + }; if let Err(e) = port.write(&grain) { tracing::warn!("audio out: {e:#}"); std::thread::sleep(Duration::from_millis(5)); } + shared.port_queued.store(port.queued_frames(), Ordering::Relaxed); + // Off the lock and after the grain is queued, so the port has its queue to play. + if let Some((w, target_ms, offset_ms)) = window { + log_window(&w, since.elapsed(), target_ms, offset_ms, port.queued_frames()); + since = Instant::now(); + } } } +/// Where the audio in `o` will play against its picture, in ms (positive is audio late): the +/// arithmetic of [`AvSync::observe`] without the smoothing, for the window's spread. `None` +/// before a picture reached the glass. +fn raw_offset_ms(o: &AvSyncObservation, channels: u8) -> Option { + let video = i128::from(o.video_e2e_ns?); + let per_ms = i128::from(crate::session::audio::SAMPLE_RATE / 1000) * i128::from(channels.max(1)); + let buffered_ns = o.buffered_ahead as i128 * 1_000_000 / per_ms; + let audio = o.now_local_ns + buffered_ns + i128::from(o.clock_offset_ns) - i128::from(o.pts_ns); + i64::try_from((audio - video) / 1_000_000).ok() +} + +fn log_window(w: &Window, took: Duration, target_ms: u32, offset_ms: i32, queued_frames: usize) { + let av = match (w.av_min_ms, w.av_max_ms) { + (Some(min), Some(max)) => format!( + "a/v {offset_ms:+} ms smoothed, {:+} mean, {min:+} to {max:+} over {} packets", + w.av_sum_ms / i64::from(w.av_n.max(1)), + w.av_n + ), + _ => "a/v not measured, no picture on the glass".to_string(), + }; + tracing::info!( + "audio[{:.0}s]: {} grains, {} priming; underruns {} silent, {} audible; {} sheds, {} inserts; depth {}-{} ms, target {target_ms} ms, device queue {} ms; {} packets, {} pauses over {} ms, longest gap {} ms; {av}", + took.as_secs_f64(), + w.grains, + w.priming, + w.silent_underruns, + w.audible_underruns, + w.sheds, + w.inserts, + w.depth_min_ms.unwrap_or(0), + w.depth_max_ms, + queued_frames / (crate::session::audio::SAMPLE_RATE / 1000) as usize, + w.packets, + w.pauses, + PAUSE.as_millis(), + w.longest_gap.as_millis(), + ); +} + /// One policy step and one read into `grain`. fn fill(ring: &mut Ring, grain: &mut [f32], underruns: &AtomicU64) { let want = grain.len(); let step = ring.policy.step(ring.samples.len(), want); + ring.window.grains += 1; + ring.window.sheds += u32::from(step.drop_front > 0); + ring.window.inserts += u32::from(step.insert_front > 0); if step.drop_front > 0 { crossfade_drop(&mut ring.samples, step.drop_front, step.crossfade); } @@ -172,6 +333,7 @@ fn fill(ring: &mut Ring, grain: &mut [f32], underruns: &AtomicU64) { } if step.silence { grain.fill(0.0); + ring.window.priming += 1; ring.policy.note_read(false); } else { let take = want.min(ring.samples.len()); @@ -182,10 +344,18 @@ fn fill(ring: &mut Ring, grain: &mut [f32], underruns: &AtomicU64) { let short = take < want; if short { underruns.fetch_add(1, Ordering::Relaxed); + if ring.last_peak < SILENT_PEAK { + ring.window.silent_underruns += 1; + } else { + ring.window.audible_underruns += 1; + } } ring.policy.note_read(short); } ring.depth_ms = ring.policy.depth_ms(ring.samples.len()); + let (depth, w) = (ring.depth_ms, &mut ring.window); + w.depth_min_ms = Some(w.depth_min_ms.map_or(depth, |m| m.min(depth))); + w.depth_max_ms = w.depth_max_ms.max(depth); } #[cfg(test)] @@ -198,7 +368,7 @@ mod tests { #[test] fn fed_audio_reaches_the_port_in_grains() { let audio = FakeAudio::default(); - let sink = AudioOutSink::start(audio.open(2, 48_000).unwrap(), 2).unwrap(); + let sink = AudioOutSink::start(audio.open(2, 48_000).unwrap(), 2, SyncCells::default()).unwrap(); let tone = vec![0.5f32; 240 * 2]; for _ in 0..40 { sink.feed(Samples::F32(&tone), 0).unwrap(); diff --git a/crates/pf-nx-app/src/session/pipeline.rs b/crates/pf-nx-app/src/session/pipeline.rs index 32e0c94..4045cef 100644 --- a/crates/pf-nx-app/src/session/pipeline.rs +++ b/crates/pf-nx-app/src/session/pipeline.rs @@ -14,7 +14,7 @@ use crate::core::media::{AudioSink, VideoSink}; use crate::platform::{AudioOut, StreamSpec, VideoDecoders}; use crate::services::join::{join_with_timeout, SHUTDOWN_JOIN_TIMEOUT}; use crate::session::audio::AudioStage; -use crate::session::audio_out::AudioOutSink; +use crate::session::audio_out::{AudioOutSink, SyncCells}; use crate::session::connect::ConnectParams; use crate::session::pump::{spawn_audio_feed, video_pump}; use crate::session::stage::{SinkConfig, VideoStage}; @@ -54,7 +54,8 @@ impl MediaPipeline { .audio .open(client.audio_channels, crate::session::audio::SAMPLE_RATE) .context("open the audio port")?; - let audio_sink: Arc = Arc::new(AudioOutSink::start(port, client.audio_channels)?); + let audio_sink: Arc = + Arc::new(AudioOutSink::start(port, client.audio_channels, SyncCells::of(client))?); let stage = AudioStage::new(audio_sink, client.audio_channels, client.audio_layout)?; let video_thread = spawn_video_thread(client, sink, first, opener, stop, stats, params.present_priority)?; let audio_thread = match spawn_audio_feed(client.clone(), stage, stop.clone()) { @@ -143,6 +144,7 @@ fn spec_of(client: &NativeClient, smoothing: usize) -> StreamSpec { shard_payload: client.shard_payload as usize, smoothing, hud: client.hud_shared(), + video_e2e: crate::platform::VideoE2e::new(client.video_e2e_shared(), client.clock_offset_shared()), } } diff --git a/crates/pf-nx-platform/src/console/audio.rs b/crates/pf-nx-platform/src/console/audio.rs index 08b1bee..edaa1f4 100644 --- a/crates/pf-nx-platform/src/console/audio.rs +++ b/crates/pf-nx-platform/src/console/audio.rs @@ -51,4 +51,8 @@ impl AudioPort for Port { } self.out.push(&self.pcm) } + + fn queued_frames(&self) -> usize { + self.out.queued_frames() + } } diff --git a/crates/pf-nx-platform/src/console/lifecycle.rs b/crates/pf-nx-platform/src/console/lifecycle.rs index 4f4e0d1..2d24c54 100644 --- a/crates/pf-nx-platform/src/console/lifecycle.rs +++ b/crates/pf-nx-platform/src/console/lifecycle.rs @@ -7,9 +7,9 @@ use std::sync::mpsc; use std::time::{Duration, Instant}; -use pf_nx_app::platform::{Lifecycle, SystemEvent}; +use pf_nx_app::platform::{Clocks, Lifecycle, SystemEvent}; use pf_nx_sys::applet::{self, FocusState, HookEvent}; -use pf_nx_sys::{mem, rt, socket}; +use pf_nx_sys::{clk, mem, rt, socket}; pub struct NxLifecycle { events: mpsc::Receiver, @@ -46,7 +46,13 @@ impl Lifecycle for NxLifecycle { self.exit(); } if !std::mem::replace(&mut self.reported, true) { - tracing::info!("lifecycle: {:?}, {:?}", applet::applet_type(), applet::operation_mode()); + tracing::info!( + "lifecycle: {:?}, {:?}, clocks {}", + applet::applet_type(), + applet::operation_mode(), + self.clocks() + .map_or_else(|| "unreadable".to_string(), |c| c.to_string()) + ); } // `appletMainLoop` in its halves, so every message is logged: no hook event and no // operation-mode change has reached a log yet, the dock change included. @@ -68,11 +74,25 @@ impl Lifecycle for NxLifecycle { }), HookEvent::Resume => out.push(SystemEvent::Resume), HookEvent::OperationMode => out.push(SystemEvent::OperationMode(super::operation_mode())), - HookEvent::Exit | HookEvent::PerformanceMode => {} + // The system moved the title to another clock configuration (WP5's question). + HookEvent::PerformanceMode => tracing::info!( + "lifecycle: performance mode changed, clocks {}", + self.clocks() + .map_or_else(|| "unreadable".to_string(), |c| c.to_string()) + ), + HookEvent::Exit => {} } } } + fn clocks(&self) -> Option { + clk::clocks().map(|c| Clocks { + cpu_hz: c.cpu_hz, + gpu_hz: c.gpu_hz, + emc_hz: c.emc_hz, + }) + } + /// Returns the socket driver's lent heap, closes the detached threads' stacks, and leaves /// through libnx's `__appExit`: back to hbl when the heap is clean, else through AM to HOME. fn exit(&self) -> ! { diff --git a/crates/pf-nx-platform/src/video.rs b/crates/pf-nx-platform/src/video.rs index 8588899..9992ff3 100644 --- a/crates/pf-nx-platform/src/video.rs +++ b/crates/pf-nx-platform/src/video.rs @@ -4,7 +4,9 @@ //! //! `feed` decodes on the session's video thread: send the access unit, then receive every //! picture straight into the presenter's next staging buffer, which `commit` copies into a -//! plane set and publishes (D7). On the framebuffer display the pictures land in scratch planes +//! plane set and publishes (D7). On the console the VIC writes into that staging buffer +//! itself where it can map it (WP5); else the picture goes through libavutil's own frame and +//! the shim copies it, and the `nvdec[10s]` line names which route ran. On the framebuffer display the pictures land in scratch planes //! and are dropped, so the emulator still runs the hwaccel. A decoder error fails this feed, //! which has the session ask for a keyframe, and the next feed starts from a flushed decoder. @@ -120,6 +122,14 @@ impl Out { } } + /// The GPU time of the last staging copy into a plane set, where the GPU can time it. + fn copy_gpu_us(&self) -> Option { + match self { + Self::Gpu(upload) => upload.copy_gpu_us(), + Self::Scratch { .. } => None, + } + } + /// `false` when the picture was dropped because every plane set is on screen. fn commit(&mut self, meta: av::PictureMeta, due: Option) -> Result { match self { @@ -182,7 +192,9 @@ impl Lane { let due = self.due_of(meta.pts); let started = Instant::now(); let shown = self.out.commit(meta, due)?; - self.timing.note(&self.decoder.stats(), wait, started.elapsed(), shown); + let upload = started.elapsed(); + self.timing + .note(&self.decoder.stats(), wait, upload, self.out.copy_gpu_us(), shown); Ok(true) } @@ -199,7 +211,9 @@ impl Lane { } /// Where the lane's time went over one [`TIMING_WINDOW`]: the wait for a free staging buffer, -/// the decoder's own time, the VIC transfer into the staging planes, and the upload's submit. +/// the decoder's own time, the transfer to NV12, the shim's copy out of libavutil's frame (none +/// when the transfer wrote the staging planes itself), the upload's submit, and the staging +/// copy's time on the GPU. struct Timing { since: Instant, pictures: u32, @@ -208,10 +222,16 @@ struct Timing { decode_max_us: u32, transfer_us: u64, transfer_max_us: u32, + copy_us: u64, + copy_max_us: u32, wait: Duration, wait_max: Duration, upload: Duration, + gpu_copy_us: u64, + gpu_copies: u32, in_flight_max: u32, + /// How the last picture reached the staging planes; `None` before the first. + route: Option, } impl Timing { @@ -224,14 +244,40 @@ impl Timing { decode_max_us: 0, transfer_us: 0, transfer_max_us: 0, + copy_us: 0, + copy_max_us: 0, wait: Duration::ZERO, wait_max: Duration::ZERO, upload: Duration::ZERO, + gpu_copy_us: 0, + gpu_copies: 0, in_flight_max: 0, + route: None, } } - fn note(&mut self, s: &av::Stats, wait: Duration, upload: Duration, shown: bool) { + fn note(&mut self, s: &av::Stats, wait: Duration, upload: Duration, gpu_copy_us: Option, shown: bool) { + // The first route, and any change, at once: a change mid-stream is the straight route + // falling back. + if self.route != Some(s.route) { + tracing::info!( + "nvdec: pictures reach the staging planes by {}{}{}", + s.route.name(), + self.route.map_or_else(String::new, |r| format!(" (was {})", r.name())), + if s.direct_err == 0 { + String::new() + } else { + format!("; the straight route failed with {}", s.direct_err) + } + ); + } + self.route = Some(s.route); + self.copy_us += u64::from(s.last_copy_us); + self.copy_max_us = self.copy_max_us.max(s.last_copy_us); + if let Some(us) = gpu_copy_us { + self.gpu_copy_us += u64::from(us); + self.gpu_copies += 1; + } self.pictures += 1; self.dropped += u32::from(!shown); self.decode_us += u64::from(s.last_decode_us); @@ -248,8 +294,12 @@ impl Timing { let n = u64::from(self.pictures); let ms = |us: u64| us as f64 / 1000.0; let per = |d: Duration| d.as_secs_f64() * 1000.0 / n as f64; + let gpu_copy = self + .gpu_copy_us + .checked_div(u64::from(self.gpu_copies)) + .map_or_else(|| "-".to_string(), |us| format!("{:.2}", us as f64 / 1000.0)); tracing::info!( - "nvdec[{}s]: {} pictures ({} dropped, {} decoded in all); staging wait mean {:.1}ms max {:.1}ms; decode mean {:.1}ms max {:.1}ms; transfer mean {:.1}ms max {:.1}ms; upload mean {:.2}ms; in flight max {}", + "nvdec[{}s]: {} pictures ({} dropped, {} decoded in all); staging wait mean {:.1}ms max {:.1}ms; decode mean {:.1}ms max {:.1}ms; transfer mean {:.1}ms max {:.1}ms; copy mean {:.1}ms max {:.1}ms; upload mean {:.2}ms; gpu copy mean {gpu_copy}ms; in flight max {}; route {}", TIMING_WINDOW.as_secs(), self.pictures, self.dropped, @@ -260,10 +310,15 @@ impl Timing { ms(u64::from(self.decode_max_us)), ms(self.transfer_us / n), ms(u64::from(self.transfer_max_us)), + ms(self.copy_us / n), + ms(u64::from(self.copy_max_us)), per(self.upload), self.in_flight_max, + self.route.map_or("-", av::Route::name), ); + let route = self.route; *self = Self::new(); + self.route = route; } } diff --git a/crates/pf-nx-sys/csrc/pf_av.c b/crates/pf-nx-sys/csrc/pf_av.c index 2b32f41..5434a9f 100644 --- a/crates/pf-nx-sys/csrc/pf_av.c +++ b/crates/pf-nx-sys/csrc/pf_av.c @@ -1,5 +1,7 @@ // NVDEC through libavcodec's nvtegra hwaccel, as six calls the Rust side binds (src/av.rs). -// One thread uses a decoder at a time; pictures leave as pitch-linear NV12 via the VIC. +// One thread uses a decoder at a time; pictures leave as pitch-linear NV12 via the VIC, +// straight into the caller's planes where the hwcontext can map them, else through +// libavutil's own frame and a copy. // Built for a Linux dev box (no __SWITCH__), the same calls decode in software against the // system's libavcodec and interleave its 4:2:0 chroma into the same NV12 planes. #include @@ -14,9 +16,23 @@ #define PF_AV_NOT_NV12 (-0x70660001) #define PF_AV_TOO_SMALL (-0x70660002) +// How the last picture reached the caller's planes (src/av.rs `Route`). +enum { + PF_AV_ROUTE_SOFTWARE = 0, + // The hwcontext's transfer into libavutil's frame, then the shim's copy. + PF_AV_ROUTE_VIC_THEN_COPY = 1, + PF_AV_ROUTE_CPU_THEN_COPY = 2, + // The hwcontext's transfer straight into the caller's planes. + PF_AV_ROUTE_VIC = 3, + PF_AV_ROUTE_CPU = 4, +}; + typedef struct PfAvStats { uint64_t decoded; - uint32_t in_flight, last_decode_us, last_transfer_us, reserved; + uint32_t in_flight, last_decode_us, last_transfer_us, last_copy_us; + uint32_t route; + // Why transfers stopped going straight into the caller's planes; 0 while they do. + int32_t direct_err; } PfAvStats; typedef struct PfAvPictureMeta { @@ -30,6 +46,8 @@ typedef struct PfAvDecoder { AVPacket *pkt; AVFrame *hw; AVFrame *sw; + // The caller's planes as the transfer's destination; NULL off the console. + AVFrame *direct; int64_t sent_at; uint64_t taken; PfAvStats stats; @@ -42,6 +60,7 @@ void pf_av_close(PfAvDecoder *d) { av_packet_free(&d->pkt); av_frame_free(&d->hw); av_frame_free(&d->sw); + av_frame_free(&d->direct); av_free(d); } @@ -70,7 +89,7 @@ int pf_av_open(int codec, int width, int height, PfAvDecoder **out) { if (!d->ctx || !d->pkt || !d->hw || !d->sw) goto fail; #ifdef __SWITCH__ - if (!(d->ctx->hw_device_ctx = av_buffer_ref(d->device))) + if (!(d->ctx->hw_device_ctx = av_buffer_ref(d->device)) || !(d->direct = av_frame_alloc())) goto fail; #endif // The default get_format picks the device's hw format when hw_device_ctx is set. @@ -103,6 +122,46 @@ int pf_av_send(PfAvDecoder *d, const uint8_t *data, int len, uint64_t pts) { return 0; } +// The caller's memory as a buffer the frame refers to; the caller owns it. +static void keep(void *opaque, uint8_t *data) { + (void)opaque; + (void)data; +} + +// The hwcontext maps the destination's buffers into the VIC per transfer and writes the +// picture there: the copy out of libavutil's frame goes away. It needs 256-byte aligned +// addresses and pitches, else it detiles on the CPU, still into these planes. `rows` is the +// picture's height rounded up to even, which is what either path writes. +static int transfer_into(PfAvDecoder *d, uint8_t *y, size_t y_stride, size_t y_len, uint8_t *uv, + size_t uv_stride, size_t uv_len) { + AVFrame *f = d->direct; + f->format = AV_PIX_FMT_NV12; + f->width = d->hw->width; + f->height = d->hw->height; + f->data[0] = y; + f->data[1] = uv; + f->linesize[0] = (int)y_stride; + f->linesize[1] = (int)uv_stride; + // One mapping when the planes are one block, as the staging buffer lays them out. + int err = AVERROR(ENOMEM); + if (uv == y + y_len) { + f->buf[0] = av_buffer_create(y, y_len + uv_len, keep, NULL, 0); + if (!f->buf[0]) goto out; + } else { + f->buf[0] = av_buffer_create(y, y_len, keep, NULL, 0); + f->buf[1] = av_buffer_create(uv, uv_len, keep, NULL, 0); + if (!f->buf[0] || !f->buf[1]) goto out; + } + err = av_hwframe_transfer_data(f, d->hw, 0); +out: + av_frame_unref(f); + return err; +} + +static int aligned_256(const void *p, size_t stride) { + return (((uintptr_t)p | stride) & 0xff) == 0; +} + static void copy_plane(uint8_t *dst, size_t dst_stride, const uint8_t *src, int src_stride, size_t row, size_t rows) { for (size_t i = 0; i < rows; i++) @@ -136,13 +195,36 @@ int pf_av_receive(PfAvDecoder *d, uint8_t *y, size_t y_stride, size_t y_len, uin meta->height = (uint32_t)d->hw->height; meta->pts = (uint64_t)d->hw->pts; AVFrame *src = d->hw; + d->stats.last_copy_us = 0; + d->stats.route = PF_AV_ROUTE_SOFTWARE; if (d->hw->hw_frames_ctx) { + size_t rows = ((size_t)d->hw->height + 1) & ~(size_t)1; + size_t w = ((size_t)d->hw->width + 1) & ~(size_t)1; + if (d->direct && d->stats.direct_err == 0 && y_stride >= w && uv_stride >= w && + y_len >= y_stride * rows && uv_len >= uv_stride * rows / 2) { + int64_t t = av_gettime_relative(); + err = transfer_into(d, y, y_stride, y_len, uv, uv_stride, uv_len); + d->stats.last_transfer_us = (uint32_t)(av_gettime_relative() - t); + if (err >= 0) { + d->stats.route = aligned_256(y, y_stride) && aligned_256(uv, uv_stride) + ? PF_AV_ROUTE_VIC + : PF_AV_ROUTE_CPU; + err = 1; + goto out; + } + // Never again for this decoder: the route through libavutil's frame is proven. + d->stats.direct_err = err < 0 ? err : -1; + } int64_t t = av_gettime_relative(); d->sw->format = AV_PIX_FMT_NV12; // TODO(verify): the VIC transfer offers NV12 (WP0.6). err = av_hwframe_transfer_data(d->sw, d->hw, 0); d->stats.last_transfer_us = (uint32_t)(av_gettime_relative() - t); if (err < 0) goto out; src = d->sw; + d->stats.route = aligned_256(src->data[0], (size_t)src->linesize[0]) && + aligned_256(src->data[1], (size_t)src->linesize[1]) + ? PF_AV_ROUTE_VIC_THEN_COPY + : PF_AV_ROUTE_CPU_THEN_COPY; } // On the console a software frame means the hwaccel did not take the stream. #ifdef __SWITCH__ @@ -165,6 +247,7 @@ int pf_av_receive(PfAvDecoder *d, uint8_t *y, size_t y_stride, size_t y_len, uin d->stats.last_transfer_us = (uint32_t)(av_gettime_relative() - t); } else { copy_plane(uv, uv_stride, src->data[1], src->linesize[1], uw, uh); + d->stats.last_copy_us = (uint32_t)(av_gettime_relative() - t); } err = 1; out: diff --git a/crates/pf-nx-sys/src/av.rs b/crates/pf-nx-sys/src/av.rs index bcf5941..5370f4e 100644 --- a/crates/pf-nx-sys/src/av.rs +++ b/crates/pf-nx-sys/src/av.rs @@ -36,8 +36,54 @@ pub struct Stats { pub in_flight: u32, /// Microseconds from the last send to the last picture out. pub last_decode_us: u32, - /// Microseconds of the last VIC transfer to NV12. + /// Microseconds of the last transfer to NV12 (the VIC's, or the CPU's detiling). pub last_transfer_us: u32, + /// Microseconds of the last copy out of libavutil's frame; 0 when there was none. + pub last_copy_us: u32, + pub route: Route, + /// Why transfers stopped going straight into the caller's planes, an `AVERROR`; 0 while + /// they do. + pub direct_err: i32, +} + +/// How the last picture reached the caller's planes (`PF_AV_ROUTE_*`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Route { + /// A software decoder's picture, copied (the dev box). + #[default] + Software, + /// The VIC into libavutil's frame, then copied. + VicThenCopy, + /// libavutil's frame was not 256-byte aligned, so the hwcontext detiled on the CPU, then + /// copied. + CpuThenCopy, + /// The VIC straight into the caller's planes. + Vic, + /// Detiled on the CPU straight into the caller's planes. + Cpu, +} + +impl Route { + fn from_raw(raw: u32) -> Self { + match raw { + 1 => Self::VicThenCopy, + 2 => Self::CpuThenCopy, + 3 => Self::Vic, + 4 => Self::Cpu, + _ => Self::Software, + } + } + + /// For the log. + pub fn name(self) -> &'static str { + match self { + Self::Software => "software, copied", + Self::VicThenCopy => "VIC, copied", + Self::CpuThenCopy => "CPU detile, copied", + Self::Vic => "VIC straight", + Self::Cpu => "CPU detile straight", + } + } } /// `PfAvStats`. @@ -48,7 +94,9 @@ pub struct RawStats { pub in_flight: u32, pub last_decode_us: u32, pub last_transfer_us: u32, - pub reserved: u32, + pub last_copy_us: u32, + pub route: u32, + pub direct_err: i32, } /// The shim's own codes, outside libavcodec's `AVERROR` space (`csrc/pf_av.c`). @@ -196,6 +244,9 @@ impl Decoder { in_flight: s.in_flight, last_decode_us: s.last_decode_us, last_transfer_us: s.last_transfer_us, + last_copy_us: s.last_copy_us, + route: Route::from_raw(s.route), + direct_err: s.direct_err, } } } @@ -266,10 +317,14 @@ mod tests { use core::mem::{offset_of, size_of}; // PfAvPictureMeta: u32 width, u32 height, u64 pts → 16. assert_eq!((size_of::(), offset_of!(PictureMeta, pts)), (16, 8)); - // PfAvStats: u64 decoded, four u32 → 24. + // PfAvStats: u64 decoded, five u32 and an i32 → 32. assert_eq!( - (size_of::(), offset_of!(RawStats, last_transfer_us)), - (24, 16) + ( + size_of::(), + offset_of!(RawStats, last_transfer_us), + offset_of!(RawStats, direct_err) + ), + (32, 16, 28) ); assert_eq!(Codec::Hevc as i32, 1); } diff --git a/crates/pf-nx-sys/tests/layout.rs b/crates/pf-nx-sys/tests/layout.rs index 2dd9d12..703e94f 100644 --- a/crates/pf-nx-sys/tests/layout.rs +++ b/crates/pf-nx-sys/tests/layout.rs @@ -117,8 +117,8 @@ fn firmware_version() { #[test] fn shim_structs() { - // PfAvPictureMeta: u32, u32, u64 → 16. PfAvStats: u64, 4 × u32 → 24. + // PfAvPictureMeta: u32, u32, u64 → 16. PfAvStats: u64, 6 × 32-bit → 32. assert_eq!(layout::(), (16, 8)); - assert_eq!(layout::(), (24, 8)); + assert_eq!(layout::(), (32, 8)); assert_eq!(offset_of!(RawStats, in_flight), 8); } diff --git a/crates/pf-nx-vk/src/lib.rs b/crates/pf-nx-vk/src/lib.rs index b77a6e7..59a77eb 100644 --- a/crates/pf-nx-vk/src/lib.rs +++ b/crates/pf-nx-vk/src/lib.rs @@ -13,6 +13,7 @@ pub mod pipelines; pub mod presenter; #[cfg(feature = "pyrowave")] pub mod pyro; +mod timer; pub mod ui; pub mod util; diff --git a/crates/pf-nx-vk/src/picture.rs b/crates/pf-nx-vk/src/picture.rs index 4496b26..9bc2240 100644 --- a/crates/pf-nx-vk/src/picture.rs +++ b/crates/pf-nx-vk/src/picture.rs @@ -17,6 +17,7 @@ use pf_client_video::video_color::ColorDesc; use pf_nx_app::platform::StreamSpec; use crate::device::GpuDevice; +use crate::timer::GpuTimer; /// Plane views in the layout the lane leaves them (`layout`). #[derive(Clone, Copy, Debug)] @@ -50,6 +51,8 @@ pub struct Picture { pub turn: Option, /// The session's overlay figures, where the display stage is noted. pub hud: Option>, + /// Where the presenter leaves the capture-to-glass time for the audio's A/V sync. + pub e2e: Option, /// Keeps the planes' images and memory alive. pub hold: Box, } @@ -250,10 +253,15 @@ pub struct Nv12Upload { stride: usize, color: ColorDesc, hud: Arc, + e2e: pf_nx_app::platform::VideoE2e, planes: Vec>, pool: vk::CommandPool, staging: Vec, next: usize, + /// Each staging buffer's copy on the GPU; `None` where the queue has no timestamps. + timer: Option, + /// The last copy's GPU time that [`Self::stage`] read back. + copy_gpu_us: Option, } // SAFETY: the mapped pointers address memory this upload owns for its whole life and only @@ -290,10 +298,13 @@ impl Nv12Upload { full_range: spec.color.full_range != 0, }, hud: spec.hud.clone(), + e2e: spec.video_e2e.clone(), planes: Vec::new(), pool, staging: Vec::with_capacity(COPIES), next: 0, + timer: None, + copy_gpu_us: None, }; let dev = &upload.dev; // SAFETY: the pool is live; the buffers are freed with it. @@ -325,10 +336,12 @@ impl Nv12Upload { in_flight: false, }); } + upload.timer = GpuTimer::new(&upload.dev, COPIES); tracing::info!( - "upload: NV12 {}x{} through {COPIES} staging buffers of {bytes} bytes, row {stride}", + "upload: NV12 {}x{} through {COPIES} staging buffers of {bytes} bytes, row {stride}, at {:p}", extent.width, - extent.height + extent.height, + upload.staging[0].ptr, ); upload.slot.set_smoothing(spec.smoothing); if spec.smoothing == 0 { @@ -409,6 +422,9 @@ impl Nv12Upload { s.cmd, &vk::CommandBufferBeginInfo::default().flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), )?; + if let Some(t) = &mut self.timer { + t.begin(dev, s.cmd, i); + } for image in planes.images { // After the presenter's last read and any earlier copy into this set, which a // replaced picture can leave in flight. @@ -450,6 +466,9 @@ impl Nv12Upload { (vk::PipelineStageFlags2::FRAGMENT_SHADER, vk::AccessFlags2::SHADER_READ), ); } + if let Some(t) = &self.timer { + t.end(dev, s.cmd, i); + } dev.device.end_command_buffer(s.cmd)?; let cmds = [s.cmd]; dev.vkd.queue_lock.yield_to_present(); @@ -476,6 +495,7 @@ impl Nv12Upload { decoded_ns: u64::try_from(punktfunk_core::client::now_realtime_ns()).unwrap_or(0), turn: None, hud: Some(self.hud.clone()), + e2e: Some(self.e2e.clone()), hold: Box::new(planes), }); Ok(true) @@ -493,10 +513,16 @@ impl Nv12Upload { .context("staging copy")?; self.dev.device.reset_fences(&[s.fence])?; } + self.copy_gpu_us = self.timer.as_ref().and_then(|t| t.read(&self.dev, i)); } Ok(()) } + /// The GPU time of the last staging copy that has finished, where the GPU can time it. + pub fn copy_gpu_us(&self) -> Option { + self.copy_gpu_us + } + /// A plane set nothing else holds, or a new one while there are fewer than [`PLANE_SETS`]. fn free_planes(&mut self) -> Result>> { if let Some(p) = self.planes.iter().find(|p| Arc::strong_count(p) == 1) { @@ -544,6 +570,9 @@ impl Drop for Nv12Upload { } // SAFETY: the device is idle and every handle below is this upload's. unsafe { + if let Some(t) = &self.timer { + t.destroy(&self.dev); + } for s in self.staging.drain(..) { self.dev.device.destroy_fence(s.fence, None); self.dev.device.destroy_buffer(s.buffer, None); diff --git a/crates/pf-nx-vk/src/presenter.rs b/crates/pf-nx-vk/src/presenter.rs index c4af318..f2a4f4b 100644 --- a/crates/pf-nx-vk/src/presenter.rs +++ b/crates/pf-nx-vk/src/presenter.rs @@ -19,6 +19,7 @@ use crate::device::{GpuDevice, Target}; use crate::output::Output; use crate::picture::{Picture, Planes, VideoSlot}; use crate::pipelines::Pipelines; +use crate::timer::GpuTimer; use crate::ui::{UiLayer, FRAMES}; /// Tonemap source peak for a PQ stream on this SDR output, in 203-nit units (~1000 nits). @@ -68,6 +69,8 @@ pub struct Presenter { want: vk::Extent2D, /// The swapchain no longer matches the window: rebuild before the next acquire. stale: bool, + /// Each frame's command buffer on the GPU; `None` where the queue has no timestamps. + timer: Option, } // SAFETY: the read-back pointer maps memory this struct owns; the presenter lives on the main @@ -143,6 +146,10 @@ impl Presenter { } let readback_sets = alloc_sets(&dev, desc_pool, &pipes)?; let ui = UiLayer::new(&dev, output.extent)?; + let timer = GpuTimer::new(&dev, FRAMES); + if timer.is_none() { + tracing::info!("presenter: the queue has no timestamps, so no GPU time per frame"); + } // The presenter runs on the main thread, decode on the pump: take present turns so a // picture handed over is presented ahead of the next decode on the shared queue. dev.vkd.queue_lock.take_turns(); @@ -166,6 +173,7 @@ impl Presenter { readback_sets, want, stale: false, + timer, }) } @@ -456,6 +464,7 @@ impl Display for Presenter { unsafe { self.dev.device.wait_for_fences(&[fence], true, u64::MAX) } .map_err(|e| anyhow!("wait_for_fences: {e}"))?; self.frames[f].in_use = None; + let gpu_us = self.timer.as_ref().and_then(|t| t.read(&self.dev, f)); if self.stale { self.rebuild()?; } @@ -464,7 +473,10 @@ impl Display for Presenter { // The acquire blocks until the panel frees an image, just after a vblank: the picture is // taken after it, so one that lands during the wait still makes this frame. let index = self.acquire(ready)?; - let mut report = FrameReport::default(); + let mut report = FrameReport { + gpu_us, + ..FrameReport::default() + }; if self.stream.is_some() { // What is presented now latches at the next vblank, a refresh from here. let latch = Instant::now() + Duration::from_nanos(self.refresh().period_ns()); @@ -493,8 +505,14 @@ impl Display for Presenter { cmd, &vk::CommandBufferBeginInfo::default().flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT), )?; + if let Some(t) = &mut self.timer { + t.begin(&self.dev, cmd, f); + } self.ui.finish(&self.dev, cmd, f); self.compose(cmd, self.pipes.pass_out, self.framebuffers[index as usize], sets); + if let Some(t) = &self.timer { + t.end(&self.dev, cmd, f); + } self.dev.device.end_command_buffer(cmd)?; } let presents = self.output.presents(); @@ -622,12 +640,16 @@ impl Display for Presenter { } } -/// The overlay's display stage: decoded → handed to scan-out, on the session's HUD. +/// The overlay's display stage: decoded → handed to scan-out, on the session's HUD, and the +/// capture-to-glass time the audio's A/V sync compares against. fn note_displayed(picture: &Picture) { + let now = u64::try_from(punktfunk_core::client::now_realtime_ns()).unwrap_or(0); if let Some(hud) = &picture.hud { - let now = u64::try_from(punktfunk_core::client::now_realtime_ns()).unwrap_or(0); hud.note_displayed(picture.pts_ns, picture.decoded_ns, now, now); } + if let Some(e2e) = &picture.e2e { + e2e.note(picture.pts_ns, now); + } } impl Drop for Presenter { @@ -644,6 +666,9 @@ impl Drop for Presenter { destroy_readback(dev, &rb); } self.ui.destroy(dev); + if let Some(t) = &self.timer { + t.destroy(dev); + } for fr in self.frames.drain(..) { dev.device.destroy_fence(fr.fence, None); dev.device.destroy_semaphore(fr.ready, None); diff --git a/crates/pf-nx-vk/src/pyro.rs b/crates/pf-nx-vk/src/pyro.rs index 92742e3..77e8bb6 100644 --- a/crates/pf-nx-vk/src/pyro.rs +++ b/crates/pf-nx-vk/src/pyro.rs @@ -70,6 +70,7 @@ impl PyroLane { decoder: Mutex::new(decoder), slot: self.slot.clone(), hud: spec.hud.clone(), + e2e: spec.video_e2e.clone(), started: Instant::now(), })) } @@ -79,6 +80,7 @@ struct PyroSink { decoder: Mutex, slot: Arc, hud: Arc, + e2e: pf_nx_app::platform::VideoE2e, started: Instant, } @@ -118,6 +120,7 @@ impl VideoSink for PyroSink { decoded_ns: u64::try_from(punktfunk_core::client::now_realtime_ns()).unwrap_or(0), turn: Some(frame.turn), hud: Some(self.hud.clone()), + e2e: Some(self.e2e.clone()), hold: Box::new(frame.hold), }); Ok(()) diff --git a/crates/pf-nx-vk/src/timer.rs b/crates/pf-nx-vk/src/timer.rs new file mode 100644 index 0000000..f447f2b --- /dev/null +++ b/crates/pf-nx-vk/src/timer.rs @@ -0,0 +1,102 @@ +//! How long a command buffer ran on the GPU, from a timestamp at its start and one at its end: +//! the margin the GPU clock leaves (plan WP5). One pair per command buffer slot, read once that +//! slot's fence has been waited, so a read never stalls. A queue without timestamps reads +//! nothing and records nothing. + +use ash::vk; + +use crate::device::GpuDevice; + +pub(crate) struct GpuTimer { + pool: vk::QueryPool, + /// Nanoseconds per tick. + period_ns: f64, + /// The bits a timestamp carries on this queue. + mask: u64, + /// Whether each slot's pair has been recorded since the pool was made. + recorded: Vec, +} + +impl GpuTimer { + /// Timestamps for `slots` command buffers, or `None` where the queue has none. + pub fn new(dev: &GpuDevice, slots: usize) -> Option { + // SAFETY: read-only queries on the live physical device. + let (limits, families) = unsafe { + ( + dev.instance.get_physical_device_properties(dev.pdev).limits, + dev.instance.get_physical_device_queue_family_properties(dev.pdev), + ) + }; + let bits = families.get(dev.qf as usize)?.timestamp_valid_bits; + if bits == 0 || limits.timestamp_period <= 0.0 { + return None; + } + // SAFETY: the create-info is a local; the pool is destroyed in `destroy`. + let pool = unsafe { + dev.device.create_query_pool( + &vk::QueryPoolCreateInfo::default() + .query_type(vk::QueryType::TIMESTAMP) + .query_count(slots as u32 * 2), + None, + ) + } + .ok()?; + Some(Self { + pool, + period_ns: f64::from(limits.timestamp_period), + mask: if bits >= 64 { u64::MAX } else { (1 << bits) - 1 }, + recorded: vec![false; slots], + }) + } + + /// Slot `i`'s last run in µs. Call only after its fence was waited. + pub fn read(&self, dev: &GpuDevice, i: usize) -> Option { + if !self.recorded[i] { + return None; + } + let mut ticks = [0u64; 2]; + // SAFETY: both queries were written by a submission whose fence the caller waited, so + // the results are available without waiting; `ticks` holds exactly two. + unsafe { + dev.device + .get_query_pool_results(self.pool, i as u32 * 2, &mut ticks, vk::QueryResultFlags::TYPE_64) + } + .ok()?; + let ticks = ticks[1].wrapping_sub(ticks[0]) & self.mask; + u32::try_from((ticks as f64 * self.period_ns / 1000.0) as u64).ok() + } + + /// Records slot `i`'s start. Outside any render pass, first in the command buffer. + /// + /// # Safety + /// `cmd` is recording and is slot `i`'s, whose last submission has finished. + pub unsafe fn begin(&mut self, dev: &GpuDevice, cmd: vk::CommandBuffer, i: usize) { + let q = i as u32 * 2; + // SAFETY: per the contract above. + unsafe { + dev.device.cmd_reset_query_pool(cmd, self.pool, q, 2); + dev.device + .cmd_write_timestamp(cmd, vk::PipelineStageFlags::TOP_OF_PIPE, self.pool, q); + } + self.recorded[i] = true; + } + + /// Records slot `i`'s end, after everything else the command buffer does. + /// + /// # Safety + /// As [`Self::begin`], which was recorded into the same `cmd`. + pub unsafe fn end(&self, dev: &GpuDevice, cmd: vk::CommandBuffer, i: usize) { + // SAFETY: per the contract above. + unsafe { + dev.device + .cmd_write_timestamp(cmd, vk::PipelineStageFlags::BOTTOM_OF_PIPE, self.pool, i as u32 * 2 + 1); + } + } + + /// # Safety + /// No submission that writes the pool is in flight. + pub unsafe fn destroy(&self, dev: &GpuDevice) { + // SAFETY: per the contract above. + unsafe { dev.device.destroy_query_pool(self.pool, None) }; + } +} From e11fdbd299c9f0e23048886eccf56ac3f4319ee4 Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:26:18 -0300 Subject: [PATCH 02/14] feat(stream): the dial's Keyboard opens the system keyboard The dial's Keyboard slot only logged that it does nothing here. It now shows libnx's swkbd applet over the stream and types the confirmed text on the host as committed characters (InputKind::TextInput), when the host can type text and grants the keyboard. The pads stay off the wire while the keyboard has them, as under the dial. swkbd's config block is versioned per firmware, so it stays in a small C shim (csrc/pf_swkbd.c) that pf-nx-sys now always builds on the console. The app reaches it through Lifecycle::keyboard, which reads None everywhere else. Co-Authored-By: Claude Opus 5.5 --- crates/pf-nx-app/src/platform/mod.rs | 6 +++ crates/pf-nx-app/src/runtime/stream.rs | 46 ++++++++++++++++++- .../pf-nx-platform/src/console/lifecycle.rs | 12 ++++- crates/pf-nx-sys/build.rs | 17 ++++--- crates/pf-nx-sys/csrc/pf_swkbd.c | 21 +++++++++ crates/pf-nx-sys/src/lib.rs | 1 + crates/pf-nx-sys/src/swkbd.rs | 46 +++++++++++++++++++ 7 files changed, 140 insertions(+), 9 deletions(-) create mode 100644 crates/pf-nx-sys/csrc/pf_swkbd.c create mode 100644 crates/pf-nx-sys/src/swkbd.rs diff --git a/crates/pf-nx-app/src/platform/mod.rs b/crates/pf-nx-app/src/platform/mod.rs index b105659..1401bfa 100644 --- a/crates/pf-nx-app/src/platform/mod.rs +++ b/crates/pf-nx-app/src/platform/mod.rs @@ -328,6 +328,12 @@ pub trait Lifecycle { fn clocks(&self) -> Option { None } + + /// Shows the system's keyboard over the title and blocks until the player closes it: the + /// text they confirmed, or `None` when they cancelled or there is no such keyboard. + fn keyboard(&mut self) -> Option { + None + } } /// The CPU, GPU and memory clocks, in Hz. diff --git a/crates/pf-nx-app/src/runtime/stream.rs b/crates/pf-nx-app/src/runtime/stream.rs index 9cab900..ea610d3 100644 --- a/crates/pf-nx-app/src/runtime/stream.rs +++ b/crates/pf-nx-app/src/runtime/stream.rs @@ -391,7 +391,8 @@ fn stream( ring.set_facts(&facts); } ring.tick(); - if let Some(n) = take_ring_commands(&mut ring, &mut hud, connected.client(), &mut pad_type) { + let mut keyboard = false; + if let Some(n) = take_ring_commands(&mut ring, &mut hud, connected.client(), &mut pad_type, &mut keyboard) { break n; } for cmd in ring.take_cmds() { @@ -410,6 +411,10 @@ fn stream( .to .send(if open { ToInput::DialOpened } else { ToInput::DialClosed }); } + // The dial closed itself before asking: its own close went out just above. + if keyboard { + type_text(app, connected.client(), &input); + } if connected.is_session_ended() { tracing::info!("session ended: {:?}", connected.end_reason()); break if connected.end_reason() == PunktfunkEndReason::Lost { @@ -516,6 +521,7 @@ fn take_ring_commands( hud: &mut StatsOverlay, client: &NativeClient, pad_type: &mut GamepadType, + keyboard: &mut bool, ) -> Option { while let Some(cmd) = ring.take_command() { match cmd { @@ -528,12 +534,50 @@ fn take_ring_commands( RingCommand::CyclePadType => *pad_type = pad_type.next_on_dial(), // Touch is the screen's one model so far: nothing to step to. RingCommand::CycleTouchMode => {} + // Shown once the dial's close has gone out to the input thread. + RingCommand::Keyboard => *keyboard = true, other => tracing::info!("dial: {other:?} does nothing on this console"), } } None } +/// The dial's Keyboard: the system keyboard over the stream, then its text typed on the host as +/// committed characters. The main thread waits on the keyboard, so no frame is presented +/// meanwhile; the stream and the audio go on. +fn type_text(app: &mut App, client: &NativeClient, input: &InputThread) { + use punktfunk_core::input::{InputEvent, InputKind}; + use punktfunk_core::quic::{GRANT_KEYBOARD, HOST_CAP_TEXT_INPUT}; + if client.host_caps() & HOST_CAP_TEXT_INPUT == 0 { + tracing::info!("keyboard: this host cannot type text"); + return; + } + if client.access_grants() & GRANT_KEYBOARD == 0 { + tracing::info!("keyboard: this host does not grant the keyboard"); + return; + } + // The pads stay off the wire while the keyboard has them, as under the dial. + let _ = input.to.send(ToInput::DialOpened); + let text = app.lifecycle.keyboard(); + let _ = input.to.send(ToInput::DialClosed); + // What the pads did meanwhile was typing, not the dial's. + while input.from.try_recv().is_ok() {} + let Some(text) = text.filter(|t| !t.is_empty()) else { + return; + }; + for c in text.chars() { + let _ = client.send_input(&InputEvent { + kind: InputKind::TextInput, + _pad: [0; 3], + code: u32::from(c), + x: 0, + y: 0, + flags: 0, + }); + } + tracing::info!("keyboard: typed {} characters on the host", text.chars().count()); +} + /// Copies the controller-mouse facts the dial shows from the session; whether they changed. fn follow_pad_mouse(client: &NativeClient, facts: &mut RingFacts) -> bool { let target = client.live_pads(); diff --git a/crates/pf-nx-platform/src/console/lifecycle.rs b/crates/pf-nx-platform/src/console/lifecycle.rs index 2d24c54..cad21be 100644 --- a/crates/pf-nx-platform/src/console/lifecycle.rs +++ b/crates/pf-nx-platform/src/console/lifecycle.rs @@ -9,7 +9,7 @@ use std::time::{Duration, Instant}; use pf_nx_app::platform::{Clocks, Lifecycle, SystemEvent}; use pf_nx_sys::applet::{self, FocusState, HookEvent}; -use pf_nx_sys::{clk, mem, rt, socket}; +use pf_nx_sys::{clk, mem, rt, socket, swkbd}; pub struct NxLifecycle { events: mpsc::Receiver, @@ -93,6 +93,16 @@ impl Lifecycle for NxLifecycle { }) } + fn keyboard(&mut self) -> Option { + match swkbd::ask("Text to type on the PC", "Send") { + Ok(text) => Some(text), + Err(rc) => { + tracing::info!("keyboard: closed without text ({})", pf_nx_sys::result_code(rc)); + None + } + } + } + /// Returns the socket driver's lent heap, closes the detached threads' stacks, and leaves /// through libnx's `__appExit`: back to hbl when the heap is clean, else through AM to HOME. fn exit(&self) -> ! { diff --git a/crates/pf-nx-sys/build.rs b/crates/pf-nx-sys/build.rs index 55c788d..48e11d0 100644 --- a/crates/pf-nx-sys/build.rs +++ b/crates/pf-nx-sys/build.rs @@ -1,4 +1,4 @@ -//! On the console target: builds `csrc/pf_av.c` and links libnx, newlib, the pinned libavcodec and +//! On the console target: builds `csrc/pf_swkbd.c`, and `csrc/pf_av.c` with `av`, and links libnx, newlib, the pinned libavcodec and //! NVK. On Linux with `av`: the same shim against the system's libavcodec, which decodes in //! software for the dev box. Elsewhere: nothing. `cfg(pf_av)` says the shim is linked. @@ -24,9 +24,10 @@ fn main() { let nvk_on = env::var_os("CARGO_FEATURE_NVK").is_some(); if av { - build_shim(&dkp, &portlibs); + build_shim(&dkp, &portlibs, "pf_av"); println!("cargo:rustc-cfg=pf_av"); } + build_shim(&dkp, &portlibs, "pf_swkbd"); let nvk = PathBuf::from(env::var_os("PF_NX_NVK_LIB").unwrap_or_else(|| "/opt/pf-nx/nvk/lib".into())); if nvk_on && !nvk.join("libvulkan_bh.a").is_file() { @@ -70,9 +71,11 @@ fn main() { } } -/// devkitA64's GCC with libnx's flags, unless the toolchain's cc variables name a compiler. -fn build_shim(dkp: &Path, portlibs: &Path) { - println!("cargo:rerun-if-changed=csrc/pf_av.c"); +/// `csrc/{name}.c` with devkitA64's GCC and libnx's flags, unless the toolchain's cc variables +/// name a compiler. +fn build_shim(dkp: &Path, portlibs: &Path, name: &str) { + let file = format!("csrc/{name}.c"); + println!("cargo:rerun-if-changed={file}"); let target = env::var("TARGET").unwrap_or_default().replace('-', "_"); let mut build = cc::Build::new(); if env::var_os(format!("CC_{target}")).is_none() && env::var_os("TARGET_CC").is_none() { @@ -85,12 +88,12 @@ fn build_shim(dkp: &Path, portlibs: &Path) { } } build - .file("csrc/pf_av.c") + .file(&file) .include(portlibs.join("include")) .include(dkp.join("libnx/include")) .define("__SWITCH__", None) .warnings(true) - .compile("pf_av"); + .compile(name); } /// The shim against the system's libavcodec (`libavcodec-dev` on Debian and Ubuntu). diff --git a/crates/pf-nx-sys/csrc/pf_swkbd.c b/crates/pf-nx-sys/csrc/pf_swkbd.c new file mode 100644 index 0000000..ab0ef59 --- /dev/null +++ b/crates/pf-nx-sys/csrc/pf_swkbd.c @@ -0,0 +1,21 @@ +// The system keyboard applet as one call the Rust side binds (src/swkbd.rs). libnx's +// SwkbdConfig is a versioned argument block it sizes per firmware, so it stays on this side. +#include + +// Shows the keyboard over the title and blocks until the player closes it. 0 with `out` +// holding the committed UTF-8 text, NUL-terminated; libnx's Result otherwise, cancel included. +uint32_t pf_swkbd_ask(const char *guide, const char *ok, uint32_t max_chars, char *out, size_t out_len) { + SwkbdConfig kbd; + out[0] = '\0'; + Result rc = swkbdCreate(&kbd, 0); + if (R_FAILED(rc)) { + return rc; + } + swkbdConfigMakePresetDefault(&kbd); + swkbdConfigSetGuideText(&kbd, guide); + swkbdConfigSetOkButtonText(&kbd, ok); + swkbdConfigSetStringLenMax(&kbd, max_chars); + rc = swkbdShow(&kbd, out, out_len); + swkbdClose(&kbd); + return rc; +} diff --git a/crates/pf-nx-sys/src/lib.rs b/crates/pf-nx-sys/src/lib.rs index 262ba74..78da52b 100644 --- a/crates/pf-nx-sys/src/lib.rs +++ b/crates/pf-nx-sys/src/lib.rs @@ -29,6 +29,7 @@ pub mod random; pub mod rt; pub mod setsys; pub mod socket; +pub mod swkbd; pub mod thread; pub mod vi; diff --git a/crates/pf-nx-sys/src/swkbd.rs b/crates/pf-nx-sys/src/swkbd.rs new file mode 100644 index 0000000..487e709 --- /dev/null +++ b/crates/pf-nx-sys/src/swkbd.rs @@ -0,0 +1,46 @@ +//! The system keyboard: libnx's `swkbd` library applet, through `csrc/pf_swkbd.c` (libnx 4.12.0 +//! `applets/swkbd.h`), which keeps the applet's versioned config block on the C side. + +/// The longest text one [`ask`] takes, in characters. +pub const MAX_CHARS: u32 = 256; + +#[cfg(target_os = "horizon")] +mod ffi { + use core::ffi::c_char; + + extern "C" { + pub fn pf_swkbd_ask( + guide: *const c_char, + ok: *const c_char, + max_chars: u32, + out: *mut c_char, + out_len: usize, + ) -> u32; + } +} + +/// Shows the keyboard with `guide` in its empty field and `ok` on its confirm button, and blocks +/// until the player closes it. The committed text, or the applet's `Result` (a cancel included). +#[cfg(target_os = "horizon")] +pub fn ask(guide: &str, ok: &str) -> Result { + let guide = std::ffi::CString::new(guide).unwrap_or_default(); + let ok = std::ffi::CString::new(ok).unwrap_or_default(); + // UTF-8 takes up to four bytes a character, and the shim NUL-terminates. + let mut out = vec![0u8; MAX_CHARS as usize * 4 + 1]; + // SAFETY: both strings are NUL-terminated and outlive the call; `out` is writable for its + // length, which the applet's reply is bounded by. + let rc = unsafe { + ffi::pf_swkbd_ask( + guide.as_ptr(), + ok.as_ptr(), + MAX_CHARS, + out.as_mut_ptr().cast(), + out.len(), + ) + }; + if rc != 0 { + return Err(rc); + } + let end = out.iter().position(|&b| b == 0).unwrap_or(out.len()); + Ok(String::from_utf8_lossy(&out[..end]).into_owned()) +} From 991b3298559eff53a7e9ba75028ab340aa262142 Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:54:37 -0300 Subject: [PATCH 03/14] feat(stream): the dial's shortcuts press their keys on the host A shortcut slot (Settings > Input > Quick actions: a key from the editor's grid, F1-F12 and Escape among them, with Ctrl, Alt, Shift or Win) only logged that it does nothing here. It now presses every key down in order and lets them up in reverse, as the desktop client does. A key name this build does not know fires nothing. Co-Authored-By: Claude Opus 5.5 --- crates/pf-nx-app/src/runtime/stream.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/crates/pf-nx-app/src/runtime/stream.rs b/crates/pf-nx-app/src/runtime/stream.rs index ea610d3..92f20e5 100644 --- a/crates/pf-nx-app/src/runtime/stream.rs +++ b/crates/pf-nx-app/src/runtime/stream.rs @@ -536,12 +536,38 @@ fn take_ring_commands( RingCommand::CycleTouchMode => {} // Shown once the dial's close has gone out to the input thread. RingCommand::Keyboard => *keyboard = true, + RingCommand::Shortcut(keys) => send_chord(client, &keys), other => tracing::info!("dial: {other:?} does nothing on this console"), } } None } +/// A dial shortcut, typed on the host: every key down in order, then up in reverse, as the +/// desktop client does. A name this build does not know fires nothing. +fn send_chord(client: &NativeClient, keys: &[String]) { + use punktfunk_core::input::{key_vk, InputEvent, InputKind}; + let Some(vks) = keys.iter().map(|k| key_vk(k)).collect::>>() else { + tracing::info!("dial: shortcut {keys:?} names a key this build does not know"); + return; + }; + let key = |kind, vk: u8| InputEvent { + kind, + _pad: [0; 3], + code: u32::from(vk), + x: 0, + y: 0, + flags: 0, + }; + for &vk in &vks { + let _ = client.send_input(&key(InputKind::KeyDown, vk)); + } + for &vk in vks.iter().rev() { + let _ = client.send_input(&key(InputKind::KeyUp, vk)); + } + tracing::info!("dial: shortcut {} sent", keys.join("+")); +} + /// The dial's Keyboard: the system keyboard over the stream, then its text typed on the host as /// committed characters. The main thread waits on the keyboard, so no frame is presented /// meanwhile; the stream and the audio go on. From 3c5b687683feffb4c04efe4fbc0960b266ac27f2 Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:11:52 -0300 Subject: [PATCH 04/14] fix(wp5): pictures go through libavutil's frame again The straight VIC route saved nothing on the console: decode fell by about 0.6 ms and the transfer rose by about the same. It is off by default now, and a file named vic-direto in the data directory turns it back on at start, so one build compares both routes. Co-Authored-By: Claude Opus 5.5 --- crates/pf-nx-platform/src/console/mod.rs | 8 ++++- crates/pf-nx-platform/src/video.rs | 39 +++++++++++++++++++----- crates/pf-nx-probe/src/nvdec.rs | 2 +- crates/pf-nx-sys/csrc/pf_av.c | 12 +++++--- crates/pf-nx-sys/src/av.rs | 22 ++++++++----- 5 files changed, 63 insertions(+), 20 deletions(-) diff --git a/crates/pf-nx-platform/src/console/mod.rs b/crates/pf-nx-platform/src/console/mod.rs index 9e64f75..066728a 100644 --- a/crates/pf-nx-platform/src/console/mod.rs +++ b/crates/pf-nx-platform/src/console/mod.rs @@ -18,6 +18,10 @@ use pf_nx_vk::{GpuDevice, Presenter, Target, VideoSlot}; /// Identity, settings, logs and bundles; the NRO itself lives here too (plan D18). const DATA_DIR: &str = "sdmc:/switch/punktfunk"; +/// A file of this name in [`DATA_DIR`], read at start, turns the straight VIC route on, so one +/// build A/Bs both routes. +const STRAIGHT_MARKER: &str = "vic-direto"; + /// The Mesa pin NVK was built from, for the log and the About screen. const MESA_PIN: &str = match option_env!("PF_MESA_PIN") { Some(p) => p, @@ -94,7 +98,9 @@ pub fn platform(kind: DisplayKind) -> Result { let ui = presenter.ui(); ( Box::new(display_nvk::NvkDisplay::new(presenter, mode)), - Arc::new(crate::video::NxVideo::gpu(dev, slot)), + Arc::new( + crate::video::NxVideo::gpu(dev, slot).straight(Path::new(DATA_DIR).join(STRAIGHT_MARKER).exists()), + ), gpu, ui, ) diff --git a/crates/pf-nx-platform/src/video.rs b/crates/pf-nx-platform/src/video.rs index 9992ff3..f2cd7a3 100644 --- a/crates/pf-nx-platform/src/video.rs +++ b/crates/pf-nx-platform/src/video.rs @@ -4,9 +4,10 @@ //! //! `feed` decodes on the session's video thread: send the access unit, then receive every //! picture straight into the presenter's next staging buffer, which `commit` copies into a -//! plane set and publishes (D7). On the console the VIC writes into that staging buffer -//! itself where it can map it (WP5); else the picture goes through libavutil's own frame and -//! the shim copies it, and the `nvdec[10s]` line names which route ran. On the framebuffer display the pictures land in scratch planes +//! plane set and publishes (D7). On the console the picture goes through libavutil's own frame +//! and the shim copies it; with [`NxVideo::straight`] the VIC writes into that staging buffer +//! itself where it can map it (WP5). The `nvdec[10s]` line names which route ran. On the +//! framebuffer display the pictures land in scratch planes //! and are dropped, so the emulator still runs the hwaccel. A decoder error fails this feed, //! which has the session ask for a keyframe, and the next feed starts from a flushed decoder. @@ -34,15 +35,30 @@ const DECODER: &str = "libavcodec (software)"; pub struct NxVideo { /// The presenter's device and slot; `None` on the framebuffer display. gpu: Option<(Arc, Arc)>, + /// The VIC writes into the staging buffer itself. Off by default: on the console it neither + /// lowered the transfer plus copy nor the end-to-end time (WP5, 2026-11-01). + straight: bool, } impl NxVideo { pub fn gpu(dev: Arc, slot: Arc) -> Self { - Self { gpu: Some((dev, slot)) } + Self { + gpu: Some((dev, slot)), + straight: false, + } } pub fn cpu() -> Self { - Self { gpu: None } + Self { + gpu: None, + straight: false, + } + } + + /// Has the VIC write straight into the staging buffer where it can. + pub fn straight(mut self, on: bool) -> Self { + self.straight = on; + self } } @@ -63,12 +79,21 @@ impl VideoDecoders for NxVideo { spec.bit_depth, if spec.chroma444 { " 4:4:4" } else { "" } ); - let decoder = av::Decoder::open(codec, spec.width, spec.height)?; + let decoder = av::Decoder::open(codec, spec.width, spec.height, self.straight)?; let out = match &self.gpu { Some((dev, slot)) => Out::Gpu(Nv12Upload::new(dev.clone(), slot.clone(), spec)?), None => Out::scratch(spec), }; - tracing::info!("{DECODER}: {codec:?} {}x{}", spec.width, spec.height); + tracing::info!( + "{DECODER}: {codec:?} {}x{}, {}", + spec.width, + spec.height, + if self.straight { + "straight VIC route asked for" + } else { + "VIC then copy" + } + ); let started = Instant::now(); Ok(Box::new(NxSink { lane: Mutex::new(Lane { diff --git a/crates/pf-nx-probe/src/nvdec.rs b/crates/pf-nx-probe/src/nvdec.rs index 667eef3..85d9734 100644 --- a/crates/pf-nx-probe/src/nvdec.rs +++ b/crates/pf-nx-probe/src/nvdec.rs @@ -90,7 +90,7 @@ fn decode(step: &str, codec: Codec, data: &[u8], out: &mut Out) -> Result @@ -65,8 +65,10 @@ void pf_av_close(PfAvDecoder *d) { } // codec: 0 H.264, 1 HEVC. The hwaccel is found by name, so no header needs its enum. +// `direct` non-zero tries the transfer straight into the caller's planes first; zero always +// goes through libavutil's frame and a copy. // TODO(verify): the device type's name is "nvtegra" in the pinned tree (WP0.6). -int pf_av_open(int codec, int width, int height, PfAvDecoder **out) { +int pf_av_open(int codec, int width, int height, int direct, PfAvDecoder **out) { *out = NULL; const AVCodec *c = avcodec_find_decoder(codec == 1 ? AV_CODEC_ID_HEVC : AV_CODEC_ID_H264); if (!c) return AVERROR_DECODER_NOT_FOUND; @@ -89,8 +91,10 @@ int pf_av_open(int codec, int width, int height, PfAvDecoder **out) { if (!d->ctx || !d->pkt || !d->hw || !d->sw) goto fail; #ifdef __SWITCH__ - if (!(d->ctx->hw_device_ctx = av_buffer_ref(d->device)) || !(d->direct = av_frame_alloc())) + if (!(d->ctx->hw_device_ctx = av_buffer_ref(d->device)) || (direct && !(d->direct = av_frame_alloc()))) goto fail; +#else + (void)direct; #endif // The default get_format picks the device's hw format when hw_device_ctx is set. d->ctx->width = width; diff --git a/crates/pf-nx-sys/src/av.rs b/crates/pf-nx-sys/src/av.rs index 5370f4e..5e9ac96 100644 --- a/crates/pf-nx-sys/src/av.rs +++ b/crates/pf-nx-sys/src/av.rs @@ -115,7 +115,13 @@ mod ffi { } extern "C" { - pub fn pf_av_open(codec: c_int, width: c_int, height: c_int, out: *mut *mut PfAvDecoder) -> c_int; + pub fn pf_av_open( + codec: c_int, + width: c_int, + height: c_int, + direct: c_int, + out: *mut *mut PfAvDecoder, + ) -> c_int; pub fn pf_av_send(d: *mut PfAvDecoder, data: *const u8, len: c_int, pts: u64) -> c_int; pub fn pf_av_receive( d: *mut PfAvDecoder, @@ -164,12 +170,14 @@ unsafe impl Send for Decoder {} #[cfg(pf_av)] impl Decoder { - /// Opens `codec` for a `width`×`height` stream. - pub fn open(codec: Codec, width: u32, height: u32) -> anyhow::Result { + /// Opens `codec` for a `width`×`height` stream. `straight` has the VIC write into the + /// caller's planes ([`Route::Vic`]) where it can; otherwise every picture goes through + /// libavutil's frame and a copy ([`Route::VicThenCopy`]). + pub fn open(codec: Codec, width: u32, height: u32, straight: bool) -> anyhow::Result { let (w, h) = (i32::try_from(width)?, i32::try_from(height)?); let mut raw = core::ptr::null_mut(); // SAFETY: `raw` is a valid out-pointer; the shim sets it only on success. - let rc = unsafe { ffi::pf_av_open(codec as i32, w, h, &mut raw) }; + let rc = unsafe { ffi::pf_av_open(codec as i32, w, h, i32::from(straight), &mut raw) }; if rc < 0 { return Err(av_error(&format!("open the {codec:?} decoder"), rc)); } @@ -261,7 +269,7 @@ impl Drop for Decoder { #[cfg(not(pf_av))] impl Decoder { - pub fn open(codec: Codec, _width: u32, _height: u32) -> anyhow::Result { + pub fn open(codec: Codec, _width: u32, _height: u32, _straight: bool) -> anyhow::Result { anyhow::bail!("open the {codec:?} decoder: no decoder on this box (NVDEC on the console, libavcodec on Linux)") } @@ -295,7 +303,7 @@ mod tests { #[cfg(not(pf_av))] #[test] fn open_fails_off_the_console() { - let err = Decoder::open(Codec::Hevc, 1920, 1080).err().unwrap(); + let err = Decoder::open(Codec::Hevc, 1920, 1080, false).err().unwrap(); assert_eq!( err.to_string(), "open the Hevc decoder: no decoder on this box (NVDEC on the console, libavcodec on Linux)" @@ -306,7 +314,7 @@ mod tests { #[test] fn a_dev_box_opens_both_codecs_in_software() { for codec in [Codec::H264, Codec::Hevc] { - let mut d = Decoder::open(codec, 1920, 1080).unwrap(); + let mut d = Decoder::open(codec, 1920, 1080, false).unwrap(); assert_eq!(d.stats(), Stats::default()); d.flush(); } From 69391cfc7a4c848d45dffb2d65a526276fe95a69 Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Sun, 1 Nov 2026 20:11:52 -0300 Subject: [PATCH 05/14] perf(net): the console reads up to 32 datagrams per bsd call The demux read one datagram per bsd call, and at 16 Mbit/s that kept the console near its receive ceiling: the 1 MiB socket buffer filled to about 450 ms behind. quinn-udp's Horizon receive now batches through libnx's recvmmsg. A first refusal other than an empty socket drops back to one datagram per call. Co-Authored-By: Claude Opus 5.5 --- docs/NOTES.md | 4 +- toolchain/patches/quinn-udp/LEG.md | 10 +- toolchain/patches/quinn-udp/src/fallback.rs | 159 +++++++++++++++++--- 3 files changed, 151 insertions(+), 22 deletions(-) diff --git a/docs/NOTES.md b/docs/NOTES.md index 9413796..12fe79f 100644 --- a/docs/NOTES.md +++ b/docs/NOTES.md @@ -13,7 +13,9 @@ What builds, and how; nothing here has run on a console. - Check 3 (`quic`) needs punktfunk-core's Horizon arm for `if-addrs` (monorepo #2180, in the pin). - aws-lc builds with its CC builder: no BIO sockets, CPU features from `-march`, getentropy for entropy and no CPU jitter entropy (`scripts/cargo-nx.sh`). rustls's provider is aws-lc-rs; ring is not linked. -- quinn-udp runs its fallback (no ECN, one datagram per call) until the console shows `sendmsg`. +- quinn-udp runs its fallback (no ECN) until the console shows `sendmsg`. Its receive takes up + to 32 datagrams per bsd call through libnx's `recvmmsg`, and drops to one per call if the first + batch is refused: unproven on the console. - tokio's I/O driver wakes through a loopback UDP socket (the mio leg): unproven on the console. - devkitPro's `switch-ffmpeg` in the image carries the `nvtegra` hwaccel (`libavutil/ hwcontext_nvtegra.h`; the symbols are in `libavcodec.a`), so plan D6 needs no FFmpeg fork. diff --git a/toolchain/patches/quinn-udp/LEG.md b/toolchain/patches/quinn-udp/LEG.md index 7ea82e8..065f609 100644 --- a/toolchain/patches/quinn-udp/LEG.md +++ b/toolchain/patches/quinn-udp/LEG.md @@ -4,7 +4,9 @@ - **Origin:** punktfunk. - **Changes:** `src/lib.rs` builds `fallback.rs` on Horizon instead of `unix.rs`, whose `cmsg` path names Linux and BSD socket options libnx does not define. `fallback.rs` (dead upstream on - every target quinn ships) receives one datagram per call with `recv_from` (socket2 has no - vectored receive on Horizon) and discards `send_to`'s byte count. -- **Cost:** no ECN, no destination address, one datagram per syscall. The `unix.rs` path comes - back behind a cfg once probe check 2 shows `sendmsg`/`recvmsg` on the console. + every target quinn ships) discards `send_to`'s byte count and, on Horizon, receives up to 32 + datagrams per call through libnx's `recvmmsg` (`BATCH_SIZE` 32). A first refusal other than + `WouldBlock` drops that socket to one datagram per call with `recv_from` (socket2 has no + vectored receive on Horizon). +- **Cost:** no ECN, no destination address. The `unix.rs` path comes back behind a cfg once probe + check 2 shows `sendmsg`/`recvmsg` on the console. diff --git a/toolchain/patches/quinn-udp/src/fallback.rs b/toolchain/patches/quinn-udp/src/fallback.rs index 932942b..2428181 100644 --- a/toolchain/patches/quinn-udp/src/fallback.rs +++ b/toolchain/patches/quinn-udp/src/fallback.rs @@ -1,6 +1,9 @@ use std::{ io::{self, IoSliceMut}, - sync::Mutex, + sync::{ + Mutex, + atomic::{AtomicU8, Ordering}, + }, time::Instant, }; @@ -13,14 +16,22 @@ use super::{IO_ERROR_LOG_INTERVAL, RecvMeta, Transmit, UdpSockRef, log_sendmsg_e #[derive(Debug)] pub struct UdpSocketState { last_send_error: Mutex, + /// Horizon (punktfunk's leg): whether `recvmmsg` works here, [`BATCH_UNKNOWN`] until its + /// first answer. + batched: AtomicU8, } +const BATCH_UNKNOWN: u8 = 0; +const BATCH_ON: u8 = 1; +const BATCH_OFF: u8 = 2; + impl UdpSocketState { pub fn new(socket: UdpSockRef<'_>) -> io::Result { socket.0.set_nonblocking(true)?; let now = Instant::now(); Ok(Self { last_send_error: Mutex::new(now.checked_sub(2 * IO_ERROR_LOG_INTERVAL).unwrap_or(now)), + batched: AtomicU8::new(BATCH_UNKNOWN), }) } @@ -52,28 +63,33 @@ impl UdpSocketState { send(socket, transmit) } + /// Horizon (punktfunk's leg): up to [`BATCH_SIZE`] datagrams in one `recvmmsg`, one bsd + /// call for the lot. A first refusal other than an empty socket turns batching off for + /// this socket and reads one datagram a call, as before. pub fn recv( &self, socket: UdpSockRef<'_>, bufs: &mut [IoSliceMut<'_>], meta: &mut [RecvMeta], ) -> io::Result { - // Horizon (punktfunk's leg): socket2 has no vectored receive there, so this reads one - // datagram into the first buffer, which quinn sizes for a whole datagram. - // Safety: `u8` and `MaybeUninit` share a layout, and `recv_from` writes only - // initialised bytes into the buffer. - let buf = unsafe { - &mut *(&mut *bufs[0] as *mut [u8] as *mut [std::mem::MaybeUninit]) - }; - let (len, addr) = socket.0.recv_from(buf)?; - meta[0] = RecvMeta { - len, - stride: len, - addr: addr.as_socket().unwrap(), - ecn: None, - dst_ip: None, - }; - Ok(1) + #[cfg(target_os = "horizon")] + if self.batched.load(Ordering::Relaxed) != BATCH_OFF { + match recv_batch(&socket, bufs, meta) { + Ok(n) => { + if self.batched.swap(BATCH_ON, Ordering::Relaxed) == BATCH_UNKNOWN { + crate::log::info!("recvmmsg: up to {BATCH_SIZE} datagrams a call, {n} in the first"); + } + return Ok(n); + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => return Err(e), + Err(e) if self.batched.load(Ordering::Relaxed) == BATCH_ON => return Err(e), + Err(e) => { + crate::log::warn!("recvmmsg refused ({e}); one datagram a call from here"); + self.batched.store(BATCH_OFF, Ordering::Relaxed); + } + } + } + recv_one(&socket, bufs, meta) } #[inline] @@ -126,4 +142,113 @@ fn send(socket: UdpSockRef<'_>, transmit: &Transmit<'_>) -> io::Result<()> { .map(|_| ()) } +/// One datagram into the first buffer, which quinn sizes for a whole datagram: socket2 has no +/// vectored receive on Horizon. +fn recv_one( + socket: &UdpSockRef<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], +) -> io::Result { + // Safety: `u8` and `MaybeUninit` share a layout, and `recv_from` writes only + // initialised bytes into the buffer. + let buf = unsafe { &mut *(&mut *bufs[0] as *mut [u8] as *mut [std::mem::MaybeUninit]) }; + let (len, addr) = socket.0.recv_from(buf)?; + meta[0] = RecvMeta { + len, + stride: len, + addr: addr.as_socket().unwrap(), + ecn: None, + dst_ip: None, + }; + Ok(1) +} + +/// Up to [`BATCH_SIZE`] datagrams, one per buffer, through libnx's `recvmmsg` (bsd's +/// RecvMMsg, firmware 7.0.0). `MSG_DONTWAIT`: an empty socket is `WouldBlock`, and the call +/// returns what is queued rather than waiting for a full batch. +#[cfg(target_os = "horizon")] +fn recv_batch( + socket: &UdpSockRef<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], +) -> io::Result { + use std::os::unix::io::AsRawFd; + let n = bufs.len().min(meta.len()).min(BATCH_SIZE); + // Safety: all-zero is a valid value of these plain C structs. + let mut names: [libc::sockaddr_storage; BATCH_SIZE] = unsafe { std::mem::zeroed() }; + let mut iovs: [libc::iovec; BATCH_SIZE] = unsafe { std::mem::zeroed() }; + let mut hdrs: [libc::mmsghdr; BATCH_SIZE] = unsafe { std::mem::zeroed() }; + for i in 0..n { + iovs[i] = libc::iovec { + iov_base: bufs[i].as_mut_ptr().cast(), + iov_len: bufs[i].len(), + }; + let h = &mut hdrs[i].msg_hdr; + h.msg_name = (&mut names[i] as *mut libc::sockaddr_storage).cast(); + h.msg_namelen = std::mem::size_of::() as libc::socklen_t; + h.msg_iov = &mut iovs[i]; + h.msg_iovlen = 1; + } + // Safety: the first `n` headers point at `n` live buffers, iovecs and address slots, all + // outliving the call; bsd writes within their stated lengths. + let got = unsafe { + libc::recvmmsg( + socket.0.as_raw_fd(), + hdrs.as_mut_ptr(), + n as libc::c_uint, + libc::MSG_DONTWAIT, + std::ptr::null_mut(), + ) + }; + if got < 0 { + return Err(io::Error::last_os_error()); + } + let got = got as usize; + for i in 0..got { + let len = hdrs[i].msg_len as usize; + let addr = socket_addr(&names[i]).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "recvmmsg: an address of no known family") + })?; + meta[i] = RecvMeta { + len, + stride: len, + addr, + ecn: None, + dst_ip: None, + }; + } + Ok(got) +} + +/// The sender's address as `recvmmsg` wrote it. +#[cfg(target_os = "horizon")] +fn socket_addr(s: &libc::sockaddr_storage) -> Option { + use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; + match libc::c_int::from(s.ss_family) { + libc::AF_INET => { + // Safety: an AF_INET address is a `sockaddr_in`, which the storage outsizes. + let a = unsafe { &*(s as *const libc::sockaddr_storage).cast::() }; + Some(SocketAddr::V4(SocketAddrV4::new( + Ipv4Addr::from(u32::from_be(a.sin_addr.s_addr)), + u16::from_be(a.sin_port), + ))) + } + libc::AF_INET6 => { + // Safety: an AF_INET6 address is a `sockaddr_in6`, which the storage outsizes. + let a = unsafe { &*(s as *const libc::sockaddr_storage).cast::() }; + Some(SocketAddr::V6(SocketAddrV6::new( + Ipv6Addr::from(a.sin6_addr.s6_addr), + u16::from_be(a.sin6_port), + a.sin6_flowinfo, + a.sin6_scope_id, + ))) + } + _ => None, + } +} + +/// Datagrams one [`UdpSocketState::recv`] takes: one `recvmmsg` batch on Horizon. +#[cfg(target_os = "horizon")] +pub(crate) const BATCH_SIZE: usize = 32; +#[cfg(not(target_os = "horizon"))] pub(crate) const BATCH_SIZE: usize = 1; From a3ef2e02bc11372c0171600b5c304419c9f5b8dc Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:32:03 -0300 Subject: [PATCH 06/14] fix(hosts): a paired host follows its advert to a new address Saved hosts were matched by address alone. When the router moved the PC from .137 to .10, the record stayed on the old address and showed offline, and adding the host again saved the same pin a second time. An advert whose fingerprint pins a saved record now moves that record to its address. If a record there already pins it, the stale copies are dropped. An address that still advertises is a second interface and stays put. Co-Authored-By: Claude Opus 5.5 --- crates/pf-nx-app/src/core/model.rs | 85 ++++++++++++++++++++++ crates/pf-nx-app/src/services/store/mod.rs | 2 +- crates/pf-nx-app/src/ui/service.rs | 50 +++++++++++-- 3 files changed, 129 insertions(+), 8 deletions(-) diff --git a/crates/pf-nx-app/src/core/model.rs b/crates/pf-nx-app/src/core/model.rs index c515fc5..47c9e20 100644 --- a/crates/pf-nx-app/src/core/model.rs +++ b/crates/pf-nx-app/src/core/model.rs @@ -141,6 +141,47 @@ pub fn upsert_known_host(hosts: &mut Vec, mut new: KnownHost) -> Opti None } +/// Follows a paired host to the address it now advertises from, matched by its pinned +/// fingerprint, so a DHCP move doesn't leave the record offline and a re-add doesn't save +/// it twice. `advertised` says whether some advert currently comes from an address: a record +/// whose own address still advertises is a second interface, not a move, and stays put. +/// +/// The advert is unauthenticated, but the pin is what every connection checks, so a forged +/// one can only point the record at an address whose handshake then fails. +/// +/// With no record at the advert's address, the record that pins `fp_hex` moves there. With +/// one there that pins the same certificate, the others pinning it are dropped as +/// duplicates. Returns whether anything changed. +pub fn follow_moved_host( + hosts: &mut Vec, + fp_hex: &str, + addr: &str, + port: u16, + advertised: impl Fn(&str, u16) -> bool, +) -> bool { + if fp_hex.is_empty() { + return false; + } + let stale = + |h: &KnownHost| h.fp_hex == fp_hex && (h.addr.as_str(), h.port) != (addr, port) && !advertised(&h.addr, h.port); + match hosts.iter().find(|h| (h.addr.as_str(), h.port) == (addr, port)) { + Some(here) if here.fp_hex == fp_hex => { + let before = hosts.len(); + hosts.retain(|h| !stale(h)); + hosts.len() != before + } + Some(_) => false, + None => { + let Some(moved) = hosts.iter_mut().find(|h| stale(h)) else { + return false; + }; + moved.addr = addr.to_string(); + moved.port = port; + true + } + } +} + fn unique_profile_name(catalog: &[StreamPreset], wanted: &str) -> String { let taken = |name: &str| catalog.iter().any(|p| p.name == name); if !taken(wanted) { @@ -430,4 +471,48 @@ mod tests { assert_eq!(hosts[0].id, h.id); assert_eq!(hosts[0].game_profile("doom"), Some("p1")); } + + fn paired_at(addr: &str, fp: u8) -> KnownHost { + let mut h = KnownHost { + shared: pf_client_core::trust::KnownHost { + name: "desk".into(), + addr: addr.into(), + port: 9777, + ..Default::default() + }, + ..Default::default() + }; + h.set_fingerprint([fp; 32]); + h + } + + /// The 2026-11-02 settings: the router moved the host from .137 to .10, the record stayed + /// on .137 and showed offline, and re-adding it saved the same pin a second time. + #[test] + fn a_paired_host_follows_its_advert_to_a_new_address() { + let fp = "5a".repeat(32); + let quiet = |_: &str, _: u16| false; + + let mut hosts = vec![paired_at("192.168.0.137", 0x5a)]; + assert!(follow_moved_host(&mut hosts, &fp, "192.168.0.10", 9777, quiet)); + assert_eq!(hosts[0].addr, "192.168.0.10"); + + let mut hosts = vec![paired_at("192.168.0.137", 0x5a), paired_at("192.168.0.10", 0x5a)]; + assert!(follow_moved_host(&mut hosts, &fp, "192.168.0.10", 9777, quiet)); + assert_eq!(hosts.len(), 1, "the stale duplicate is dropped"); + assert_eq!(hosts[0].addr, "192.168.0.10"); + + // A second interface that still advertises is not a move. + let mut hosts = vec![paired_at("192.168.0.137", 0x5a)]; + let both = |a: &str, _: u16| a == "192.168.0.137"; + assert!(!follow_moved_host(&mut hosts, &fp, "192.168.0.10", 9777, both)); + assert_eq!(hosts[0].addr, "192.168.0.137"); + + // Another certificate, or another host already at the address, moves nothing. + let mut hosts = vec![paired_at("192.168.0.137", 0x11)]; + assert!(!follow_moved_host(&mut hosts, &fp, "192.168.0.10", 9777, quiet)); + let mut hosts = vec![paired_at("192.168.0.137", 0x5a), paired_at("192.168.0.10", 0x11)]; + assert!(!follow_moved_host(&mut hosts, &fp, "192.168.0.10", 9777, quiet)); + assert_eq!(hosts.len(), 2); + } } diff --git a/crates/pf-nx-app/src/services/store/mod.rs b/crates/pf-nx-app/src/services/store/mod.rs index 28d5e54..6ce2c7f 100644 --- a/crates/pf-nx-app/src/services/store/mod.rs +++ b/crates/pf-nx-app/src/services/store/mod.rs @@ -15,7 +15,7 @@ use anyhow::{Context, Result}; use crate::core::VERSION; pub use crate::core::model::{ - seed_new_host_profiles, upsert_known_host, CodecPref, ExitAction, GamepadType, Persisted, + follow_moved_host, seed_new_host_profiles, upsert_known_host, CodecPref, ExitAction, GamepadType, Persisted, }; pub use crate::core::settings::NxSettings; pub use crate::services::paths::app_dir; diff --git a/crates/pf-nx-app/src/ui/service.rs b/crates/pf-nx-app/src/ui/service.rs index ed0f095..f5453e4 100644 --- a/crates/pf-nx-app/src/ui/service.rs +++ b/crates/pf-nx-app/src/ui/service.rs @@ -161,13 +161,13 @@ impl Service { /// The background's answers, on the service cadence: adverts, sweeps, games, art, pairing. fn service(&mut self) { - if let Some(discovery) = &mut self.discovery { - for host in discovery.poll() { - self.rows_dirty = true; - self.discovered - .retain(|d| !(d.addr == host.addr && d.port == host.port)); - self.discovered.push(host); - } + let arrived = self.discovery.as_mut().map(Discovery::poll).unwrap_or_default(); + for host in arrived { + self.rows_dirty = true; + self.discovered + .retain(|d| !(d.addr == host.addr && d.port == host.port)); + self.follow_moved_host(&host); + self.discovered.push(host); } self.drain_sweep(); self.drain_games(); @@ -775,6 +775,42 @@ impl Service { } } + /// Moves a saved host to the address its advert now comes from, by its pinned fingerprint. + fn follow_moved_host(&mut self, advert: &DiscoveredHost) { + let discovered = &self.discovered; + let advertised = |addr: &str, port: u16| discovered.iter().any(|d| (d.addr.as_str(), d.port) == (addr, port)); + let moved = self.store.edit(|state| { + let moved = store::follow_moved_host( + &mut state.known_hosts, + &advert.fp_hex, + &advert.addr, + advert.port, + advertised, + ); + // A selection on the old address names a record that is no longer there. + if moved + && state + .selected_host + .as_ref() + .is_some_and(|(h, p)| !state.known_hosts.iter().any(|k| k.addr == *h && k.port == *p)) + { + state.selected_host = None; + } + moved + }); + if moved { + tracing::info!( + "console: {} now answers at {}:{}", + advert.name, + advert.addr, + advert.port + ); + self.last_sweep = None; + self.store + .with(|s| crate::services::art::reconcile_host_caches(&s.known_hosts)); + } + } + fn forget_host(&mut self, key: &str) { let mut gone: Option = None; self.store.edit(|state| { From 2f4002ac9f9c929932667c8dc72bb870a811dd50 Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:32:03 -0300 Subject: [PATCH 07/14] perf(wp5): a one-frame smoothness budget adds no whole period Smoothness added the budget as whole stream periods on top of core's adaptive cushion, so one frame held every picture a refresh longer than the jitter asked for. Handheld at one frame, the display stage ran 21-30 ms; under the latency intent it ran 13 ms. Pictures still go out in order, so none are replaced as they are in the latency mode. The adaptive cushion is now the budget's first frame. Two and three frames add one and two periods. Co-Authored-By: Claude Opus 5.5 --- crates/pf-nx-app/src/session/stage/mod.rs | 17 ++++++++++++++++- crates/pf-nx-app/src/session/timeline.rs | 16 +++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/crates/pf-nx-app/src/session/stage/mod.rs b/crates/pf-nx-app/src/session/stage/mod.rs index 506c177..feca0db 100644 --- a/crates/pf-nx-app/src/session/stage/mod.rs +++ b/crates/pf-nx-app/src/session/stage/mod.rs @@ -872,7 +872,22 @@ mod tests { }; let (bare, paced) = (cushion(bare), cushion(paced)); assert!(paced > bare, "the clocked stage reserved the budget: {paced} vs {bare}"); - assert!(paced - bare >= 2 * 16_666_666, "two stream periods of extra lead"); + assert!( + paced - bare >= 16_666_666, + "the second frame is a stream period of extra lead" + ); + } + + /// A one-frame budget is the adaptive cushion alone: no whole period on top of it. + #[test] + fn a_one_frame_budget_adds_no_period() { + let cushion = |budget| { + let mut h = harness_with(FakeSink::default(), budget); + h.stage.submit(&frame(1, None, true, false)); + h.stage.pacing_health().cushion_ns + }; + let latency = cushion(PresentPriority::Latency); + assert_eq!(cushion(PresentPriority::Smooth { buffer: 1 }), latency); } /// An intra-only sink never holds for a re-anchor: the frame after a loss decodes alone. diff --git a/crates/pf-nx-app/src/session/timeline.rs b/crates/pf-nx-app/src/session/timeline.rs index 11d1be5..ed1fef5 100644 --- a/crates/pf-nx-app/src/session/timeline.rs +++ b/crates/pf-nx-app/src/session/timeline.rs @@ -3,14 +3,14 @@ use pf_client_core::trust::PresentPriority; /// Correct clock offset and drift without replacing source cadence with arrival cadence. -/// Smoothness reserves the selected number of stream periods on TOP of core's adaptive -/// cushion; the default keeps the adaptive cushion alone. Neither mode delays compressed -/// input in a Rust queue. +/// Smoothness counts core's adaptive cushion as its first frame and reserves each further one +/// as a whole stream period on top. Neither mode delays compressed input in a Rust queue. pub struct Pacing { clock: punktfunk_core::phase::CadenceClock, /// Source period, and the ceiling core clamps its own adaptive cushion to. source_interval_ns: i64, - /// Extra lead Smoothness adds over the adaptive cushion; `0` under the latency intent. + /// Extra lead Smoothness adds over the adaptive cushion; `0` under the latency intent and + /// for a one-frame budget. smooth_cushion_ns: i64, /// Stamps never go backwards, across recovery too. last_base_ns: u64, @@ -32,13 +32,15 @@ pub struct Pacing { impl Pacing { /// Use source period, not panel period. Priority includes Automatic from settings. /// - /// ⚠ This client reads frame count as EXTRA (core clamps adaptive to 1 period, making - /// one-frame budget unreachable); the honest fix needs additive semantics in core. + /// Core clamps its adaptive cushion to one period, so that cushion is the budget's first + /// frame. Reading the count as extra periods on top held every picture a whole refresh past + /// what the jitter asked for: handheld at one frame the display stage ran 21-30 ms against + /// 13 ms under the latency intent (log of 2026-11-02). pub fn new(source_interval_ns: u64, priority: PresentPriority) -> Self { let interval = i64::try_from(source_interval_ns).unwrap_or(i64::MAX).max(1); // `fifo_capacity` is core's own reading of the enum: 0 means the latency intent, and // `PresentPriority::resolve` has already clamped the buffer to 1–3. - let frames = priority.fifo_capacity(); + let frames = priority.fifo_capacity().saturating_sub(1); let smooth_cushion_ns = interval.saturating_mul(i64::from(frames)); Self { // Default tuning assumes about half a refresh of latch slack; the first pacing record From b549aff774ac82df98dd95cae86a7b498ff58605 Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:16:45 -0300 Subject: [PATCH 08/14] docs: the contract for a Switch platform in the shared shell The client fronts the shell as the PS5, so About > Stream controls shows a DualSense and its touchpad chords, which do nothing on the Switch. Improvements now lists what the monorepo would add: - Platform::Switch and Client::Switch; - the Joy-Con pad group with Minus + A and the escape chord; - confirm on east; - a decision for every PS5 branch. It also lists the lines this client drops once a release carries them. The smoothness section now says that one frame no longer adds a period on top of core's cushion. Co-Authored-By: Claude Opus 5.5 --- docs/improvements.md | 69 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/docs/improvements.md b/docs/improvements.md index 63f120c..3ac5ffe 100644 --- a/docs/improvements.md +++ b/docs/improvements.md @@ -6,12 +6,13 @@ the first changes what a player sees by default, the others live in punktfunk's ## The smoothness buffer's Automatic on the Switch The shared setting resolves Automatic to two frames on every client -(`pf-client-core::trust::PresentPriority::resolve`). This client adds the frames on top of -core's adaptive cushion (`session/timeline.rs`), so one frame here already holds up to about -two refreshes. On the console, one frame presented 59–60 pictures a second at 15–23 ms decode to -present; two frames presented the same at 31–40 ms (NOTES, 2026-10-31). The proposal is that -Automatic picks one frame on the Switch, through a per-platform default in the shared crate or -an override in this client. +(`pf-client-core::trust::PresentPriority::resolve`). This client counts core's adaptive cushion +as the first frame and adds a stream period for each further one (`session/timeline.rs`). Until +2f4002a it added every frame on top of the cushion, so one frame held up to about two +refreshes. On the console, one frame presented 59–60 pictures a second at 15–23 ms decode to +present; two frames presented the same at 31–40 ms (NOTES, 2026-10-31). Both runs predate +2f4002a. The proposal is that Automatic picks one frame on the Switch, through a per-platform +default in the shared crate or an override in this client. Open: one tester's Wi-Fi so far. A busier network may want the second frame back. @@ -50,3 +51,59 @@ the Switch's own step goes. The Windows host built a Joy-Con pair in every conso (NOTES, 2026-10-31); a Switch Pro there is untested. Open: whether every host can build a Switch Pro, or the lists should offer it per host. + +## A Switch platform in the shared shell + +The shell has no Switch, so this client fronts it as `Platform::Ps5` (`runtime/menu.rs`), the +nearest: a console title that quits, with no window. That borrows what is the PS5's alone. The +tester found the first on 2026-11-02: + +- **Stream controls.** About > Stream controls draws a DualSense card that lists "Touchpad + + Cross" and "L1 + R1 + Options + Touchpad". On the Switch the dial is Minus + A + (`core/dial.rs`), and holding L + R + Plus + Minus for a second ends the stream + (`runtime/pads.rs`, `CHORD_HOLD`). The page tells a player to press buttons that do nothing. +- **Confirm side.** The shell confirms on south. Here A, on east, confirms, so `legend_pref` + (`runtime/menu.rs`) swaps the Nintendo legend for the letters one. Without that, the hints + read "B Select" over the button that selects. +- **Codecs.** Settings takes the PS5's list, which includes PyroWave; only `pyrowave_ok: false` + keeps it off. The Switch decodes H.264 and HEVC through NVDEC, has no PyroWave lane on the + console (`video caps` logs `pyrowave: false`) and no AV1. + +The contract this client builds against, in the monorepo: + +1. `pf-console-ui::platform::Platform::Switch`, in `Platform::ALL`, with `can_quit` true. +2. `pf-client-core::shortcuts::Client::Switch`, mapped from it in `screens/controls.rs`. Its pad + group is titled "Joy-Con" and holds: + - `item("Minus + A", "Open the quick actions dial")`; + - `item("L + R + Plus + Minus", "Hold to disconnect")`. + + The group's icon and badges come from the first pad's pref, as now. The pad art for + `joyconpair` and `switchpro` already exists (`pad_art.rs`). +3. On `Platform::Switch`, the Nintendo glyph style confirms on east and backs on south. Every + hint that says Select or Back follows that. +4. The rest of the PS5's branches, decided for the Switch: + + | Where | PS5 now | Switch | + | --- | --- | --- | + | `codecs` | `CODECS_PS5`, with PyroWave | H.264 and HEVC only: `CODECS[..3]` | + | `pad_types` | drops Xbox 360, Steam Deck, Steam Controller 2 | the same, plus the Nintendo kinds from the section above | + | `own_stats_corner` | top left | top left | + | `has_window` | false | false | + | `bitrate_ceiling_kbps` | the custom maximum | the custom maximum | + | `ring_editor` platform | Desktop | Desktop | + | `GlyphStyle::keys` | Remote | Remote | + | help-text branches (mouse mode, stats verbosity, gamepad UI) | the no-chord wording | the same | + + The handheld screen is a touchscreen, and this client sends its touches to the host as a + direct pointer. That is a later row, not part of this contract. + +On this side, once the client builds on a release that has it: + +- `runtime/menu.rs` sets `platform: Platform::Switch`; +- `legend_pref` goes, and the pads report their own pref; +- both `TODO(core)` lines go. + +Until then, the guide names the two chords, because the Stream controls page does not. + +Open: whether the shell should take the dial and escape chords from the client, not from a +table, so a client that rebinds them cannot drift from the page. From f19b7cbdd3e77385bc28e4668e4ce6ad9cfb3344 Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:19:11 -0300 Subject: [PATCH 09/14] docs: the Switch keeps PyroWave in its codec list NVK gives the console Vulkan 1.3, so PyroWave may yet run here. The contract keeps it behind pyrowave_ok, as on the PS5, rather than dropping it. Co-Authored-By: Claude Opus 5.5 --- docs/improvements.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/improvements.md b/docs/improvements.md index 3ac5ffe..f53fc1c 100644 --- a/docs/improvements.md +++ b/docs/improvements.md @@ -65,9 +65,6 @@ tester found the first on 2026-11-02: - **Confirm side.** The shell confirms on south. Here A, on east, confirms, so `legend_pref` (`runtime/menu.rs`) swaps the Nintendo legend for the letters one. Without that, the hints read "B Select" over the button that selects. -- **Codecs.** Settings takes the PS5's list, which includes PyroWave; only `pyrowave_ok: false` - keeps it off. The Switch decodes H.264 and HEVC through NVDEC, has no PyroWave lane on the - console (`video caps` logs `pyrowave: false`) and no AV1. The contract this client builds against, in the monorepo: @@ -85,7 +82,7 @@ The contract this client builds against, in the monorepo: | Where | PS5 now | Switch | | --- | --- | --- | - | `codecs` | `CODECS_PS5`, with PyroWave | H.264 and HEVC only: `CODECS[..3]` | + | `codecs` | `CODECS_PS5`: no AV1, PyroWave behind `pyrowave_ok` | the same. NVDEC has no AV1, and PyroWave runs on NVK's Vulkan 1.3, untested on the console (`pyrowave: false` until then) | | `pad_types` | drops Xbox 360, Steam Deck, Steam Controller 2 | the same, plus the Nintendo kinds from the section above | | `own_stats_corner` | top left | top left | | `has_window` | false | false | From 4f8371a91f696e3a0050acf681d035edc3d57cbb Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:23:27 -0300 Subject: [PATCH 10/14] docs: the Switch contract opens a pad's test and rumble menu The Controllers card is display-only on every platform but Android and Apple. The Switch pads rumble in every stream, so the contract offers the two rows there, and this client answers them. Co-Authored-By: Claude Opus 5.5 --- docs/improvements.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/improvements.md b/docs/improvements.md index f53fc1c..89b7baf 100644 --- a/docs/improvements.md +++ b/docs/improvements.md @@ -91,6 +91,13 @@ The contract this client builds against, in the monorepo: | `GlyphStyle::keys` | Remote | Remote | | help-text branches (mouse mode, stats verbosity, gamepad UI) | the no-chord wording | the same | + `players.rs` opens a pad's menu (Test controller, Rumble) on Android and Apple only + (`can_test`, `can_rumble`). On every other platform, OK on a Controllers card does nothing, so + the Switch's Joy-Con card is display-only. The Switch can do both: its pads rumble in every + stream. The contract lets `Platform::Switch` offer the two rows. This client then answers + `ConsoleCmd::PadTest` and the `rumble` `PadAction`, which `ui/service.rs` logs and drops + today. + The handheld screen is a touchscreen, and this client sends its touches to the host as a direct pointer. That is a later row, not part of this contract. From a31015243f3b935508b4838a635d10ded8d14ccb Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:09:40 -0300 Subject: [PATCH 11/14] feat(wp5): measure whether a picture's due time holds it Smooth at one frame and at two showed the same on the console: e2e 42.5 against 42.7 ms, display 20.5 against 22.5. The extra stream period of the second frame should have added about 17 ms, so the due times may not hold the pictures at all. The nvdec[10s] line now says how many pictures came out with no due time, and how far ahead of it the rest came out. A new slot[10s] line counts, under smoothness: - the vblanks a due time held a picture back; - the takes with no due, or taken early; - how late the takes ran; - how many pictures stood behind each. Co-Authored-By: Claude Opus 5.5 --- crates/pf-nx-platform/src/video.rs | 45 +++++++++++- crates/pf-nx-vk/src/picture.rs | 112 ++++++++++++++++++++++++++++- 2 files changed, 154 insertions(+), 3 deletions(-) diff --git a/crates/pf-nx-platform/src/video.rs b/crates/pf-nx-platform/src/video.rs index f2cd7a3..73748ec 100644 --- a/crates/pf-nx-platform/src/video.rs +++ b/crates/pf-nx-platform/src/video.rs @@ -215,6 +215,7 @@ impl Lane { .receive_into(planes.y, planes.y_stride, planes.uv, planes.uv_stride)?; let Some(meta) = got else { return Ok(false) }; let due = self.due_of(meta.pts); + self.timing.note_due(due); let started = Instant::now(); let shown = self.out.commit(meta, due)?; let upload = started.elapsed(); @@ -257,6 +258,13 @@ struct Timing { in_flight_max: u32, /// How the last picture reached the staging planes; `None` before the first. route: Option, + /// Pictures whose host PTS found no due time: they are shown at once, whatever the budget. + undue: u32, + /// The due time less the moment the picture came out of the decoder, µs, over the pictures + /// that had one. Negative is already late. + slack_us: i64, + slack_min_us: i64, + slack_max_us: i64, } impl Timing { @@ -278,9 +286,43 @@ impl Timing { gpu_copies: 0, in_flight_max: 0, route: None, + undue: 0, + slack_us: 0, + slack_min_us: i64::MAX, + slack_max_us: i64::MIN, } } + /// Whether the picture out now kept its due time, and how far ahead of it. + fn note_due(&mut self, due: Option) { + let Some(due) = due else { + self.undue += 1; + return; + }; + let now = Instant::now(); + let us = |d: Duration| i64::try_from(d.as_micros()).unwrap_or(i64::MAX); + let slack = if due >= now { us(due - now) } else { -us(now - due) }; + self.slack_us += slack; + self.slack_min_us = self.slack_min_us.min(slack); + self.slack_max_us = self.slack_max_us.max(slack); + } + + /// `"due slack mean … min … max … ms, N without a due"` over this window's pictures. + fn due_summary(&self) -> String { + let timed = i64::from(self.pictures.saturating_sub(self.undue)); + let ms = |us: i64| us as f64 / 1000.0; + if timed == 0 { + return format!("no due times, {} without", self.undue); + } + format!( + "due slack mean {:.1}ms min {:.1}ms max {:.1}ms, {} without a due", + ms(self.slack_us / timed), + ms(self.slack_min_us), + ms(self.slack_max_us), + self.undue + ) + } + fn note(&mut self, s: &av::Stats, wait: Duration, upload: Duration, gpu_copy_us: Option, shown: bool) { // The first route, and any change, at once: a change mid-stream is the straight route // falling back. @@ -324,7 +366,7 @@ impl Timing { .checked_div(u64::from(self.gpu_copies)) .map_or_else(|| "-".to_string(), |us| format!("{:.2}", us as f64 / 1000.0)); tracing::info!( - "nvdec[{}s]: {} pictures ({} dropped, {} decoded in all); staging wait mean {:.1}ms max {:.1}ms; decode mean {:.1}ms max {:.1}ms; transfer mean {:.1}ms max {:.1}ms; copy mean {:.1}ms max {:.1}ms; upload mean {:.2}ms; gpu copy mean {gpu_copy}ms; in flight max {}; route {}", + "nvdec[{}s]: {} pictures ({} dropped, {} decoded in all); staging wait mean {:.1}ms max {:.1}ms; decode mean {:.1}ms max {:.1}ms; transfer mean {:.1}ms max {:.1}ms; copy mean {:.1}ms max {:.1}ms; upload mean {:.2}ms; gpu copy mean {gpu_copy}ms; in flight max {}; route {}; {}", TIMING_WINDOW.as_secs(), self.pictures, self.dropped, @@ -340,6 +382,7 @@ impl Timing { per(self.upload), self.in_flight_max, self.route.map_or("-", av::Route::name), + self.due_summary(), ); let route = self.route; *self = Self::new(); diff --git a/crates/pf-nx-vk/src/picture.rs b/crates/pf-nx-vk/src/picture.rs index 9bc2240..ea14a08 100644 --- a/crates/pf-nx-vk/src/picture.rs +++ b/crates/pf-nx-vk/src/picture.rs @@ -77,7 +77,14 @@ impl VideoSlot { /// The picture for the vblank at `latch`, if a new one is due by then. pub fn take(&self, latch: Instant) -> Option> { - self.lock().take(latch) + let mut q = self.lock(); + let picture = q.take(latch); + if q.smoothing > 0 { + if let Some(w) = q.window.close(latch) { + tracing::info!("{}", w.line()); + } + } + picture } /// Drops the waiting pictures: the stream ended. @@ -113,12 +120,86 @@ pub struct Drops { pub overflow: u32, } +/// How often the slot reports how its pictures met their due times. +const WINDOW: std::time::Duration = std::time::Duration::from_secs(10); + +/// One window of the smoothing slot's takes: whether a due time ever held a picture back, how +/// late the ones taken were against it, and how many waited behind each. A standing picture +/// behind every take is a refresh of latency the budget did not ask for. +#[derive(Default)] +struct Window { + since: Option, + taken: u32, + /// Vblanks with a picture waiting that was not due yet. + held: u32, + /// Taken with no due time. + undue: u32, + /// Taken before their due time, to keep the queue inside the budget. + early: u32, + /// Latch less due over the pictures taken at or past it, µs. + late_us: u64, + late_max_us: u64, + late_n: u32, + /// Pictures left waiting after each take: none, one, two or more. + left: [u32; 3], +} + +impl Window { + fn note_take(&mut self, latch: Instant, due: Option, left: usize) { + self.taken += 1; + self.left[left.min(2)] += 1; + match due { + None => self.undue += 1, + Some(d) if d > latch => self.early += 1, + Some(d) => { + let us = u64::try_from((latch - d).as_micros()).unwrap_or(u64::MAX); + self.late_us += us; + self.late_max_us = self.late_max_us.max(us); + self.late_n += 1; + } + } + } + + /// The finished window at `now`, arming the next; `None` until one has run its length. + fn close(&mut self, now: Instant) -> Option { + let since = *self.since.get_or_insert(now); + (now.duration_since(since) >= WINDOW).then(|| { + std::mem::replace( + self, + Self { + since: Some(now), + ..Self::default() + }, + ) + }) + } + + fn line(&self) -> String { + let ms = |us: u64| us as f64 / 1000.0; + format!( + "slot[{}s]: {} taken, {} vblanks held for a due, {} without a due, {} early; late mean {:.1}ms max {:.1}ms; left behind 0:{} 1:{} 2+:{}", + WINDOW.as_secs(), + self.taken, + self.held, + self.undue, + self.early, + ms(self.late_us.checked_div(u64::from(self.late_n)).unwrap_or(0)), + ms(self.late_max_us), + self.left[0], + self.left[1], + self.left[2], + ) + } +} + /// The slot's rule, apart from the pictures, so the tests drive it with plain values. struct Queue { items: VecDeque<(Option, T)>, /// `0`: newest wins. smoothing: usize, drops: Drops, + /// Smoothing only: the takes since the last report. + window: Window, } impl Default for Queue { @@ -127,6 +208,7 @@ impl Default for Queue { items: VecDeque::with_capacity(MAX_QUEUED + 1), smoothing: 0, drops: Drops::default(), + window: Window::default(), } } } @@ -165,12 +247,15 @@ impl Queue { // still for seconds (smoothing log of 2026-10-31, 151 trimmed in one window). let due = due.max(self.items.len().saturating_sub(self.smoothing + 1)); if due == 0 { + self.window.held += u32::from(!self.items.is_empty()); return None; } let behind = due.saturating_sub(2); self.drops.replaced += behind as u32; self.items.drain(..behind); - self.items.pop_front().map(|(_, item)| item) + let (at, item) = self.items.pop_front()?; + self.window.note_take(latch, at, self.items.len()); + Some(item) } fn clear(&mut self) { @@ -677,4 +762,27 @@ mod tests { q.push(None, 1); assert_eq!(q.take(t0), Some(1)); } + + /// The window says whether a due time held anything, how late the takes ran and what + /// stood behind each: the figures that tell a binding budget from a plain queue. + #[test] + fn the_window_counts_holds_lateness_and_what_waits() { + let t0 = Instant::now(); + let mut q = Queue::default(); + q.set_smoothing(1); + q.push(Some(ms(t0, 40)), 1); + assert_eq!(q.take(ms(t0, 16)), None, "not due: held"); + q.push(Some(ms(t0, 45)), 2); + assert_eq!(q.take(ms(t0, 50)), Some(1), "10 ms late, one behind it"); + q.push(None, 3); + assert_eq!(q.take(ms(t0, 66)), Some(2)); + assert_eq!(q.take(ms(t0, 83)), Some(3)); + let w = &q.window; + assert_eq!((w.taken, w.held, w.undue, w.early), (3, 1, 1, 0)); + assert_eq!((w.late_n, w.late_max_us), (2, 21_000)); + assert_eq!(w.left, [1, 2, 0]); + assert!(q.window.close(ms(t0, 0)).is_none(), "the first close arms the window"); + assert!(q.window.close(ms(t0, 10_000)).is_some()); + assert_eq!(q.window.taken, 0, "the next window starts empty"); + } } From 5d6075dc3711477fdd2c108a6b75600e93885607 Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Mon, 2 Nov 2026 13:47:46 -0300 Subject: [PATCH 12/14] fix(wp5): the smoothing slot skips a standing backlog A burst on the wire left one due picture waiting behind every take for the rest of the session. No due time ever held a picture, and Smooth 1 and 2 waited the same ~37 ms from decode to take. After 30 takes in a row with a second due picture, the slot now skips one, and the slot[10s] line counts it as caught up. Co-Authored-By: Claude Opus 5.5 --- crates/pf-nx-vk/src/picture.rs | 62 ++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/crates/pf-nx-vk/src/picture.rs b/crates/pf-nx-vk/src/picture.rs index ea14a08..4935734 100644 --- a/crates/pf-nx-vk/src/picture.rs +++ b/crates/pf-nx-vk/src/picture.rs @@ -107,6 +107,11 @@ const MAX_SMOOTHING: usize = 3; /// Pictures a smoothing queue holds past its budget: the pacing's own cushion, up to a period, /// and one picture of slack before the oldest is dropped. const SMOOTHING_SLACK: usize = 2; +/// Takes in a row with a second due picture waiting before the slot skips one. A burst on the +/// wire leaves that picture standing for the rest of the session, a refresh of latency at every +/// budget (log of 2026-11-02: one left behind on 571 of 598 takes, no take ever held for a due). +/// Due times straddling a vblank leave one now and then, never half a second running. +const STANDING_TAKES: u32 = 30; /// The most pictures the slot ever holds. const MAX_QUEUED: usize = MAX_SMOOTHING + SMOOTHING_SLACK; @@ -136,6 +141,8 @@ struct Window { undue: u32, /// Taken before their due time, to keep the queue inside the budget. early: u32, + /// Standing backlogs skipped, one picture each. + caught_up: u32, /// Latch less due over the pictures taken at or past it, µs. late_us: u64, late_max_us: u64, @@ -177,12 +184,13 @@ impl Window { fn line(&self) -> String { let ms = |us: u64| us as f64 / 1000.0; format!( - "slot[{}s]: {} taken, {} vblanks held for a due, {} without a due, {} early; late mean {:.1}ms max {:.1}ms; left behind 0:{} 1:{} 2+:{}", + "slot[{}s]: {} taken, {} vblanks held for a due, {} without a due, {} early, {} caught up; late mean {:.1}ms max {:.1}ms; left behind 0:{} 1:{} 2+:{}", WINDOW.as_secs(), self.taken, self.held, self.undue, self.early, + self.caught_up, ms(self.late_us.checked_div(u64::from(self.late_n)).unwrap_or(0)), ms(self.late_max_us), self.left[0], @@ -198,6 +206,8 @@ struct Queue { /// `0`: newest wins. smoothing: usize, drops: Drops, + /// Smoothing only: takes in a row that left a due picture waiting. + standing: u32, /// Smoothing only: the takes since the last report. window: Window, } @@ -208,6 +218,7 @@ impl Default for Queue { items: VecDeque::with_capacity(MAX_QUEUED + 1), smoothing: 0, drops: Drops::default(), + standing: 0, window: Window::default(), } } @@ -231,6 +242,7 @@ impl Queue { /// Newest wins ignores `due`. Smoothing shows the oldest picture due by `latch` and lets one /// more due wait for the next vblank: due times that straddle a vblank then take one refresh /// each. A third due is a backlog, and its oldest are dropped so latency stays the budget. + /// A second due left waiting [`STANDING_TAKES`] takes running is one too, and is skipped. /// More waiting than the budget holds means the due times run ahead of the arrivals, and the /// oldest goes out early. fn take(&mut self, latch: Instant) -> Option { @@ -250,7 +262,13 @@ impl Queue { self.window.held += u32::from(!self.items.is_empty()); return None; } - let behind = due.saturating_sub(2); + let mut behind = due.saturating_sub(2); + self.standing = if due >= 2 { self.standing + 1 } else { 0 }; + if self.standing >= STANDING_TAKES { + behind = due - 1; + self.standing = 0; + self.window.caught_up += 1; + } self.drops.replaced += behind as u32; self.items.drain(..behind); let (at, item) = self.items.pop_front()?; @@ -260,6 +278,7 @@ impl Queue { fn clear(&mut self) { self.items.clear(); + self.standing = 0; } fn trim(&mut self) { @@ -763,6 +782,45 @@ mod tests { assert_eq!(q.take(t0), Some(1)); } + #[test] + fn a_standing_backlog_is_skipped() { + let t0 = Instant::now(); + let mut q = Queue::default(); + q.set_smoothing(1); + q.push(Some(t0), 0); + let mut shown = Vec::new(); + for i in 1..=u64::from(STANDING_TAKES) + 2 { + q.push(Some(ms(t0, i * 16)), i); + shown.extend(q.take(ms(t0, i * 16 + 1))); + } + let skip = STANDING_TAKES as usize - 1; + assert_eq!(shown[skip - 1], skip as u64 - 1); + assert_eq!(shown[skip], skip as u64 + 1, "one picture skipped"); + assert_eq!(q.items.len(), 0, "nothing stands behind the take after it"); + assert_eq!((q.drops.replaced, q.window.caught_up), (1, 1)); + } + + #[test] + fn a_backlog_that_drains_is_not_skipped() { + let t0 = Instant::now(); + let mut q = Queue::default(); + q.set_smoothing(1); + for i in 0..200_u64 { + match i % 10 { + // One picture arrives late with the next; the host later skips one, which drains it. + 2 | 6 => {} + 3 => { + q.push(Some(ms(t0, (i - 1) * 16)), i - 1); + q.push(Some(ms(t0, i * 16)), i); + } + _ => q.push(Some(ms(t0, i * 16)), i), + } + q.take(ms(t0, i * 16 + 1)); + } + assert_eq!(q.drops, Drops::default()); + assert_eq!(q.window.caught_up, 0); + } + /// The window says whether a due time held anything, how late the takes ran and what /// stood behind each: the figures that tell a binding budget from a plain queue. #[test] From 8d113ec47893463f874821dc6d96fb7bf616c057 Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Mon, 2 Nov 2026 14:57:02 -0300 Subject: [PATCH 13/14] feat(wp5): Automatic bitrate tops out at 20 Mb/s without PyroWave Core finds the link's edge by crossing it. Handheld, its climbs to 22-25 Mb/s preceded every 60-110 ms delay spike, while a pinned 20 Mb/s ran clean at stock clocks. Where only NVDEC decodes, Automatic now stops at 20 Mb/s and still cuts below it for a dock or a weak Wi-Fi. A title that decodes PyroWave keeps the 650 Mb/s ceiling its quality floor needs. Co-Authored-By: Claude Opus 5.5 --- crates/pf-nx-app/src/core/model.rs | 20 +++++++++++++++++--- crates/pf-nx-app/src/runtime/mod.rs | 11 +++++++---- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/crates/pf-nx-app/src/core/model.rs b/crates/pf-nx-app/src/core/model.rs index 47c9e20..513abef 100644 --- a/crates/pf-nx-app/src/core/model.rs +++ b/crates/pf-nx-app/src/core/model.rs @@ -314,12 +314,26 @@ impl GamepadType { } } -/// Automatic bitrate's ceiling, handed to `punktfunk_core::abr` through `run` -/// (`PUNKTFUNK_ABR_MAX_MBPS`). It binds every codec, so it sits above Automatic `PyroWave`'s +/// Automatic bitrate's ceiling where `PyroWave` decodes, handed to `punktfunk_core::abr` through +/// `run` (`PUNKTFUNK_ABR_MAX_MBPS`). It binds every codec, so it sits above Automatic `PyroWave`'s /// quality floor (0.5 bit/pixel), below which core clamps a `PyroWave` session to that floor. -/// HEVC and H.264 stop well below it, at the link's and the decoder's headroom. pub const BITRATE_MAX_KBPS: u32 = 650_000; +/// Automatic bitrate's ceiling where only NVDEC decodes: the rate a pinned stream ran clean at, +/// handheld at stock clocks. Core finds the link's edge by crossing it, and its climbs to +/// 22-25 Mb/s preceded every 60-110 ms delay spike (2026-10-31). Under the ceiling it still cuts +/// for a dock or a weak Wi-Fi. A raised CPU clock is what would carry more. +pub const NVDEC_BITRATE_MAX_KBPS: u32 = 20_000; + +/// Automatic bitrate's ceiling for a title whose video lane can or cannot decode `PyroWave`. +pub const fn bitrate_max_kbps(pyrowave: bool) -> u32 { + if pyrowave { + BITRATE_MAX_KBPS + } else { + NVDEC_BITRATE_MAX_KBPS + } +} + /// 70% of measured goodput (headroom for FEC, loss). /// Integer arithmetic order matters (`/ 10 * 7`, not `* 0.7`) — all clients must agree exactly. pub fn recommended_bitrate_kbps(throughput_kbps: u32) -> u32 { diff --git a/crates/pf-nx-app/src/runtime/mod.rs b/crates/pf-nx-app/src/runtime/mod.rs index 28ebf55..c7dccac 100644 --- a/crates/pf-nx-app/src/runtime/mod.rs +++ b/crates/pf-nx-app/src/runtime/mod.rs @@ -14,7 +14,7 @@ mod touch; use std::sync::atomic::{AtomicBool, Ordering}; -use crate::core::model::BITRATE_MAX_KBPS; +use crate::core::model::bitrate_max_kbps; use crate::platform::{Display, Lifecycle, Pads, Platform, SystemEvent}; use crate::services::power::ExitPlan; use crate::services::store; @@ -78,14 +78,17 @@ const ABR_PROBE_KBPS: u32 = 320_000; /// The two automatic-bitrate knobs punktfunk-core reads from the environment. Set before any /// thread exists: `set_var` is not thread-safe. -fn set_abr_env() { +fn set_abr_env(pyrowave: bool) { std::env::set_var("PUNKTFUNK_ABR_PROBE_KBPS", ABR_PROBE_KBPS.to_string()); - std::env::set_var("PUNKTFUNK_ABR_MAX_MBPS", (BITRATE_MAX_KBPS / 1_000).to_string()); + std::env::set_var( + "PUNKTFUNK_ABR_MAX_MBPS", + (bitrate_max_kbps(pyrowave) / 1_000).to_string(), + ); } /// Runs the title until the player leaves it. Never returns. pub fn run(platform: Platform) -> ! { - set_abr_env(); + set_abr_env(platform.video.caps().pyrowave); let Platform { display, video, From 25419195581a4c5732cb315c16d59c86eb7b7ff9 Mon Sep 17 00:00:00 2001 From: Gava <21977618+broslife@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:02:19 -0300 Subject: [PATCH 14/14] docs: WP5 closes and its open items leave the plan The ten-hour soak goes to public use, applet mode does not launch, and the CPU clock explains the menu's long frames. Automatic keeps a 20 Mb/s ceiling where only NVDEC decodes. Clock control and PyroWave on NVK become their own tasks after the baseline. Co-Authored-By: Claude Opus 5.5 --- docs/plan.md | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/docs/plan.md b/docs/plan.md index f90ec75..ea0db79 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -1,7 +1,7 @@ # Nintendo Switch client — a Rust homebrew title on NVDEC > **Status: designed 2026-10-28; WP0's probe passed on the console and WP1 closed on 2026-10-29 -> (`docs/NOTES.md`, §4); WP2–WP4 closed by 2026-11-01. WP5 is next.** Target: the Switch 1 (Erista and Mariko, Tegra +> (`docs/NOTES.md`, §4); WP2–WP4 closed by 2026-11-01, WP5 on 2026-11-02. WP6 is next.** Target: the Switch 1 (Erista and Mariko, Tegra > X1), on Atmosphère, as an `.nro`. The Switch 2 is a later leg behind the same seams (§9); no > modded one exists to test on. The maintainer's fellow developer holds the only test device, so > every work package is written for a developer who never holds the console — the PS5 client's @@ -28,7 +28,8 @@ answer for anyone who will not install a CFW. settings are the console UI's shared catalogue (`clients/shared/settings-catalog.json`). - **Contract twins are pinned with the shared vectors**, not FFI: `gamepad-kind-vectors.json`, `touch-vectors.json`, `device-gyro-vectors.json`, `jitter-vectors.json`. -- **Automatic means automatic**: no bitrate limit, no decoder pick, no clock boost in v1. +- **Automatic picks the bitrate under a ceiling**: 20 Mb/s where only NVDEC decodes, the rate + the console receives clean at stock clocks. No decoder pick. Clock control is its own task (§10). - **Prior art is pinned, not forked loosely**: every vendored crate leg names its upstream version and the project it came from (§3.1) in its README. @@ -48,7 +49,7 @@ answer for anyone who will not install a CFW. | D10 | **Face buttons follow the OS: A confirms, B backs**, in the console UI on this platform. Every Switch app does; a client that swaps them is a bug to the player. (The Android Nintendo-menu question stays the maintainer's; this is the native case of it.) | | D11 | **Stream mode follows the operation mode**: 1280×720@60 handheld, 1920×1080@60 docked, read at connect and on `AppletHookType_OnOperationMode`; a dock change mid-stream asks the host for the new size (host owns the delivered mode). Refresh is 60. | | D12 | **Sleep ends the stream** (`OnFocusState`/sleep hook → clean close); wake shows the host list. Reconnect is the player's press. No session survives suspend. | -| D13 | **Full-memory mode first.** The title is validated launched by title override (hold R over an installed title); applet mode (album) is detected (`appletGetAppletType`) and warned about on the start screen, and WP5 measures whether the client fits in it. | +| D13 | **Full-memory mode first.** The title is validated launched by title override (hold R over an installed title); applet mode (album) is detected (`appletGetAppletType`) and warned about on the start screen. A launch from the Album crashed before the first menu, so applet mode is not supported: the manual says to hold R over a game (WP6). | | D14 | **Codecs advertised**: `CODEC_H264 | CODEC_HEVC`, 8-bit, SDR. No AV1 on the X1 (accepted). Main10 only if WP0's probe decodes it through the VIC path. PyroWave is out of v1: the lane lifts with `pf-ps5-vk`, but a 256-core Maxwell at 307–460 MHz has to be measured against the wavelet decode before it is advertised (§10). | | D15 | **Dev loop without the console**: `pf-nx-host` runs the app on a Linux box through the real presenter on lavapipe and on the Mac on CPU raster, as `pf-ps5-host` does; **Ryujinx (Ryubing) on the Mac runs the real NRO on the framebuffer `Display`** — libnx, sockets, the nvservices the hwaccel talks to are emulated, so everything but the GPU path is proven here. Stock emulators do not run NVK builds (`bevy-horizon`'s finding). Timing from the emulator is never cited. `nxlink -s` is the deploy and the stdout on the device. | | D16 | **CI**: GitHub Actions like `client-ps5`: a host lane (test, clippy, the host client on lavapipe) and a console lane in a Docker image from `devkitpro/devkita64` plus the nightly, rust-src, and three archives built once per pin (`task nvk`, `task ffmpeg`, `task skia`). The NRO is a CI artifact per PR; releases are tags. | @@ -223,7 +224,8 @@ bundle to `sdmc:/switch/punktfunk/verdict/` (`docs/NOTES.md`, title at `927533b` is gone. Ryubing recognised the NRO and then closed itself, untraced; the framebuffer fallback waits for a console where NVK fails. - The menu holds 60 fps in most ten-second windows, not all: some lose up to 33 frames to - single frames of 65–213 ms. WP5 traces them. + single frames of 65–213 ms. WP5's clock sweep put them on the CPU clock: at 1785 MHz the menu + lagged less. They go with clock control (§10). - The bundle is a folder of PNGs and JSON on the SD card, not a zip, fetched by hand. ### WP3 — The video plane (three weeks) @@ -275,13 +277,26 @@ package there. Departures: - The Nintendo controller types in the shared Settings list and labels wait for a monorepo PR (`docs/improvements.md`). Audio underruns climb in silence with nothing audible; left for WP5. -### WP5 — Reliability and latency (two weeks) +### WP5 — Reliability and latency (closed 2026-11-02) -Measured, not guessed: the picture copy (D7) and nvmap import if it shows; the audio sync cell -if the presentation clock drifts from it; Wi-Fi loss behaviour against the host's recovery; -memory in applet mode; the GPU clock the title gets and whether the official handheld -configuration is reachable without an opt-in; PyroWave on NVK against the GPU budget (D14). -Ten-hour soak. +Measured on the console: + +- The picture copy (D7) costs what the straight VIC route saves, so the copy stays and nvmap + import is not needed. +- Audio plays 40–90 ms behind the picture, steady, with no drift. +- The receive path reads 32 datagrams a call. It holds 20 Mb/s handheld with no loss; docked + 1080p wants 10–12 Mb/s. +- The smoothness budget adds no whole period, and the slot no longer keeps a picture standing. +- The stats line carries the CPU, GPU and memory clocks. A handheld sweep found that the CPU + clock is the one that cuts latency. + +Taken out: + +- The ten-hour soak: public use answers it better than one tester can. +- Memory in applet mode: the title does not launch there (D13). +- A trace of the menu's long frames: the CPU clock explains them. + +Clock control and PyroWave on NVK are their own tasks after the baseline (§10). ### WP6 — Package and release (one week) @@ -343,7 +358,7 @@ client-switch/ | Skia archive | §3.3 | raster fallback | | `nvtegra` is not in devkitPro's `switch-ffmpeg` at the pinned version | WP0.6 | build the pinned fork; `scripts/build-ffmpeg.sh` exists for it either way | | The crate legs drift | every nightly or pin bump | `scripts/pins.env` pins them; a bump is a PR with the probe re-run | -| Applet mode out of memory | WP0.9, WP5 | title-override documented as required; applet mode best effort | +| Applet mode out of memory | WP0.9 | title override documented as required; applet mode unsupported (D13) | | The emulator hides a device-only fault | every round | the tester's bundle decides; Ryujinx is never cited | | One device, one tester | throughput | rounds are batched; the probe lists what to run | @@ -358,10 +373,12 @@ is a second `pf-nx-platform` cfg, not a fork. ## 10. After the baseline -PyroWave on NVK if WP5's budget allows; nvmap import for the stream image (if WP5 leaves it); +Clock control from the title (CPU, GPU, memory): a probe first, for whether clkrst sets them +under hbloader, against sys-clk, and how they come back on exit, sleep and a dock change. Then an +opt-in performance mode, the menu's long frames, and Automatic's ceiling raised to about 25 Mb/s. +PyroWave on NVK, measured against the GPU budget (D14), as a task of its own. USB keyboard and mouse on the dock (`hidKeyboard*`/`hidMouse*` → the keymap contract); -`hwopus`; the GPU-clock question if WP5 shows a case; the Homebrew App Store listing; the -Switch 2. +`hwopus`; the Homebrew App Store listing; the Switch 2. ## 11. Open before WP0 starts (the maintainer's)