jetson h265: encoder

.changeset/advertise_pass_through_codecs_without_a_real_encoder.md

@@ -0,0 +1,10 @@
+---
+webrtc-sys: patch
+libwebrtc: minor
+livekit: patch
+livekit-ffi: patch
+livekit-uniffi: patch
+livekit-capture: patch
+---
+
+Pre-encoded video publishing no longer requires a local encoder for its codec, so pre-encoded H265 works on hosts without an H265 encoder, a camera track requesting a codec no local encoder produces now publishes VP8 instead of a codec it cannot encode, and `PeerConnectionFactory::video_encoder_codecs` reports which codecs each encoder backend can produce.

examples/local_video/src/subscriber.rs

@@ -417,6 +417,10 @@ struct Args {
     /// Shared encryption key for E2EE (enables AES-GCM end-to-end encryption when set; must match publisher's key)
     #[arg(long)]
     e2ee_key: Option<String>,
+
+    /// Run without a window: still subscribe, decode, and log stats, but render nothing
+    #[arg(long)]
+    headless: bool,
 }
 
 struct SharedYuv {
@@ -1818,6 +1822,14 @@ async fn run(args: Args, ctrl_c_received: Arc<AtomicBool>) -> Result<()> {
         }
     });
 
+    if args.headless {
+        info!("Headless mode: decoding without a window, press Ctrl-C to exit");
+        while !ctrl_c_received.load(Ordering::Acquire) {
+            tokio::time::sleep(Duration::from_millis(200)).await;
+        }
+        return Ok(());
+    }
+
     let viewport = AspectConstrainedViewport::new(None);
     // Start UI
     let app = VideoApp {

libwebrtc/src/native/peer_connection_factory.rs

@@ -26,6 +26,7 @@ use crate::{
     peer_connection::PeerConnection,
     peer_connection_factory::RtcConfiguration,
     rtp_parameters::RtpCapabilities,
+    rtp_sender::VideoEncoderBackend,
     video_source::native::NativeVideoSource,
     video_track::RtcVideoTrack,
     MediaType, RtcError,
@@ -141,6 +142,10 @@ impl PeerConnectionFactory {
         self.sys_handle.rtp_receiver_capabilities(media_type.into()).into()
     }
 
+    pub fn video_encoder_codecs(&self, backend: VideoEncoderBackend) -> Vec<String> {
+        self.sys_handle.video_encoder_backend_codecs(backend.into())
+    }
+
     // ===== Device Management Methods =====
 
     /// Get the number of playout (output) devices

libwebrtc/src/peer_connection_factory.rs

@@ -16,7 +16,7 @@ use std::fmt::Debug;
 
 use crate::{
     imp::peer_connection_factory as imp_pcf, peer_connection::PeerConnection,
-    rtp_parameters::RtpCapabilities, MediaType, RtcError,
+    rtp_parameters::RtpCapabilities, rtp_sender::VideoEncoderBackend, MediaType, RtcError,
 };
 
 #[derive(Debug, Clone)]
@@ -114,11 +114,83 @@ impl PeerConnectionFactory {
     pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities {
         self.handle.get_rtp_receiver_capabilities(media_type)
     }
+
+    /// SDP codec names (`VP8`, `VP9`, `AV1`, `H264`, `H265`) that `backend` can
+    /// produce with this factory's video encoders.
+    ///
+    /// For [`VideoEncoderBackend::Auto`] this is every codec a real encoder on
+    /// this host produces, which is also what a sender falls back to when its
+    /// requested backend is unavailable. For [`VideoEncoderBackend::PreEncoded`]
+    /// it is every codec the pass-through can forward, whether or not a local
+    /// encoder exists for it.
+    ///
+    /// ```
+    /// use libwebrtc::{peer_connection_factory::PeerConnectionFactory, rtp_sender::VideoEncoderBackend};
+    ///
+    /// let factory = PeerConnectionFactory::default();
+    /// let codecs = factory.video_encoder_codecs(VideoEncoderBackend::Software);
+    /// assert!(codecs.iter().any(|codec| codec == "VP8"));
+    /// ```
+    pub fn video_encoder_codecs(&self, backend: VideoEncoderBackend) -> Vec<String> {
+        self.handle.video_encoder_codecs(backend)
+    }
 }
 
 #[cfg(all(test, not(target_arch = "wasm32")))]
 mod tests {
     use super::PeerConnectionFactory;
+    use crate::rtp_sender::VideoEncoderBackend;
+
+    fn has_codec(codecs: &[String], name: &str) -> bool {
+        codecs.iter().any(|codec| codec.eq_ignore_ascii_case(name))
+    }
+
+    #[test]
+    fn pre_encoded_backend_forwards_every_codec_on_every_host() {
+        // Pre-encoded publishing must not depend on a real encoder existing
+        // for the codec, so the pass-through set is the same on every host.
+        let codecs =
+            PeerConnectionFactory::default().video_encoder_codecs(VideoEncoderBackend::PreEncoded);
+        for name in ["VP8", "VP9", "AV1", "H264", "H265"] {
+            assert!(has_codec(&codecs, name), "pass-through is missing {name}: {codecs:?}");
+        }
+        // HEVC is reported under its canonical H265 name only.
+        assert!(!has_codec(&codecs, "HEVC"), "{codecs:?}");
+    }
+
+    #[test]
+    fn software_backend_has_no_h265_encoder() {
+        // libvpx/libaom/OpenH264 never produce H265; the software set is
+        // what a normal session falls back to when no hardware encoder exists.
+        let codecs =
+            PeerConnectionFactory::default().video_encoder_codecs(VideoEncoderBackend::Software);
+        assert!(has_codec(&codecs, "VP8"), "{codecs:?}");
+        assert!(!has_codec(&codecs, "H265"), "{codecs:?}");
+    }
+
+    #[test]
+    fn auto_backend_is_a_superset_of_software() {
+        let factory = PeerConnectionFactory::default();
+        let auto = factory.video_encoder_codecs(VideoEncoderBackend::Auto);
+        let software = factory.video_encoder_codecs(VideoEncoderBackend::Software);
+        for codec in &software {
+            assert!(has_codec(&auto, codec), "auto is missing {codec}: {auto:?}");
+        }
+    }
+
+    #[test]
+    fn video_encoder_codecs_are_released_with_their_factory() {
+        // The codec query borrows the encoder factory the peer connection
+        // factory owns, so repeated create/query/drop must fully tear the
+        // native factory down each time instead of retaining encoders.
+        for _ in 0..3 {
+            let factory = PeerConnectionFactory::default();
+            assert!(has_codec(&factory.video_encoder_codecs(VideoEncoderBackend::Auto), "VP8"));
+            let native = factory.handle.sys_handle.downgrade();
+            drop(factory);
+            assert!(native.upgrade().is_null(), "peer connection factory outlived its owner");
+        }
+    }
 
     #[test]
     fn zero_playout_delay_factory_uses_force_playout_delay_field_trial() {

livekit/src/room/options.rs

@@ -403,6 +403,35 @@ pub(crate) fn video_quality_from_i32_or_default(quality: i32) -> proto::VideoQua
     proto::VideoQuality::try_from(quality).unwrap_or(DEFAULT_VIDEO_QUALITY)
 }
 
+/// Backend whose codecs a sender configured with `backend` can negotiate.
+///
+/// A pre-encoded sender can only forward its own frames, while any other sender
+/// falls back to whichever real encoder on the host produces the codec.
+pub(crate) fn codec_source_backend(backend: VideoEncoderBackend) -> VideoEncoderBackend {
+    if backend == VideoEncoderBackend::PreEncoded {
+        VideoEncoderBackend::PreEncoded
+    } else {
+        VideoEncoderBackend::Auto
+    }
+}
+
+/// Codec a video sender should negotiate for `requested`, given the SDP codec
+/// names its [`codec_source_backend`] can produce.
+///
+/// A raw sender falls back to VP8, which every host encodes in software, rather
+/// than negotiate a codec it cannot encode. Returns `None` for a pre-encoded
+/// sender whose codec cannot be forwarded, since it cannot transcode.
+pub(crate) fn select_video_codec(
+    requested: VideoCodec,
+    backend: VideoEncoderBackend,
+    producible: &[String],
+) -> Option<VideoCodec> {
+    if producible.iter().any(|codec| codec.eq_ignore_ascii_case(requested.as_str())) {
+        return Some(requested);
+    }
+    (backend != VideoEncoderBackend::PreEncoded).then_some(VideoCodec::VP8)
+}
+
 pub fn video_layers_from_encodings(
     width: u32,
     height: u32,
@@ -556,16 +585,80 @@ pub mod screenshare {
 #[cfg(test)]
 mod tests {
     use super::{
-        get_default_degradation_preference, DegradationPreference, TrackPublishOptions,
-        VideoEncoderBackend,
+        codec_source_backend, get_default_degradation_preference, select_video_codec,
+        DegradationPreference, TrackPublishOptions, VideoCodec, VideoEncoderBackend,
     };
     use crate::prelude::TrackSource;
 
+    fn codecs(names: &[&str]) -> Vec<String> {
+        names.iter().map(|name| name.to_string()).collect()
+    }
+
     #[test]
     fn track_publish_options_default_encoder_is_auto() {
         assert_eq!(TrackPublishOptions::default().video_encoder, VideoEncoderBackend::Auto);
     }
 
+    #[test]
+    fn only_pre_encoded_senders_negotiate_pass_through_codecs() {
+        assert_eq!(
+            codec_source_backend(VideoEncoderBackend::PreEncoded),
+            VideoEncoderBackend::PreEncoded
+        );
+        for backend in [
+            VideoEncoderBackend::Auto,
+            VideoEncoderBackend::Software,
+            VideoEncoderBackend::Hardware,
+            VideoEncoderBackend::Nvenc,
+        ] {
+            assert_eq!(codec_source_backend(backend), VideoEncoderBackend::Auto);
+        }
+    }
+
+    #[test]
+    fn producible_codec_is_kept() {
+        let host = codecs(&["VP8", "H264", "H265"]);
+        assert_eq!(
+            select_video_codec(VideoCodec::H265, VideoEncoderBackend::Auto, &host),
+            Some(VideoCodec::H265)
+        );
+        // SDP names are case-insensitive.
+        assert_eq!(
+            select_video_codec(VideoCodec::H264, VideoEncoderBackend::Auto, &codecs(&["h264"])),
+            Some(VideoCodec::H264)
+        );
+    }
+
+    #[test]
+    fn raw_sender_falls_back_to_vp8_without_an_encoder() {
+        let host = codecs(&["VP8", "VP9", "AV1"]);
+        assert_eq!(
+            select_video_codec(VideoCodec::H265, VideoEncoderBackend::Auto, &host),
+            Some(VideoCodec::VP8)
+        );
+        assert_eq!(
+            select_video_codec(VideoCodec::H265, VideoEncoderBackend::Hardware, &host),
+            Some(VideoCodec::VP8)
+        );
+    }
+
+    #[test]
+    fn pre_encoded_sender_never_switches_codec() {
+        let pass_through = codecs(&["VP8", "VP9", "AV1", "H264", "H265"]);
+        assert_eq!(
+            select_video_codec(VideoCodec::H265, VideoEncoderBackend::PreEncoded, &pass_through),
+            Some(VideoCodec::H265)
+        );
+        assert_eq!(
+            select_video_codec(
+                VideoCodec::H265,
+                VideoEncoderBackend::PreEncoded,
+                &codecs(&["VP8"])
+            ),
+            None
+        );
+    }
+
     #[test]
     fn degradation_preference_defaults_to_none() {
         assert_eq!(TrackPublishOptions::default().degradation_preference, None);

livekit/src/room/participant/local_participant.rs

@@ -64,6 +64,29 @@
     !options.frame_metadata_features.is_empty() || has_publish_timing_subscribers
 }
 
+/// Codec a video track published with `options` can actually produce on this host.
+fn negotiable_video_codec(options: &TrackPublishOptions) -> RoomResult<options::VideoCodec> {
+    let requested = options.video_codec;
+    let producible = LkRuntime::instance()
+        .pc_factory()
+        .video_encoder_codecs(options::codec_source_backend(options.video_encoder));
+    let Some(codec) = options::select_video_codec(requested, options.video_encoder, &producible)
+    else {
+        return Err(RoomError::Internal(format!(
+            "pre-encoded video publishing cannot forward codec {}",
+            requested.as_str()
+        )));
+    };
+    if codec != requested {
+        log::warn!(
+            "no video encoder on this host can produce {}; publishing {} instead",
+            requested.as_str(),
+            codec.as_str()
+        );
+    }
+    Ok(codec)
+}
+
 #[derive(Default)]
 struct LocalEvents {
     local_track_published: Mutex<Option<LocalTrackPublishedHandler>>,
@@ -376,9 +399,16 @@
     async fn publish_track_with_video_send_encodings(
         &self,
         track: LocalTrack,
-        options: TrackPublishOptions,
+        mut options: TrackPublishOptions,
         video_send_encodings: Option<Vec<RtpEncodingParameters>>,
     ) -> RoomResult<LocalTrackPublication> {
+        if let LocalTrack::Video(_) = &track {
+            // Settle the codec before announcing the track, so the server,
+            // the encodings, and a later republish all use the codec this
+            // sender will actually produce.
+            options.video_codec = negotiable_video_codec(&options)?;
+        }
+
         let disable_red = self.local.encryption_type != EncryptionType::None || !options.red;
 
         let mut req = proto::AddTrackRequest {
@@ -387,7 +417,7 @@
             r#type: proto::TrackType::from(track.kind()) as i32,
             muted: track.is_muted(),
             source: proto::TrackSource::from(options.source) as i32,
             disable_dtx: !options.dtx,
             disable_red,
             encryption: proto::encryption::Type::from(self.local.encryption_type) as i32,
             stream: options.stream.clone(),

livekit/src/rtc_engine/rtc_session.rs

@@ -2064,20 +2064,29 @@ impl SessionInner {
         if track.kind() == TrackKind::Video {
             transceiver.sender().set_video_encoder_backend(options.video_encoder);
 
-            let capabilities = LkRuntime::instance().pc_factory().get_rtp_sender_capabilities(
-                match track.kind() {
-                    TrackKind::Video => MediaType::Video,
-                    TrackKind::Audio => MediaType::Audio,
-                },
-            );
+            // LocalParticipant already settled on a codec this sender can
+            // produce. Offer only that codec, and never fall back to an empty
+            // preference list: libwebrtc would then offer every codec the
+            // encoder factory lists, including pass-through-only codecs a raw
+            // sender cannot encode.
+            let lk_runtime = LkRuntime::instance();
+            let pc_factory = lk_runtime.pc_factory();
+            let producible = pc_factory
+                .video_encoder_codecs(crate::options::codec_source_backend(options.video_encoder));
+            let is_producible = |mime_type: &str| {
+                producible
+                    .iter()
+                    .any(|codec| mime_type == format!("video/{}", codec.to_lowercase()))
+            };
+            let requested_mime_type = format!("video/{}", options.video_codec.as_str());
 
             let mut matched = Vec::new();
             let mut partial_matched = Vec::new();
-            let mut unmatched = Vec::new();
+            let mut producible_codecs = Vec::new();
 
-            for codec in capabilities.codecs {
+            for codec in pc_factory.get_rtp_sender_capabilities(MediaType::Video).codecs {
                 let mime_type = codec.mime_type.to_lowercase();
-                if mime_type == format!("video/{}", options.video_codec.as_str()) {
+                if mime_type == requested_mime_type {
                     if let Some(sdp_fmtp_line) = codec.sdp_fmtp_line.as_ref() {
                         // for h264 codecs that have sdpFmtpLine available, use only if the
                         // profile-level-id is 42e01f for cross-browser compatibility
@@ -2087,12 +2096,18 @@ impl SessionInner {
                         }
                     }
                     partial_matched.push(codec);
-                } else {
-                    unmatched.push(codec);
+                } else if is_producible(&mime_type) {
+                    producible_codecs.push(codec);
                 }
             }
 
             matched.append(&mut partial_matched);
+            if matched.is_empty() {
+                log::warn!(
+                    "no sender capability for {requested_mime_type}; offering the codecs this sender can produce"
+                );
+                matched = producible_codecs;
+            }
 
             transceiver.set_codec_preferences(matched)?;
         }