From e7785db1159abd68ec22d549643835ef22062924 Mon Sep 17 00:00:00 2001 From: Stephen DeRosa Date: Tue, 22 Sep 2026 18:37:08 -0600 Subject: [PATCH 1/3] h265 support for jetson --- ...s_through_codecs_without_a_real_encoder.md | 9 ++ libwebrtc/src/native/rtp_sender.rs | 38 ++++++++ libwebrtc/src/rtp_sender.rs | 11 +++ livekit/src/rtc_engine/rtc_session.rs | 33 ++++++- .../include/livekit/video_encoder_factory.h | 9 ++ webrtc-sys/include/livekit/webrtc.h | 2 + webrtc-sys/src/video_encoder_factory.cpp | 97 ++++++++++++++++--- webrtc-sys/src/webrtc.rs | 1 + 8 files changed, 187 insertions(+), 13 deletions(-) create mode 100644 .changeset/advertise_pass_through_codecs_without_a_real_encoder.md diff --git a/.changeset/advertise_pass_through_codecs_without_a_real_encoder.md b/.changeset/advertise_pass_through_codecs_without_a_real_encoder.md new file mode 100644 index 000000000..78d1b7131 --- /dev/null +++ b/.changeset/advertise_pass_through_codecs_without_a_real_encoder.md @@ -0,0 +1,9 @@ +--- +webrtc-sys: patch +libwebrtc: minor +livekit: patch +livekit-ffi: patch +livekit-uniffi: patch +--- + +Pre-encoded video publishing no longer requires a real encoder for the codec on the publishing host, so pre-encoded H265 now works on Linux arm64 builds without the Jetson encoder, and `VideoEncoderBackend::supported_codecs` reports which codecs each backend can produce. diff --git a/libwebrtc/src/native/rtp_sender.rs b/libwebrtc/src/native/rtp_sender.rs index e90b43a75..c6ef31db3 100644 --- a/libwebrtc/src/native/rtp_sender.rs +++ b/libwebrtc/src/native/rtp_sender.rs @@ -119,6 +119,10 @@ pub fn video_encoder_backend_list() -> Vec { sys_webrtc::ffi::video_encoder_backend_list().into_iter().map(Into::into).collect() } +pub fn video_encoder_backend_codecs(backend: VideoEncoderBackend) -> Vec { + sys_webrtc::ffi::video_encoder_backend_codecs(backend.into()).into_iter().collect() +} + #[cfg(test)] mod tests { use super::{sys_webrtc, VideoEncoderBackend}; @@ -140,4 +144,38 @@ mod tests { assert_eq!(VideoEncoderBackend::from(expected), backend); } } + + fn has_codec(codecs: &[String], name: &str) -> bool { + codecs.iter().any(|codec| codec.eq_ignore_ascii_case(name)) + } + + #[test] + fn pre_encoded_backend_forwards_every_codec_on_every_host() { + // Pre-encoded publishing must not depend on a real encoder existing + // for the codec, so the pass-through set is the same on every host. + let codecs = super::video_encoder_backend_codecs(VideoEncoderBackend::PreEncoded); + for name in ["VP8", "VP9", "AV1", "H264", "H265"] { + assert!(has_codec(&codecs, name), "pass-through is missing {name}: {codecs:?}"); + } + // HEVC is reported under its canonical H265 name only. + assert!(!has_codec(&codecs, "HEVC"), "{codecs:?}"); + } + + #[test] + fn software_backend_has_no_h265_encoder() { + // libvpx/libaom/OpenH264 never produce H265; the software set is + // what a normal session falls back to when no hardware encoder exists. + let codecs = super::video_encoder_backend_codecs(VideoEncoderBackend::Software); + assert!(has_codec(&codecs, "VP8"), "{codecs:?}"); + assert!(!has_codec(&codecs, "H265"), "{codecs:?}"); + } + + #[test] + fn auto_backend_is_a_superset_of_software() { + let auto = super::video_encoder_backend_codecs(VideoEncoderBackend::Auto); + let software = super::video_encoder_backend_codecs(VideoEncoderBackend::Software); + for codec in &software { + assert!(has_codec(&auto, codec), "auto is missing {codec}: {auto:?}"); + } + } } diff --git a/libwebrtc/src/rtp_sender.rs b/libwebrtc/src/rtp_sender.rs index 1bdf96add..52b199c57 100644 --- a/libwebrtc/src/rtp_sender.rs +++ b/libwebrtc/src/rtp_sender.rs @@ -56,6 +56,17 @@ impl VideoEncoderBackend { pub fn list_available() -> impl IntoIterator { imp_rs::video_encoder_backend_list() } + + /// SDP codec names (`VP8`, `VP9`, `AV1`, `H264`, `H265`) that this backend + /// can produce on the current host. + /// + /// `Auto` is the union of every real encoder available here, which is also + /// the set a sender falls back to when its requested backend is missing. + /// `PreEncoded` lists the codecs the pass-through can forward, and does not + /// depend on any real encoder being present. + pub fn supported_codecs(self) -> impl IntoIterator { + imp_rs::video_encoder_backend_codecs(self) + } } #[derive(Clone)] diff --git a/livekit/src/rtc_engine/rtc_session.rs b/livekit/src/rtc_engine/rtc_session.rs index 83d5540cc..ac8e61ca4 100644 --- a/livekit/src/rtc_engine/rtc_session.rs +++ b/livekit/src/rtc_engine/rtc_session.rs @@ -2064,6 +2064,37 @@ impl SessionInner { if track.kind() == TrackKind::Video { transceiver.sender().set_video_encoder_backend(options.video_encoder); + // The encoder factory advertises every pass-through codec so that + // pre-encoded publishing works without a real encoder for it. A + // normal session must still only negotiate a codec some real + // encoder can produce; otherwise the sender would negotiate a codec + // it cannot encode. A pre-encoded sender must negotiate exactly the + // codec of its frames, since the pass-through cannot transcode. + let requested_codec = options.video_codec.as_str(); + let producing_backend = if options.video_encoder == VideoEncoderBackend::PreEncoded { + VideoEncoderBackend::PreEncoded + } else { + VideoEncoderBackend::Auto + }; + let codec_producible = producing_backend + .supported_codecs() + .into_iter() + .any(|codec| codec.eq_ignore_ascii_case(requested_codec)); + if !codec_producible { + if producing_backend == VideoEncoderBackend::PreEncoded { + return Err(EngineError::Internal( + format!( + "pre-encoded video publishing cannot forward codec {requested_codec}" + ) + .into(), + )); + } + log::warn!( + "no video encoder on this host can produce {requested_codec}; \ + falling back to the default codec" + ); + } + let capabilities = LkRuntime::instance().pc_factory().get_rtp_sender_capabilities( match track.kind() { TrackKind::Video => MediaType::Video, @@ -2077,7 +2108,7 @@ impl SessionInner { for codec in capabilities.codecs { let mime_type = codec.mime_type.to_lowercase(); - if mime_type == format!("video/{}", options.video_codec.as_str()) { + if codec_producible && mime_type == format!("video/{requested_codec}") { if let Some(sdp_fmtp_line) = codec.sdp_fmtp_line.as_ref() { // for h264 codecs that have sdpFmtpLine available, use only if the // profile-level-id is 42e01f for cross-browser compatibility diff --git a/webrtc-sys/include/livekit/video_encoder_factory.h b/webrtc-sys/include/livekit/video_encoder_factory.h index 36c72f9b1..3f7b11854 100644 --- a/webrtc-sys/include/livekit/video_encoder_factory.h +++ b/webrtc-sys/include/livekit/video_encoder_factory.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "api/video_codecs/video_encoder.h" @@ -48,6 +49,8 @@ class VideoEncoderFactory : public webrtc::VideoEncoderFactory { std::unique_ptr Create( const webrtc::Environment& env, const webrtc::SdpVideoFormat& format) override; + std::vector BackendCodecs(VideoEncoderBackend backend) const; + private: std::vector factories_; }; @@ -66,6 +69,12 @@ class VideoEncoderFactory : public webrtc::VideoEncoderFactory { std::unique_ptr Create( const webrtc::Environment& env, const webrtc::SdpVideoFormat& format) override; + // SDP codec names (VP8, VP9, AV1, H264, H265) that the given backend can + // produce on this host. Auto is the union of every real encoder, which is + // also the set Create() falls back to when a requested backend is missing. + // PreEncoded lists the codecs the pass-through can forward. + std::vector BackendCodecs(VideoEncoderBackend backend) const; + private: std::unique_ptr internal_factory_; }; diff --git a/webrtc-sys/include/livekit/webrtc.h b/webrtc-sys/include/livekit/webrtc.h index 64fd496ef..2a1e20762 100644 --- a/webrtc-sys/include/livekit/webrtc.h +++ b/webrtc-sys/include/livekit/webrtc.h @@ -116,5 +116,7 @@ std::unique_ptr new_log_sink( rust::String create_random_uuid(); rust::Vec video_encoder_backend_list(); +rust::Vec video_encoder_backend_codecs( + VideoEncoderBackend backend); } // namespace livekit_ffi diff --git a/webrtc-sys/src/video_encoder_factory.cpp b/webrtc-sys/src/video_encoder_factory.cpp index e18449c87..e60e350a0 100644 --- a/webrtc-sys/src/video_encoder_factory.cpp +++ b/webrtc-sys/src/video_encoder_factory.cpp @@ -393,23 +393,26 @@ VideoEncoderFactory::InternalFactory::GetSupportedFormats() const { supported_formats.end()); } - // The pass-through factory would otherwise advertise codecs no real - // encoder implements (e.g. H265 on desktops); a normal session - // negotiating such a codec would end up with a sender that cannot create - // an encoder. Only advertise pass-through formats for codecs some real - // encoder already supports. - const size_t real_format_count = formats.size(); + // Pre-encoded publishing forwards access units the application already + // encoded, so it must not depend on this host having a real encoder for + // the codec (e.g. H265 on a Linux box without a hardware encoder). Make + // every pass-through codec negotiable by adding one entry for each codec + // no real encoder listed. The pass-through matches by codec name at + // Create(), so the real encoder's own entry serves it when one exists. A + // normal session is kept from negotiating a codec no real encoder + // implements by the codec-preference selection in the Rust publish path, + // which consults BackendCodecs(). for (const auto& backend_factory : factories_) { if (backend_factory.backend != VideoEncoderBackend::PreEncoded) { continue; } for (const auto& format : backend_factory.factory->GetSupportedFormats()) { - const bool codec_available = std::any_of( - formats.begin(), formats.begin() + real_format_count, - [&](const webrtc::SdpVideoFormat& existing) { - return IsSameCodecName(existing.name, format.name); - }); - if (codec_available) { + const bool codec_listed = + std::any_of(formats.begin(), formats.end(), + [&](const webrtc::SdpVideoFormat& existing) { + return IsSameCodecName(existing.name, format.name); + }); + if (!codec_listed) { formats.push_back(format); } } @@ -417,6 +420,58 @@ VideoEncoderFactory::InternalFactory::GetSupportedFormats() const { return formats; } +std::vector VideoEncoderFactory::InternalFactory::BackendCodecs( + VideoEncoderBackend backend) const { + std::vector codecs; + auto add_formats = [&](const std::vector& formats) { + for (const auto& format : formats) { + // Report one canonical name per codec; HEVC is an alias of H265. + std::string name = format.name; + if (IsSameCodecName(name, "H265")) { + name = "H265"; + } + const bool listed = std::any_of(codecs.begin(), codecs.end(), + [&](const std::string& existing) { + return IsSameCodecName(existing, name); + }); + if (!listed) { + codecs.push_back(std::move(name)); + } + } + }; + + switch (backend) { + case VideoEncoderBackend::PreEncoded: + for (const auto& backend_factory : factories_) { + if (backend_factory.backend == VideoEncoderBackend::PreEncoded) { + add_formats(backend_factory.factory->GetSupportedFormats()); + } + } + break; + case VideoEncoderBackend::Software: + add_formats(Factory().GetSupportedFormats()); + break; + case VideoEncoderBackend::Auto: + // Everything a real encoder on this host can produce: the set Create() + // falls back to when a requested backend is unavailable. + for (const auto& backend_factory : factories_) { + if (IsAutomaticFallbackBackend(backend_factory.backend)) { + add_formats(backend_factory.factory->GetSupportedFormats()); + } + } + add_formats(Factory().GetSupportedFormats()); + break; + default: + for (const auto& backend_factory : factories_) { + if (BackendMatches(backend, backend_factory.backend)) { + add_formats(backend_factory.factory->GetSupportedFormats()); + } + } + break; + } + return codecs; +} + std::vector VideoEncoderFactory::InternalFactory::GetImplementations() const { std::vector formats; @@ -599,6 +654,24 @@ std::vector VideoEncoderFactory::GetSupportedFormats() return internal_factory_->GetSupportedFormats(); } +std::vector VideoEncoderFactory::BackendCodecs( + VideoEncoderBackend backend) const { + return internal_factory_->BackendCodecs(backend); +} + +rust::Vec video_encoder_backend_codecs( + VideoEncoderBackend backend) { + // Probing hardware encoders is not free; keep one query-only factory for + // the life of the process instead of building one per call. + static const VideoEncoderFactory* const query_factory = + new VideoEncoderFactory(); + rust::Vec codecs; + for (const auto& codec : query_factory->BackendCodecs(backend)) { + codecs.push_back(codec); + } + return codecs; +} + std::vector VideoEncoderFactory::GetImplementations() const { return internal_factory_->GetImplementations(); diff --git a/webrtc-sys/src/webrtc.rs b/webrtc-sys/src/webrtc.rs index fc7b7d099..ab4214f19 100644 --- a/webrtc-sys/src/webrtc.rs +++ b/webrtc-sys/src/webrtc.rs @@ -73,6 +73,7 @@ pub mod ffi { fn create_random_uuid() -> String; fn video_encoder_backend_list() -> Vec; + fn video_encoder_backend_codecs(backend: VideoEncoderBackend) -> Vec; fn new_log_sink(fnc: fn(String, LoggingSeverity)) -> UniquePtr; } } From 6b487b97c640fb7002a314920da3907c9f11240b Mon Sep 17 00:00:00 2001 From: Stephen DeRosa Date: Tue, 20 Oct 2026 06:59:02 +0800 Subject: [PATCH 2/3] Let pre-encoded senders send pass-through-only codecs safely Address review on #1451: - List pass-through-only codecs in GetImplementations(), last and untagged. libwebrtc only sends a negotiated codec that appears there, so pre-encoded H265 on a host without an H265 encoder failed with "No video codecs in common". A raw sender that still lands on such a codec fails encoder creation and libwebrtc falls back to another negotiated codec. - Settle a video track's codec in LocalParticipant before AddTrackRequest: a raw sender whose codec no local encoder produces publishes VP8, so the server, encodings, and republish agree with what is sent. create_sender no longer falls back to an empty preference list, which offered pass-through-only codecs to raw senders. - Replace the leaked process-wide query factory with PeerConnectionFactory::video_encoder_codecs, which queries the encoder factory the peer connection factory owns, and add a repeated create/query/drop test asserting the native factory is destroyed. - Add livekit-capture to the changeset. Co-Authored-By: Claude Opus 5.5 --- ...s_through_codecs_without_a_real_encoder.md | 3 +- .../src/native/peer_connection_factory.rs | 5 + libwebrtc/src/native/rtp_sender.rs | 38 -------- libwebrtc/src/peer_connection_factory.rs | 74 +++++++++++++- libwebrtc/src/rtp_sender.rs | 11 --- livekit/src/room/options.rs | 97 ++++++++++++++++++- .../src/room/participant/local_participant.rs | 32 +++++- livekit/src/rtc_engine/rtc_session.rs | 66 +++++-------- .../include/livekit/peer_connection_factory.h | 9 ++ webrtc-sys/include/livekit/webrtc.h | 2 - webrtc-sys/src/peer_connection_factory.cpp | 18 +++- webrtc-sys/src/peer_connection_factory.rs | 8 ++ webrtc-sys/src/video_encoder_factory.cpp | 44 ++++++--- webrtc-sys/src/webrtc.rs | 1 - 14 files changed, 292 insertions(+), 116 deletions(-) diff --git a/.changeset/advertise_pass_through_codecs_without_a_real_encoder.md b/.changeset/advertise_pass_through_codecs_without_a_real_encoder.md index 78d1b7131..ff6a323ea 100644 --- a/.changeset/advertise_pass_through_codecs_without_a_real_encoder.md +++ b/.changeset/advertise_pass_through_codecs_without_a_real_encoder.md @@ -4,6 +4,7 @@ libwebrtc: minor livekit: patch livekit-ffi: patch livekit-uniffi: patch +livekit-capture: patch --- -Pre-encoded video publishing no longer requires a real encoder for the codec on the publishing host, so pre-encoded H265 now works on Linux arm64 builds without the Jetson encoder, and `VideoEncoderBackend::supported_codecs` reports which codecs each backend can produce. +Pre-encoded video publishing no longer requires a local encoder for its codec, so pre-encoded H265 works on hosts without an H265 encoder, a camera track requesting a codec no local encoder produces now publishes VP8 instead of a codec it cannot encode, and `PeerConnectionFactory::video_encoder_codecs` reports which codecs each encoder backend can produce. diff --git a/libwebrtc/src/native/peer_connection_factory.rs b/libwebrtc/src/native/peer_connection_factory.rs index 8f1cd4993..c79f3af75 100644 --- a/libwebrtc/src/native/peer_connection_factory.rs +++ b/libwebrtc/src/native/peer_connection_factory.rs @@ -26,6 +26,7 @@ use crate::{ peer_connection::PeerConnection, peer_connection_factory::RtcConfiguration, rtp_parameters::RtpCapabilities, + rtp_sender::VideoEncoderBackend, video_source::native::NativeVideoSource, video_track::RtcVideoTrack, MediaType, RtcError, @@ -141,6 +142,10 @@ impl PeerConnectionFactory { self.sys_handle.rtp_receiver_capabilities(media_type.into()).into() } + pub fn video_encoder_codecs(&self, backend: VideoEncoderBackend) -> Vec { + self.sys_handle.video_encoder_backend_codecs(backend.into()) + } + // ===== Device Management Methods ===== /// Get the number of playout (output) devices diff --git a/libwebrtc/src/native/rtp_sender.rs b/libwebrtc/src/native/rtp_sender.rs index c6ef31db3..e90b43a75 100644 --- a/libwebrtc/src/native/rtp_sender.rs +++ b/libwebrtc/src/native/rtp_sender.rs @@ -119,10 +119,6 @@ pub fn video_encoder_backend_list() -> Vec { sys_webrtc::ffi::video_encoder_backend_list().into_iter().map(Into::into).collect() } -pub fn video_encoder_backend_codecs(backend: VideoEncoderBackend) -> Vec { - sys_webrtc::ffi::video_encoder_backend_codecs(backend.into()).into_iter().collect() -} - #[cfg(test)] mod tests { use super::{sys_webrtc, VideoEncoderBackend}; @@ -144,38 +140,4 @@ mod tests { assert_eq!(VideoEncoderBackend::from(expected), backend); } } - - fn has_codec(codecs: &[String], name: &str) -> bool { - codecs.iter().any(|codec| codec.eq_ignore_ascii_case(name)) - } - - #[test] - fn pre_encoded_backend_forwards_every_codec_on_every_host() { - // Pre-encoded publishing must not depend on a real encoder existing - // for the codec, so the pass-through set is the same on every host. - let codecs = super::video_encoder_backend_codecs(VideoEncoderBackend::PreEncoded); - for name in ["VP8", "VP9", "AV1", "H264", "H265"] { - assert!(has_codec(&codecs, name), "pass-through is missing {name}: {codecs:?}"); - } - // HEVC is reported under its canonical H265 name only. - assert!(!has_codec(&codecs, "HEVC"), "{codecs:?}"); - } - - #[test] - fn software_backend_has_no_h265_encoder() { - // libvpx/libaom/OpenH264 never produce H265; the software set is - // what a normal session falls back to when no hardware encoder exists. - let codecs = super::video_encoder_backend_codecs(VideoEncoderBackend::Software); - assert!(has_codec(&codecs, "VP8"), "{codecs:?}"); - assert!(!has_codec(&codecs, "H265"), "{codecs:?}"); - } - - #[test] - fn auto_backend_is_a_superset_of_software() { - let auto = super::video_encoder_backend_codecs(VideoEncoderBackend::Auto); - let software = super::video_encoder_backend_codecs(VideoEncoderBackend::Software); - for codec in &software { - assert!(has_codec(&auto, codec), "auto is missing {codec}: {auto:?}"); - } - } } diff --git a/libwebrtc/src/peer_connection_factory.rs b/libwebrtc/src/peer_connection_factory.rs index 1eb0392e3..41d2ef690 100644 --- a/libwebrtc/src/peer_connection_factory.rs +++ b/libwebrtc/src/peer_connection_factory.rs @@ -16,7 +16,7 @@ use std::fmt::Debug; use crate::{ imp::peer_connection_factory as imp_pcf, peer_connection::PeerConnection, - rtp_parameters::RtpCapabilities, MediaType, RtcError, + rtp_parameters::RtpCapabilities, rtp_sender::VideoEncoderBackend, MediaType, RtcError, }; #[derive(Debug, Clone)] @@ -114,11 +114,83 @@ impl PeerConnectionFactory { pub fn get_rtp_receiver_capabilities(&self, media_type: MediaType) -> RtpCapabilities { self.handle.get_rtp_receiver_capabilities(media_type) } + + /// SDP codec names (`VP8`, `VP9`, `AV1`, `H264`, `H265`) that `backend` can + /// produce with this factory's video encoders. + /// + /// For [`VideoEncoderBackend::Auto`] this is every codec a real encoder on + /// this host produces, which is also what a sender falls back to when its + /// requested backend is unavailable. For [`VideoEncoderBackend::PreEncoded`] + /// it is every codec the pass-through can forward, whether or not a local + /// encoder exists for it. + /// + /// ``` + /// use libwebrtc::{peer_connection_factory::PeerConnectionFactory, rtp_sender::VideoEncoderBackend}; + /// + /// let factory = PeerConnectionFactory::default(); + /// let codecs = factory.video_encoder_codecs(VideoEncoderBackend::Software); + /// assert!(codecs.iter().any(|codec| codec == "VP8")); + /// ``` + pub fn video_encoder_codecs(&self, backend: VideoEncoderBackend) -> Vec { + self.handle.video_encoder_codecs(backend) + } } #[cfg(all(test, not(target_arch = "wasm32")))] mod tests { use super::PeerConnectionFactory; + use crate::rtp_sender::VideoEncoderBackend; + + fn has_codec(codecs: &[String], name: &str) -> bool { + codecs.iter().any(|codec| codec.eq_ignore_ascii_case(name)) + } + + #[test] + fn pre_encoded_backend_forwards_every_codec_on_every_host() { + // Pre-encoded publishing must not depend on a real encoder existing + // for the codec, so the pass-through set is the same on every host. + let codecs = + PeerConnectionFactory::default().video_encoder_codecs(VideoEncoderBackend::PreEncoded); + for name in ["VP8", "VP9", "AV1", "H264", "H265"] { + assert!(has_codec(&codecs, name), "pass-through is missing {name}: {codecs:?}"); + } + // HEVC is reported under its canonical H265 name only. + assert!(!has_codec(&codecs, "HEVC"), "{codecs:?}"); + } + + #[test] + fn software_backend_has_no_h265_encoder() { + // libvpx/libaom/OpenH264 never produce H265; the software set is + // what a normal session falls back to when no hardware encoder exists. + let codecs = + PeerConnectionFactory::default().video_encoder_codecs(VideoEncoderBackend::Software); + assert!(has_codec(&codecs, "VP8"), "{codecs:?}"); + assert!(!has_codec(&codecs, "H265"), "{codecs:?}"); + } + + #[test] + fn auto_backend_is_a_superset_of_software() { + let factory = PeerConnectionFactory::default(); + let auto = factory.video_encoder_codecs(VideoEncoderBackend::Auto); + let software = factory.video_encoder_codecs(VideoEncoderBackend::Software); + for codec in &software { + assert!(has_codec(&auto, codec), "auto is missing {codec}: {auto:?}"); + } + } + + #[test] + fn video_encoder_codecs_are_released_with_their_factory() { + // The codec query borrows the encoder factory the peer connection + // factory owns, so repeated create/query/drop must fully tear the + // native factory down each time instead of retaining encoders. + for _ in 0..3 { + let factory = PeerConnectionFactory::default(); + assert!(has_codec(&factory.video_encoder_codecs(VideoEncoderBackend::Auto), "VP8")); + let native = factory.handle.sys_handle.downgrade(); + drop(factory); + assert!(native.upgrade().is_null(), "peer connection factory outlived its owner"); + } + } #[test] fn zero_playout_delay_factory_uses_force_playout_delay_field_trial() { diff --git a/libwebrtc/src/rtp_sender.rs b/libwebrtc/src/rtp_sender.rs index 52b199c57..1bdf96add 100644 --- a/libwebrtc/src/rtp_sender.rs +++ b/libwebrtc/src/rtp_sender.rs @@ -56,17 +56,6 @@ impl VideoEncoderBackend { pub fn list_available() -> impl IntoIterator { imp_rs::video_encoder_backend_list() } - - /// SDP codec names (`VP8`, `VP9`, `AV1`, `H264`, `H265`) that this backend - /// can produce on the current host. - /// - /// `Auto` is the union of every real encoder available here, which is also - /// the set a sender falls back to when its requested backend is missing. - /// `PreEncoded` lists the codecs the pass-through can forward, and does not - /// depend on any real encoder being present. - pub fn supported_codecs(self) -> impl IntoIterator { - imp_rs::video_encoder_backend_codecs(self) - } } #[derive(Clone)] diff --git a/livekit/src/room/options.rs b/livekit/src/room/options.rs index f443f6087..70d55a9b7 100644 --- a/livekit/src/room/options.rs +++ b/livekit/src/room/options.rs @@ -403,6 +403,35 @@ pub(crate) fn video_quality_from_i32_or_default(quality: i32) -> proto::VideoQua proto::VideoQuality::try_from(quality).unwrap_or(DEFAULT_VIDEO_QUALITY) } +/// Backend whose codecs a sender configured with `backend` can negotiate. +/// +/// A pre-encoded sender can only forward its own frames, while any other sender +/// falls back to whichever real encoder on the host produces the codec. +pub(crate) fn codec_source_backend(backend: VideoEncoderBackend) -> VideoEncoderBackend { + if backend == VideoEncoderBackend::PreEncoded { + VideoEncoderBackend::PreEncoded + } else { + VideoEncoderBackend::Auto + } +} + +/// Codec a video sender should negotiate for `requested`, given the SDP codec +/// names its [`codec_source_backend`] can produce. +/// +/// A raw sender falls back to VP8, which every host encodes in software, rather +/// than negotiate a codec it cannot encode. Returns `None` for a pre-encoded +/// sender whose codec cannot be forwarded, since it cannot transcode. +pub(crate) fn select_video_codec( + requested: VideoCodec, + backend: VideoEncoderBackend, + producible: &[String], +) -> Option { + if producible.iter().any(|codec| codec.eq_ignore_ascii_case(requested.as_str())) { + return Some(requested); + } + (backend != VideoEncoderBackend::PreEncoded).then_some(VideoCodec::VP8) +} + pub fn video_layers_from_encodings( width: u32, height: u32, @@ -556,16 +585,80 @@ pub mod screenshare { #[cfg(test)] mod tests { use super::{ - get_default_degradation_preference, DegradationPreference, TrackPublishOptions, - VideoEncoderBackend, + codec_source_backend, get_default_degradation_preference, select_video_codec, + DegradationPreference, TrackPublishOptions, VideoCodec, VideoEncoderBackend, }; use crate::prelude::TrackSource; + fn codecs(names: &[&str]) -> Vec { + names.iter().map(|name| name.to_string()).collect() + } + #[test] fn track_publish_options_default_encoder_is_auto() { assert_eq!(TrackPublishOptions::default().video_encoder, VideoEncoderBackend::Auto); } + #[test] + fn only_pre_encoded_senders_negotiate_pass_through_codecs() { + assert_eq!( + codec_source_backend(VideoEncoderBackend::PreEncoded), + VideoEncoderBackend::PreEncoded + ); + for backend in [ + VideoEncoderBackend::Auto, + VideoEncoderBackend::Software, + VideoEncoderBackend::Hardware, + VideoEncoderBackend::Nvenc, + ] { + assert_eq!(codec_source_backend(backend), VideoEncoderBackend::Auto); + } + } + + #[test] + fn producible_codec_is_kept() { + let host = codecs(&["VP8", "H264", "H265"]); + assert_eq!( + select_video_codec(VideoCodec::H265, VideoEncoderBackend::Auto, &host), + Some(VideoCodec::H265) + ); + // SDP names are case-insensitive. + assert_eq!( + select_video_codec(VideoCodec::H264, VideoEncoderBackend::Auto, &codecs(&["h264"])), + Some(VideoCodec::H264) + ); + } + + #[test] + fn raw_sender_falls_back_to_vp8_without_an_encoder() { + let host = codecs(&["VP8", "VP9", "AV1"]); + assert_eq!( + select_video_codec(VideoCodec::H265, VideoEncoderBackend::Auto, &host), + Some(VideoCodec::VP8) + ); + assert_eq!( + select_video_codec(VideoCodec::H265, VideoEncoderBackend::Hardware, &host), + Some(VideoCodec::VP8) + ); + } + + #[test] + fn pre_encoded_sender_never_switches_codec() { + let pass_through = codecs(&["VP8", "VP9", "AV1", "H264", "H265"]); + assert_eq!( + select_video_codec(VideoCodec::H265, VideoEncoderBackend::PreEncoded, &pass_through), + Some(VideoCodec::H265) + ); + assert_eq!( + select_video_codec( + VideoCodec::H265, + VideoEncoderBackend::PreEncoded, + &codecs(&["VP8"]) + ), + None + ); + } + #[test] fn degradation_preference_defaults_to_none() { assert_eq!(TrackPublishOptions::default().degradation_preference, None); diff --git a/livekit/src/room/participant/local_participant.rs b/livekit/src/room/participant/local_participant.rs index 2b8c38e08..a00251dae 100644 --- a/livekit/src/room/participant/local_participant.rs +++ b/livekit/src/room/participant/local_participant.rs @@ -64,6 +64,29 @@ fn needs_video_sender_transformer( !options.frame_metadata_features.is_empty() || has_publish_timing_subscribers } +/// Codec a video track published with `options` can actually produce on this host. +fn negotiable_video_codec(options: &TrackPublishOptions) -> RoomResult { + let requested = options.video_codec; + let producible = LkRuntime::instance() + .pc_factory() + .video_encoder_codecs(options::codec_source_backend(options.video_encoder)); + let Some(codec) = options::select_video_codec(requested, options.video_encoder, &producible) + else { + return Err(RoomError::Internal(format!( + "pre-encoded video publishing cannot forward codec {}", + requested.as_str() + ))); + }; + if codec != requested { + log::warn!( + "no video encoder on this host can produce {}; publishing {} instead", + requested.as_str(), + codec.as_str() + ); + } + Ok(codec) +} + #[derive(Default)] struct LocalEvents { local_track_published: Mutex>, @@ -376,9 +399,16 @@ impl LocalParticipant { async fn publish_track_with_video_send_encodings( &self, track: LocalTrack, - options: TrackPublishOptions, + mut options: TrackPublishOptions, video_send_encodings: Option>, ) -> RoomResult { + if let LocalTrack::Video(_) = &track { + // Settle the codec before announcing the track, so the server, + // the encodings, and a later republish all use the codec this + // sender will actually produce. + options.video_codec = negotiable_video_codec(&options)?; + } + let disable_red = self.local.encryption_type != EncryptionType::None || !options.red; let mut req = proto::AddTrackRequest { diff --git a/livekit/src/rtc_engine/rtc_session.rs b/livekit/src/rtc_engine/rtc_session.rs index ac8e61ca4..83056e96f 100644 --- a/livekit/src/rtc_engine/rtc_session.rs +++ b/livekit/src/rtc_engine/rtc_session.rs @@ -2064,51 +2064,29 @@ impl SessionInner { if track.kind() == TrackKind::Video { transceiver.sender().set_video_encoder_backend(options.video_encoder); - // The encoder factory advertises every pass-through codec so that - // pre-encoded publishing works without a real encoder for it. A - // normal session must still only negotiate a codec some real - // encoder can produce; otherwise the sender would negotiate a codec - // it cannot encode. A pre-encoded sender must negotiate exactly the - // codec of its frames, since the pass-through cannot transcode. - let requested_codec = options.video_codec.as_str(); - let producing_backend = if options.video_encoder == VideoEncoderBackend::PreEncoded { - VideoEncoderBackend::PreEncoded - } else { - VideoEncoderBackend::Auto + // LocalParticipant already settled on a codec this sender can + // produce. Offer only that codec, and never fall back to an empty + // preference list: libwebrtc would then offer every codec the + // encoder factory lists, including pass-through-only codecs a raw + // sender cannot encode. + let lk_runtime = LkRuntime::instance(); + let pc_factory = lk_runtime.pc_factory(); + let producible = pc_factory + .video_encoder_codecs(crate::options::codec_source_backend(options.video_encoder)); + let is_producible = |mime_type: &str| { + producible + .iter() + .any(|codec| mime_type == format!("video/{}", codec.to_lowercase())) }; - let codec_producible = producing_backend - .supported_codecs() - .into_iter() - .any(|codec| codec.eq_ignore_ascii_case(requested_codec)); - if !codec_producible { - if producing_backend == VideoEncoderBackend::PreEncoded { - return Err(EngineError::Internal( - format!( - "pre-encoded video publishing cannot forward codec {requested_codec}" - ) - .into(), - )); - } - log::warn!( - "no video encoder on this host can produce {requested_codec}; \ - falling back to the default codec" - ); - } - - let capabilities = LkRuntime::instance().pc_factory().get_rtp_sender_capabilities( - match track.kind() { - TrackKind::Video => MediaType::Video, - TrackKind::Audio => MediaType::Audio, - }, - ); + let requested_mime_type = format!("video/{}", options.video_codec.as_str()); let mut matched = Vec::new(); let mut partial_matched = Vec::new(); - let mut unmatched = Vec::new(); + let mut producible_codecs = Vec::new(); - for codec in capabilities.codecs { + for codec in pc_factory.get_rtp_sender_capabilities(MediaType::Video).codecs { let mime_type = codec.mime_type.to_lowercase(); - if codec_producible && mime_type == format!("video/{requested_codec}") { + if mime_type == requested_mime_type { if let Some(sdp_fmtp_line) = codec.sdp_fmtp_line.as_ref() { // for h264 codecs that have sdpFmtpLine available, use only if the // profile-level-id is 42e01f for cross-browser compatibility @@ -2118,12 +2096,18 @@ impl SessionInner { } } partial_matched.push(codec); - } else { - unmatched.push(codec); + } else if is_producible(&mime_type) { + producible_codecs.push(codec); } } matched.append(&mut partial_matched); + if matched.is_empty() { + log::warn!( + "no sender capability for {requested_mime_type}; offering the codecs this sender can produce" + ); + matched = producible_codecs; + } transceiver.set_codec_preferences(matched)?; } diff --git a/webrtc-sys/include/livekit/peer_connection_factory.h b/webrtc-sys/include/livekit/peer_connection_factory.h index 4f2e4abd6..859fcf7cf 100644 --- a/webrtc-sys/include/livekit/peer_connection_factory.h +++ b/webrtc-sys/include/livekit/peer_connection_factory.h @@ -31,6 +31,7 @@ namespace livekit_ffi { class PeerConnectionFactory; class AudioDeviceController; class PeerConnectionObserverWrapper; +class VideoEncoderFactory; } // namespace livekit_ffi #include "webrtc-sys/src/peer_connection_factory.rs.h" @@ -73,6 +74,11 @@ class PeerConnectionFactory { RtpCapabilities rtp_receiver_capabilities(MediaType type) const; + // SDP codec names (VP8, VP9, AV1, H264, H265) that `backend` can produce + // with this factory's video encoders. + rust::Vec video_encoder_backend_codecs( + VideoEncoderBackend backend) const; + std::shared_ptr rtc_runtime() const { return rtc_runtime_; } std::shared_ptr audio_device() const; bool zero_playout_delay_enabled() const; @@ -86,6 +92,9 @@ class PeerConnectionFactory { webrtc::scoped_refptr adm_proxy_; std::shared_ptr audio_device_; webrtc::scoped_refptr peer_factory_; + // Not owned: the media engine inside peer_factory_ owns it, so it lives as + // long as peer_factory_ does. + const VideoEncoderFactory* video_encoder_factory_ = nullptr; webrtc::Environment env_; }; diff --git a/webrtc-sys/include/livekit/webrtc.h b/webrtc-sys/include/livekit/webrtc.h index 2a1e20762..64fd496ef 100644 --- a/webrtc-sys/include/livekit/webrtc.h +++ b/webrtc-sys/include/livekit/webrtc.h @@ -116,7 +116,5 @@ std::unique_ptr new_log_sink( rust::String create_random_uuid(); rust::Vec video_encoder_backend_list(); -rust::Vec video_encoder_backend_codecs( - VideoEncoderBackend backend); } // namespace livekit_ffi diff --git a/webrtc-sys/src/peer_connection_factory.cpp b/webrtc-sys/src/peer_connection_factory.cpp index 2595f27ad..944a9c9ce 100644 --- a/webrtc-sys/src/peer_connection_factory.cpp +++ b/webrtc-sys/src/peer_connection_factory.cpp @@ -186,8 +186,10 @@ PeerConnectionFactory::PeerConnectionFactory( dependencies.adm = adm_proxy_; - dependencies.video_encoder_factory = - std::move(std::make_unique()); + auto video_encoder_factory = + std::make_unique(); + video_encoder_factory_ = video_encoder_factory.get(); + dependencies.video_encoder_factory = std::move(video_encoder_factory); dependencies.video_decoder_factory = std::move(std::make_unique()); dependencies.audio_encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory(); @@ -276,6 +278,18 @@ RtpCapabilities PeerConnectionFactory::rtp_receiver_capabilities( static_cast(type))); } +rust::Vec PeerConnectionFactory::video_encoder_backend_codecs( + VideoEncoderBackend backend) const { + rust::Vec codecs; + if (!peer_factory_) { + return codecs; + } + for (const auto& codec : video_encoder_factory_->BackendCodecs(backend)) { + codecs.push_back(codec); + } + return codecs; +} + std::shared_ptr PeerConnectionFactory::audio_device() const { return audio_device_; } diff --git a/webrtc-sys/src/peer_connection_factory.rs b/webrtc-sys/src/peer_connection_factory.rs index 552a3958d..f35ab056b 100644 --- a/webrtc-sys/src/peer_connection_factory.rs +++ b/webrtc-sys/src/peer_connection_factory.rs @@ -79,6 +79,7 @@ pub mod ffi { type MediaStreamTrack = crate::media_stream::ffi::MediaStreamTrack; type SessionDescription = crate::jsep::ffi::SessionDescription; type MediaType = crate::webrtc::ffi::MediaType; + type VideoEncoderBackend = crate::webrtc::ffi::VideoEncoderBackend; } unsafe extern "C++" { @@ -130,8 +131,15 @@ pub mod ffi { self: &PeerConnectionFactory, kind: MediaType, ) -> RtpCapabilities; + + fn video_encoder_backend_codecs( + self: &PeerConnectionFactory, + backend: VideoEncoderBackend, + ) -> Vec; } + impl WeakPtr {} + extern "Rust" { type PeerConnectionObserverWrapper; diff --git a/webrtc-sys/src/video_encoder_factory.cpp b/webrtc-sys/src/video_encoder_factory.cpp index e60e350a0..f294c9ea3 100644 --- a/webrtc-sys/src/video_encoder_factory.cpp +++ b/webrtc-sys/src/video_encoder_factory.cpp @@ -399,9 +399,9 @@ VideoEncoderFactory::InternalFactory::GetSupportedFormats() const { // every pass-through codec negotiable by adding one entry for each codec // no real encoder listed. The pass-through matches by codec name at // Create(), so the real encoder's own entry serves it when one exists. A - // normal session is kept from negotiating a codec no real encoder - // implements by the codec-preference selection in the Rust publish path, - // which consults BackendCodecs(). + // raw sender is kept off a codec no real encoder implements by the Rust + // publish path, which picks its codec from BackendCodecs(), and otherwise + // by the encoder fallback described in GetImplementations(). for (const auto& backend_factory : factories_) { if (backend_factory.backend != VideoEncoderBackend::PreEncoded) { continue; @@ -491,6 +491,31 @@ VideoEncoderFactory::InternalFactory::GetImplementations() const { formats.push_back(WithBackend(format, VideoEncoderBackend::Software)); } + // libwebrtc only sends a negotiated codec that appears here, so a codec + // only the pass-through implements must be listed too, or a pre-encoded + // sender fails with "No video codecs in common". List it last and + // untagged: real encoders keep priority for every codec they cover, and a + // raw sender that still lands on it fails encoder creation (Create() never + // falls back to the pass-through), letting libwebrtc switch to another + // negotiated codec. A pre-encoded sender's selector moves it onto the + // pass-through backend instead. + const size_t real_format_count = formats.size(); + for (const auto& backend_factory : factories_) { + if (backend_factory.backend != VideoEncoderBackend::PreEncoded) { + continue; + } + for (const auto& format : backend_factory.factory->GetImplementations()) { + const bool codec_implemented = std::any_of( + formats.begin(), formats.begin() + real_format_count, + [&](const webrtc::SdpVideoFormat& existing) { + return IsSameCodecName(existing.name, format.name); + }); + if (!codec_implemented) { + formats.push_back(format); + } + } + } + return formats; } @@ -659,19 +684,6 @@ std::vector VideoEncoderFactory::BackendCodecs( return internal_factory_->BackendCodecs(backend); } -rust::Vec video_encoder_backend_codecs( - VideoEncoderBackend backend) { - // Probing hardware encoders is not free; keep one query-only factory for - // the life of the process instead of building one per call. - static const VideoEncoderFactory* const query_factory = - new VideoEncoderFactory(); - rust::Vec codecs; - for (const auto& codec : query_factory->BackendCodecs(backend)) { - codecs.push_back(codec); - } - return codecs; -} - std::vector VideoEncoderFactory::GetImplementations() const { return internal_factory_->GetImplementations(); diff --git a/webrtc-sys/src/webrtc.rs b/webrtc-sys/src/webrtc.rs index ab4214f19..fc7b7d099 100644 --- a/webrtc-sys/src/webrtc.rs +++ b/webrtc-sys/src/webrtc.rs @@ -73,7 +73,6 @@ pub mod ffi { fn create_random_uuid() -> String; fn video_encoder_backend_list() -> Vec; - fn video_encoder_backend_codecs(backend: VideoEncoderBackend) -> Vec; fn new_log_sink(fnc: fn(String, LoggingSeverity)) -> UniquePtr; } } From d0c2e838e219c666d558c2cbbfec4357c519b4a9 Mon Sep 17 00:00:00 2001 From: Stephen DeRosa Date: Thu, 22 Oct 2026 04:06:54 +0800 Subject: [PATCH 3/3] --headless arg in subscriber --- examples/local_video/src/subscriber.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/examples/local_video/src/subscriber.rs b/examples/local_video/src/subscriber.rs index 8e28ae2a0..7b4936ead 100644 --- a/examples/local_video/src/subscriber.rs +++ b/examples/local_video/src/subscriber.rs @@ -417,6 +417,10 @@ struct Args { /// Shared encryption key for E2EE (enables AES-GCM end-to-end encryption when set; must match publisher's key) #[arg(long)] e2ee_key: Option, + + /// Run without a window: still subscribe, decode, and log stats, but render nothing + #[arg(long)] + headless: bool, } struct SharedYuv { @@ -1818,6 +1822,14 @@ async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { } }); + if args.headless { + info!("Headless mode: decoding without a window, press Ctrl-C to exit"); + while !ctrl_c_received.load(Ordering::Acquire) { + tokio::time::sleep(Duration::from_millis(200)).await; + } + return Ok(()); + } + let viewport = AspectConstrainedViewport::new(None); // Start UI let app = VideoApp {