From 88411b7992ee854e15a19474b8d3795f1a2ab114 Mon Sep 17 00:00:00 2001 From: Daniel Piers Date: Tue, 13 Oct 2026 23:49:40 +0000 Subject: [PATCH 1/2] Fixing the HEVC VPS detection and the Annex-B framing in the RTP packer The VPS type test compared the wrong way around: & binds looser than ==, so nalptr[0] >> 1 & 0x3F == H265_NAL_TYPE_VPS evaluated as (nalptr[0] >> 1) & 0 and a type-32 NAL never populated the VPS cache. Group the shift and mask before the comparison, matching the SPS and PPS checks below it, and reject VPS payloads shorter than four bytes before they reach the MIME encoder. The RTP packer only took the Annex-B path when a pack opened with a four-byte start code, so a three-byte prefix, or a run of leading zeros before either prefix, was sent as a raw NAL with the framing bytes still in the payload. Count the leading zeros, accept either prefix, and hand the pack to the existing splitter. Extracted from #59, authored by Daniel Piers. --- src/rtsp/rtp.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/rtsp/rtp.c b/src/rtsp/rtp.c index 080a83d7..e6d7bf50 100644 --- a/src/rtsp/rtp.c +++ b/src/rtsp/rtp.c @@ -363,7 +363,8 @@ static inline int __retrieve_sprop(rtsp_handle h, unsigned char *buf, size_t len nalptr = buf; single_len = 0; while (nal_split(buf, &nalptr, &single_len, len) == SUCCESS) { - if (nalptr[0] >> 1 & 0x3F == H265_NAL_TYPE_VPS) { + if ((nalptr[0] >> 1 & 0x3F) == H265_NAL_TYPE_VPS) { + ASSERT(single_len >= 4, return FAILURE); ASSERT(base64 = mime_base64_create((char *)&(nalptr[0]), single_len), return FAILURE); DASSERT(base64->base == 64, return FAILURE); @@ -522,8 +523,12 @@ int rtp_send_h26x(rtsp_handle h, hal_vidstream *stream, char isH265) for (int i = 0; i < stream->count; i++) { unsigned char *data = stream->pack[i].data + stream->pack[i].offset; size_t length = stream->pack[i].length - stream->pack[i].offset; - if (length >= 4 && data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1) { - unsigned char *nalptr = data; + size_t leading_zeros = 0; + while (leading_zeros < length && data[leading_zeros] == 0) + leading_zeros++; + if (leading_zeros >= 2 && leading_zeros < length && data[leading_zeros] == 1) { + /* Accept either Annex-B prefix and skip any leading_zero_8bits. */ + unsigned char *nalptr = data + leading_zeros - 2; size_t single_len = 0; while (nal_split(data, &nalptr, &single_len, length) == SUCCESS) { ASSERT(__transfer_nal_h26x(&(trans.list_head), nalptr, single_len, h->isH265) == SUCCESS, goto error); From 9c12c2d41f6f5ad34ee21dbb1b7f79dd16d22c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?William=20B=C3=A9rub=C3=A9?= Date: Tue, 13 Oct 2026 23:49:58 +0000 Subject: [PATCH 2/2] Running every RTP pack through the shared NAL splitter The RTP packer decided for itself whether a pack held Annex-B framing before splitting it, and kept a raw path for anything that did not match. nal_split() already scans for both start code lengths and hands the whole pack back as a single NAL when none is present, so the hand-rolled detection and the separate raw path were redundant, and out of step with the parameter cache, which has always split unconditionally. Drop the branch and run nal_split() over every pack. --- src/rtsp/rtp.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/rtsp/rtp.c b/src/rtsp/rtp.c index e6d7bf50..242228e4 100644 --- a/src/rtsp/rtp.c +++ b/src/rtsp/rtp.c @@ -523,18 +523,10 @@ int rtp_send_h26x(rtsp_handle h, hal_vidstream *stream, char isH265) for (int i = 0; i < stream->count; i++) { unsigned char *data = stream->pack[i].data + stream->pack[i].offset; size_t length = stream->pack[i].length - stream->pack[i].offset; - size_t leading_zeros = 0; - while (leading_zeros < length && data[leading_zeros] == 0) - leading_zeros++; - if (leading_zeros >= 2 && leading_zeros < length && data[leading_zeros] == 1) { - /* Accept either Annex-B prefix and skip any leading_zero_8bits. */ - unsigned char *nalptr = data + leading_zeros - 2; - size_t single_len = 0; - while (nal_split(data, &nalptr, &single_len, length) == SUCCESS) { - ASSERT(__transfer_nal_h26x(&(trans.list_head), nalptr, single_len, h->isH265) == SUCCESS, goto error); - } - } else { - ASSERT(__transfer_nal_h26x(&(trans.list_head), data, length, h->isH265) == SUCCESS, goto error); + unsigned char *nalptr = data; + size_t single_len = 0; + while (nal_split(data, &nalptr, &single_len, length) == SUCCESS) { + ASSERT(__transfer_nal_h26x(&(trans.list_head), nalptr, single_len, h->isH265) == SUCCESS, goto error); } } ASSERT(list_map_inline(&(trans.list_head), (__tcp_flush_each), NULL) == SUCCESS, goto error);