Fix HEVC VPS detection and Annex-B RTP framing

.github/workflows/build.yml

@@ -9,6 +9,18 @@ on:
   workflow_dispatch:
 
 jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Test RTSP cache and RTP framing
+        run: make -C tests test
+
   build:
     runs-on: ubuntu-latest
     steps:

src/rtsp/rtp.c

@@ -363,7 +363,8 @@ static inline int __retrieve_sprop(rtsp_handle h, unsigned char *buf, size_t len
         nalptr = buf;
         single_len = 0;
         while (nal_split(buf, &nalptr, &single_len, len) == SUCCESS) {
-            if (nalptr[0] >> 1 & 0x3F == H265_NAL_TYPE_VPS) {
+            if ((nalptr[0] >> 1 & 0x3F) == H265_NAL_TYPE_VPS) {
+                ASSERT(single_len >= 4, return FAILURE);
                 ASSERT(base64 = mime_base64_create((char *)&(nalptr[0]), single_len), return FAILURE);
 
                 DASSERT(base64->base == 64, return FAILURE);
@@ -522,8 +523,12 @@ int rtp_send_h26x(rtsp_handle h, hal_vidstream *stream, char isH265)
         for (int i = 0; i < stream->count; i++) {
             unsigned char *data = stream->pack[i].data + stream->pack[i].offset;
             size_t length = stream->pack[i].length - stream->pack[i].offset;
-            if (length >= 4 && data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1) {
-                unsigned char *nalptr = data;
+            size_t leading_zeros = 0;
+            while (leading_zeros < length && data[leading_zeros] == 0)
+                leading_zeros++;
+            if (leading_zeros >= 2 && leading_zeros < length && data[leading_zeros] == 1) {
+                /* Accept either Annex-B prefix and skip any leading_zero_8bits. */
+                unsigned char *nalptr = data + leading_zeros - 2;
                 size_t single_len = 0;
                 while (nal_split(data, &nalptr, &single_len, length) == SUCCESS) {
                     ASSERT(__transfer_nal_h26x(&(trans.list_head), nalptr, single_len, h->isH265) == SUCCESS, goto error);

tests/.gitignore

@@ -0,0 +1 @@
+/build/

tests/Makefile

@@ -0,0 +1,42 @@
+# Tests of the actual private RTSP parameter cache and public RTP packetizer.
+# No RTSP server, sockets, camera, hardware or vendor binary is run.
+CC ?= cc
+BUILD_DIR ?= build
+# Optional existing output directory for serialized framing-test packets.
+FRAMING_CAPTURE_DIR ?=
+COMMON_FLAGS = -std=gnu11 -ffunction-sections -fdata-sections
+SPROP_SOURCES = test_rtsp_sprop.c ../src/rtsp/mime.c
+FRAMING_SOURCES = test_rtp_framing.c ../src/rtsp/mime.c
+
+ifeq ($(shell uname -s),Darwin)
+COMMON_FLAGS += -Wno-gnu-designator
+GC_SECTIONS = -Wl,-dead_strip
+else
+GC_SECTIONS = -Wl,--gc-sections
+endif
+
+.PHONY: test test-native test-sanitize test-sprop-native test-sprop-sanitize test-framing-native test-framing-sanitize
+test: test-native test-sanitize
+test-native: test-sprop-native test-framing-native
+test-sanitize: test-sprop-sanitize test-framing-sanitize
+
+test-sprop-native:
+	mkdir -p $(BUILD_DIR)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $(COMMON_FLAGS) -O2 $(SPROP_SOURCES) $(LDFLAGS) $(GC_SECTIONS) -pthread -o $(BUILD_DIR)/sprop-native
+	$(BUILD_DIR)/sprop-native
+
+test-sprop-sanitize:
+	mkdir -p $(BUILD_DIR)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $(COMMON_FLAGS) -O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all -fno-omit-frame-pointer $(SPROP_SOURCES) $(LDFLAGS) $(GC_SECTIONS) -pthread -o $(BUILD_DIR)/sprop-sanitized
+	$(BUILD_DIR)/sprop-sanitized
+	$(BUILD_DIR)/sprop-sanitized --short-vps-two
+
+test-framing-native:
+	mkdir -p $(BUILD_DIR)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $(COMMON_FLAGS) -O2 $(FRAMING_SOURCES) $(LDFLAGS) $(GC_SECTIONS) -pthread -o $(BUILD_DIR)/framing-native
+	$(BUILD_DIR)/framing-native $(if $(FRAMING_CAPTURE_DIR),--capture $(FRAMING_CAPTURE_DIR)/framing-native.bin)
+
+test-framing-sanitize:
+	mkdir -p $(BUILD_DIR)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $(COMMON_FLAGS) -O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all -fno-omit-frame-pointer $(FRAMING_SOURCES) $(LDFLAGS) $(GC_SECTIONS) -pthread -o $(BUILD_DIR)/framing-sanitized
+	$(BUILD_DIR)/framing-sanitized $(if $(FRAMING_CAPTURE_DIR),--capture $(FRAMING_CAPTURE_DIR)/framing-sanitized.bin)

tests/framing-fixtures/README.md

@@ -0,0 +1,44 @@
+# RTP framing fixtures
+
+`fixtures.h` contains four locally generated, complete one-frame Annex-B streams
+and independently extracted NAL offsets/lengths. No camera, vendor input, network,
+FFmpeg or external project is needed to compile or run the tests. They include
+small AVC/HEVC NALs and large AVC/HEVC IDR NALs that exercise RTP fragmentation.
+All test NAL payloads are at least six bytes.
+
+`provenance.json` records the original FFmpeg 8.0.1 build, exact encode/decode
+commands, stream and individual NAL SHA256 values, and one-frame decoded output
+MD5 values. The input is FFmpeg's `lavfi testsrc2` at 32x32 or 512x512, yuv420p,
+one frame, using libx264/libx265 with fixed single-thread encoder options.
+`<out>` in the recorded commands means a fresh local output directory. Generation
+is optional provenance work; encoding may differ with another encoder build.
+
+For the header, read each recorded stream as bytes, locate 3/4-byte Annex-B
+prefixes in order, and define each NAL slice after its prefix through the next
+prefix (excluding zero padding at the end of that slice). `frames[].nals` gives
+the exact frozen offsets, lengths, types and hashes used by this test. The C
+header serializes each complete stream in a `static const unsigned char[]` and
+its slice table in `struct fixture_nal[]`; its hash is in the provenance file.
+The payload oracle uses these frozen slices, not the production NAL splitter.
+
+Run from the repository root:
+
+```
+make -C tests test
+```
+
+This runs native and ASan/UBSan tests for the accepted parameter-cache behavior
+and the actual public RTP packetizer. `test-native`, `test-sanitize`,
+`test-framing-native` and `test-framing-sanitize` are available separately.
+`BUILD_DIR` may point at a writable external build directory. Optionally set
+`FRAMING_CAPTURE_DIR` to an existing directory to retain packet files; the test
+refuses to replace an existing capture file.
+
+The framing harness replaces UDP `send` with capture/error injection. A supplied
+positive timestamp is required in every test; an aborting `millis` stub detects
+any unexpected wall-clock fallback. It creates no sockets, server or workers.
+The tests check byte-exact single/FU NAL reconstruction, sequence/timestamp wrap,
+unchanged input and partial send failure. They do not test marker policy,
+parameter-set replacement, concurrent calls, RTSP negotiation, actual network
+delivery or playback. No unsafe short-MIME/SPS fault reproducer is included.
+The existing VPS suite retains its corrected-code short-VPS rejection checks.