Memory regression tests

.github/workflows/ci.yml

@@ -55,6 +55,7 @@ jobs:
               - client-sdk-rust
               - client-sdk-rust/**
               - cmake/**
+              - scripts/memory-tester.sh
               - scripts/set-test-tokens.sh
               - scripts/run-with-backtrace.sh
               - CMakeLists.txt

.github/workflows/tests.yml

@@ -357,7 +357,11 @@ jobs:
             --gtest_output="xml:${{ env.BUILD_DIR }}\unit-test-results.xml"
 
       - name: Start livekit-server
-        if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests)
+        if: >-
+          matrix.e2e-testing &&
+          (inputs.integration_repeat > 0 ||
+           inputs.run_stress_tests ||
+           (matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release'))
         id: livekit_server
         uses: livekit/dev-server-action@5d4d5337a875e2d1afd37bed03c601d159dab002 # v1.1.1
         with:
@@ -368,7 +372,11 @@ jobs:
 
       # Needed by token helper script
       - name: Install livekit-cli
-        if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests)
+        if: >-
+          matrix.e2e-testing &&
+          (inputs.integration_repeat > 0 ||
+           inputs.run_stress_tests ||
+           (matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release'))
         shell: bash
         env:
           # Windows installs lk via `gh api` / `gh release download`, which need this env var
@@ -425,6 +433,21 @@ jobs:
             --gtest_recreate_environments_when_repeating=1 \
             --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml
 
+      - name: Run memory tester
+        if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release'
+        timeout-minutes: 10
+        shell: bash
+        run: |
+          set -euo pipefail
+          source scripts/set-test-tokens.sh
+          scripts/memory-tester.sh \
+            --build-dir "${{ env.BUILD_DIR }}" \
+            --scenario all \
+            --iterations 100 \
+            --warmup 20 \
+            --max-rss-growth-mib 8 \
+            --max-thread-growth 0
+
       - name: Run stress tests
         if: matrix.e2e-testing && inputs.run_stress_tests
         timeout-minutes: ${{ inputs.stress_timeout_minutes }}

docs/testing.md

@@ -1,7 +1,21 @@
 # Testing
 
-The SDK includes integration and stress tests using
-[Google Test](https://github.com/google/googletest).
+The SDK includes unit, integration and stress tests using
+[Google Test](https://github.com/google/googletest). It also
+includes a memory-leak/resource checking tester.
+
+Unit, integration, and memory tests are run on every PR in CI.
+Stress tests are reserved for nightly tests in CI due to their
+length.
+
+## Test binaries
+
+| Executable | Description |
+|------------|-------------|
+| `livekit_unit_tests` | Unit tests (no server required) |
+| `livekit_integration_tests` | Integration tests (server required) |
+| `livekit_stress_tests` | Long-running stress tests (server required) |
+| `livekit_memory_tester` | Resource leak tester |
 
 ## Building the test binaries
 
@@ -17,33 +31,35 @@ The SDK includes integration and stress tests using
 .\build.cmd release-tests
 ```
 
-## Running tests
+After building, test binaries are available at:
 
-After building, run tests using `ctest` or invoke the binaries directly:
+```bash
+# Linux/macOS
+<build-folder>/bin/<test binaries>
+
+# Windows
+<build-folder>\bin\<test binaries>
+```
+
+## Running tests
 
 ```bash
-# Run all tests via ctest
+# Run all tests via ctest (warning, runs all GTest-based tests including stress)
 cd build-debug
 ctest --output-on-failure
 
 # Or run test executables directly
+./build-debug/bin/livekit_unit_tests
 ./build-debug/bin/livekit_integration_tests
 ./build-debug/bin/livekit_stress_tests
 
 # Run specific test suites
+./build-debug/bin/livekit_unit_tests --gtest_filter="*Room*"
 ./build-debug/bin/livekit_integration_tests --gtest_filter="*Rpc*"
 ./build-debug/bin/livekit_stress_tests --gtest_filter="*MaxPayloadStress*"
 ```
 
-__Note:__ The tests require tokens and a running LiveKit server. See the section below for details.
-
-## Test binaries
-
-| Executable | Description |
-|------------|-------------|
-| `livekit_unit_tests` | Pure unit tests (no server required) |
-| `livekit_integration_tests` | Quick tests (~1-2 minutes) for SDK functionality |
-| `livekit_stress_tests` | Long-running tests (configurable, default 1 hour) |
+__Note:__ The integration and stress tests require tokens and a running LiveKit server. See the section below for details.
 
 ## Running a local LiveKit server for tests
 
@@ -52,9 +68,10 @@ path is `livekit-server --dev`, which uses the well-known dev API
 key/secret (`devkey` / `secret`).
 
 Install [`livekit-server`](https://docs.livekit.io/home/self-hosting/local/)
-and start it with data tracks enabled:
+and start it with data blobs enabled:
 
 ```bash
+LIVEKIT_CONFIG="enable_participant_data_blob: true"
 livekit-server --dev
 ```
 
@@ -106,7 +123,69 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret -
 - **RPC**: round-trip calls, max payload (15 KB), timeouts, errors, concurrent calls.
 - **Stress**: high throughput, bidirectional RPC, memory pressure.
 
-## Memory checks (valgrind)
+## Memory tester
+
+> Note: Linux only
+
+The memory tester binary catches resource leak regressions in an
+automated fashion, and is invoked in CI. This single binary runs individual
+or groups of scenarios:
+
+- `audio-track` repeatedly initializes the SDK, connects, publishes an audio
+  track, captures synthetic PCM frames, unpublishes, disconnects, and shuts down.
+- `video-track` repeatedly initializes the SDK, connects, publishes a video
+  track, captures synthetic I420 frames, unpublishes, disconnects, and shuts down.
+- `data-track` repeatedly connects, publishes a data track, pushes synthetic
+  payloads, unpublishes it, and disconnects.
+- `room-reuse` repeatedly connects and disconnects the same room object.
+- `room-client-leave` repeatedly connects, disconnects, and destroys a room.
+- `room-server-delete` repeatedly connects, publishes one synthetic video
+  track, captures I420 frames without unpublishing, and uses `lk` to delete
+  the local test room. It then verifies the disconnect and end-of-stream
+  callbacks.
+
+Each scenario reports process RSS and thread count at cycle 0, after warmup
+cycle 20, and after final cycle 100. Checkpoint sampling waits for the thread
+count to remain stable for 500 ms so short-lived teardown work does not cause a
+false failure. The gate applies to warmup-to-final growth and fails on thread
+growth or more than the configured amount of RSS growth. If threads grow, the
+tester also reports when growth was first observed and groups the warmup and
+final threads by their Linux thread names.
+
+The binary can be run directly, or via a helper script at `scripts/memory-tester.sh`,
+which invokes the binary but also has GitHub actions support for a results summary table.
+The helper defaults `MALLOC_ARENA_MAX` to `1` when launching each tester process.
+This limits `glibc`'s allocator arenas, reducing Linux thread-churn RSS noise, but it is
+not evidence that the default allocator returns all freed pages to the OS.
+
+Start `livekit-server --dev`, install the `lk` CLI, and source the local test
+tokens before running any scenario:
+
+```bash
+./build.sh release-tests
+source scripts/set-test-tokens.sh
+scripts/memory-tester.sh --scenario audio-track \
+  --iterations 100 --warmup 20 \
+  --max-rss-growth-mib 8 --max-thread-growth 0
+scripts/memory-tester.sh --scenario video-track \
+  --iterations 100 --warmup 20 \
+  --max-rss-growth-mib 8 --max-thread-growth 0
+```
+
+Use `--scenario all` to run all six scenarios, each in a fresh tester process:
+
+```bash
+scripts/memory-tester.sh --scenario data-track
+scripts/memory-tester.sh --scenario room-reuse
+scripts/memory-tester.sh --scenario room-client-leave
+scripts/memory-tester.sh --scenario room-server-delete
+scripts/memory-tester.sh --scenario all
+```
+
+The limits are command-line options so a longer local run can use different
+budgets.
+
+## Additional memory checks (valgrind)
 
 Run `valgrind` against the test binaries to check for memory leaks and other
 issues. See [tools.md](tools.md) for the recipe.

scripts/memory-tester.sh

@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+#
+# Copyright 2026 LiveKit
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage: memory-tester.sh [OPTIONS]
+
+Run the Linux memory tester and append a table to the
+GitHub step summary when GITHUB_STEP_SUMMARY is set.
+
+Options:
+  --build-dir PATH            Build directory (default: build-release)
+  --scenario NAME             audio-track, video-track, data-track,
+                              room-reuse, room-client-leave,
+                              room-server-delete, or all (default: audio-track)
+  --iterations N              Lifecycle cycles (default: 100)
+  --warmup N                  Baseline cycle (default: 20)
+  --max-rss-growth-mib N      Allowed RSS growth (default: 8)
+  --max-thread-growth N       Allowed thread growth (default: 0)
+  -h, --help                  Show this help
+
+Environment:
+  MEMORY_TESTER_BUILD_DIR, MEMORY_TESTER_SCENARIO,
+  MEMORY_TESTER_ITERATIONS,
+  MEMORY_TESTER_WARMUP, MEMORY_TESTER_MAX_RSS_GROWTH_MIB,
+  MEMORY_TESTER_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX.
+  All scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A.
+EOF
+}
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+build_dir="${MEMORY_TESTER_BUILD_DIR:-build-release}"
+scenario="${MEMORY_TESTER_SCENARIO:-audio-track}"
+iterations="${MEMORY_TESTER_ITERATIONS:-100}"
+warmup="${MEMORY_TESTER_WARMUP:-20}"
+max_rss_growth_mib="${MEMORY_TESTER_MAX_RSS_GROWTH_MIB:-8}"
+max_thread_growth="${MEMORY_TESTER_MAX_THREAD_GROWTH:-0}"
+
+while (($#)); do
+  case "$1" in
+    --build-dir) build_dir="$2"; shift 2 ;;
+    --scenario) scenario="$2"; shift 2 ;;
+    --iterations) iterations="$2"; shift 2 ;;
+    --warmup) warmup="$2"; shift 2 ;;
+    --max-rss-growth-mib) max_rss_growth_mib="$2"; shift 2 ;;
+    --max-thread-growth) max_thread_growth="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "ERROR: unknown option: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if [[ "${build_dir}" != /* ]]; then
+  build_dir="${repo_root}/${build_dir}"
+fi
+tester="${build_dir}/bin/livekit_memory_tester"
+if [[ ! -x "${tester}" ]]; then
+  echo "ERROR: ${tester} not found. Run ./build.sh release-tests first." >&2
+  exit 1
+fi
+
+case "${scenario}" in
+  audio-track|video-track|data-track|room-reuse|room-client-leave|room-server-delete)
+    scenarios=("${scenario}")
+    ;;
+  all)
+    scenarios=(audio-track video-track data-track room-reuse room-client-leave room-server-delete)
+    ;;
+  *)
+    echo "ERROR: invalid scenario: ${scenario}" >&2
+    usage >&2
+    exit 2
+    ;;
+esac
+
+for current_scenario in "${scenarios[@]}"; do
+  if [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then
+    echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2
+    echo "Run: source scripts/set-test-tokens.sh" >&2
+    exit 1
+  fi
+done
+
+if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+  {
+    echo "## Memory tester results"
+    echo
+    echo "Iterations: ${iterations}"
+    echo "Warmup: ${warmup}"
+    echo "Threshold: RSS growth ≤ ${max_rss_growth_mib} MiB; thread growth ≤ ${max_thread_growth}."
+    echo
+    echo "| Scenario | RSS 0→${warmup} | RSS ${warmup}→${iterations} | Threads 0→${warmup} | Threads ${warmup}→${iterations} | Status |"
+    echo "|---|---:|---:|---:|---:|---|"
+  } >> "${GITHUB_STEP_SUMMARY}"
+fi
+
+report_regex='memory tester: scenario=([a-z-]+), RSS 0 ([0-9]+\.[0-9]+) -> warmup ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\) -> final ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)'
+
+format_growth() {
+  if [[ "$1" == "0" || "$1" == "0.00" || "$1" == -* ]]; then
+    printf '%s' "$1"
+  else
+    printf '+%s' "$1"
+  fi
+}
+
+overall_status=0
+for current_scenario in "${scenarios[@]}"; do
+  set +e
+  output="$(
+    MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" "${tester}" \
+      --scenario "${current_scenario}" \
+      --iterations "${iterations}" \
+      --warmup "${warmup}" \
+      --max-rss-growth-mib "${max_rss_growth_mib}" \
+      --max-thread-growth "${max_thread_growth}" 2>&1
+  )"
+  status=$?
+  set -e
+  printf '%s\n' "${output}"
+
+  if [[ "${output}" =~ ${report_regex} ]]; then
+    if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+      if [[ "${BASH_REMATCH[12]}" == "PASS" ]]; then
+        verdict=":white_check_mark: PASS"
+      else
+        verdict=":x: FAIL"
+      fi
+      rss_warmup_growth="$(format_growth "${BASH_REMATCH[4]}")"
+      rss_final_growth="$(format_growth "${BASH_REMATCH[6]}")"
+      thread_warmup_growth="$(format_growth "${BASH_REMATCH[9]}")"
+      thread_final_growth="$(format_growth "${BASH_REMATCH[11]}")"
+      echo "| ${BASH_REMATCH[1]} | ${rss_warmup_growth} MiB | ${rss_final_growth} MiB | ${thread_warmup_growth} | ${thread_final_growth} | ${verdict} |" \
+        >> "${GITHUB_STEP_SUMMARY}"
+    fi
+  else
+    echo "ERROR: ${current_scenario} did not produce a memory report." >&2
+    if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+      echo "| ${current_scenario} | — | — | — | — | :x: ERROR |" >> "${GITHUB_STEP_SUMMARY}"
+    fi
+    status=1
+  fi
+
+  if [[ "${status}" -ne 0 ]]; then
+    overall_status=1
+  fi
+done
+
+exit "${overall_status}"

src/tests/CMakeLists.txt

@@ -348,3 +348,22 @@ endif()
 if(TARGET livekit_stress_tests)
   add_dependencies(run_all_tests livekit_stress_tests)
 endif()
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+  add_executable(livekit_memory_tester
+    "${CMAKE_CURRENT_SOURCE_DIR}/memory/memory_tester.cpp"
+  )
+  target_link_libraries(livekit_memory_tester PRIVATE livekit)
+  target_include_directories(livekit_memory_tester PRIVATE
+    ${LIVEKIT_ROOT_DIR}/include
+  )
+  add_custom_command(TARGET livekit_memory_tester POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      $<TARGET_FILE:livekit>
+      $<TARGET_FILE_DIR:livekit_memory_tester>
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      "$<TARGET_FILE_DIR:livekit>/liblivekit_ffi.so"
+      $<TARGET_FILE_DIR:livekit_memory_tester>
+    COMMENT "Copying shared libraries next to livekit_memory_tester"
+  )
+endif()