From 0f26cc38b4a8fdfa2df84323a00a7a0bd06eded7 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Fri, 9 Oct 2026 23:12:13 +0000 Subject: [PATCH] Use the sysmon coverage core on the unit, GPU and regression lanes Python 3.12 defaults to coverage's C tracer, which slows every line of Python that runs under it. Every lane starts coverage in each spawned worker and subprocess through COVERAGE_PROCESS_START, so the import-heavy start-ups pay for it: the 76 s Megatron worker-pool start, the 22 s hf_ptq processes of the unified-export tests, the puzzletron spawns, the vLLM engine core. COVERAGE_CORE=sysmon records the same lines through sys.monitoring at a fraction of the overhead. Measured (the first three here, the last by #2727 on its hf_ptq example lane): - serial tests/unit run on 4 CPUs: 572 s without coverage, 825 s with the C tracer, 640 s with sysmon; - `import modelopt.torch.quantization.extensions` with this repo's coverage config: 17.1 s native, 23.6 s with the C tracer, 18.3 s with sysmon, and the child process of a spawn writes its data file too, with the same 362 files and 13,821 executed lines; - one TinyLlama fp8 plus TRT-LLM deploy test: 64.8 s without coverage, 103.3 s with the C tracer, 70.5 s with sysmon, identical covered lines. Applies to the unit `linux` job (Python 3.12), gpu_tests.yml and regression_tests.yml. The example runner is left to #2727, which makes the same change there. sysmon needs Python 3.12 or newer; on an older interpreter coverage warns and keeps its default core, so a lane that turns out to run an older Python is slower, not broken. The gpu and gpu_megatron logs show 3.12. Testing: actionlint is clean on all workflows. Not yet run on a real CI runner; the first run should be compared against the previous coverage.xml. Signed-off-by: Shengliang Xu --- .github/workflows/gpu_tests.yml | 3 +++ .github/workflows/regression_tests.yml | 3 +++ .github/workflows/unit_tests.yml | 2 ++ 3 files changed, 8 insertions(+) diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index b73f85affeb..5094075dafd 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -96,6 +96,9 @@ jobs: COVERAGE_FILE: ${{ github.workspace }}/.coverage # Explicit: HOME is /github/home in container jobs, not the /root the cache steps use MODELOPT_WHEELHOUSE: /root/.cache/wheelhouse + # sys.monitoring records the same lines as the default C tracer at a fraction of the + # overhead on import-heavy worker processes. Needs Python 3.12+. + COVERAGE_CORE: sysmon run: | # Use `python3` (the vllm image has no `python` on PATH) python3 -m pip install nox && nox -s ${{ matrix.example }} diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 9c1154a2b65..dd41c3993b9 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -60,6 +60,9 @@ jobs: env: COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml COVERAGE_FILE: ${{ github.workspace }}/.coverage + # sys.monitoring records the same lines as the default C tracer at a fraction of the + # overhead on import-heavy worker processes. Needs Python 3.12+. + COVERAGE_CORE: sysmon run: python -m pip install nox && nox -s regression - name: Upload regression coverage to Codecov uses: codecov/codecov-action@v7 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index c3bae2e41be..3442e6147c1 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -69,6 +69,8 @@ jobs: COVERAGE_FILE: ${{ github.workspace }}/.coverage # Everything but PRs also runs the slow tests marked ``manual`` (e.g. AutoNAS). PYTEST_EXTRA: ${{ github.event_name != 'pull_request' && '--run-manual' || '' }} + # sys.monitoring records the same lines as the default C tracer at a fraction of the overhead + COVERAGE_CORE: sysmon # One single-threaded xdist worker per vCPU (`-n auto` counts physical cores); torch's # default threads per worker oversubscribe the runner and run slower than serial. PYTEST_ADDOPTS: -n 4 --dist loadfile