From 7d5752c2defda0a06c88ae5a5bc61e5154d914d9 Mon Sep 17 00:00:00 2001 From: asayed18 <33804215+asayed18@users.noreply.github.com> Date: Thu, 1 Oct 2026 01:51:44 +0200 Subject: [PATCH 1/2] Convert VideoToolbox frames to software instead of dropping blocked output On macOS VLC decodes with VideoToolbox, so icop received opaque CVPX frames. With no CVPX backend it fell back to image conversion: blocked frames were dropped (black screen instead of blur/warning), the debug overlay could not render, and synchronous analysis held decoder buffers until VideoToolbox stalled (pic_holder_wait timed out). Decline CVPX input in Open so VLC's chain filter inserts the cvpx converter and reopens icop with I420, which runs the normal CPU path. Only decline when the cvpx module exists; otherwise keep the opaque fallback so icop is never dropped from the chain (fail-closed). Verified with VLC 3.0.23 on Apple Silicon: blur and overlay render, no decoder stalls or dropped frames. The macOS smoke test now fails if blocked frames would be dropped. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +++++ src/plugin/nsfw_backend.c | 49 +++++++++++++++++++++++++++++++++++ tools/macos_vlc_smoke_test.sh | 3 ++- 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abcee25..3112a62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,13 @@ first, and short hashes identify the commit that introduced each change. - Add `install_icop_plugin.sh --uninstall`, and publish releases to the `asayed18/homebrew-icop` tap (`brew install --cask icop`). A new workflow notifies the tap when a release is published. +- Decline VideoToolbox (CVPX) frames so VLC inserts its `cvpx` converter + and icop runs on software frames. Previously blocked frames were dropped + (a black screen instead of blur or warning), the debug overlay never showed, + and synchronous analysis held decoder buffers until VideoToolbox stalled + (`pic_holder_wait timed out`). The opaque fallback is kept for VLC builds + without the `cvpx` module, so icop never ends up playing video unfiltered. + The macOS smoke test now fails if blocked frames would be dropped. - Bump the version to 0.1.7 so the fixed macOS packages do not reuse the broken v0.1.6 draft. diff --git a/src/plugin/nsfw_backend.c b/src/plugin/nsfw_backend.c index d60c7a8..dd039e0 100644 --- a/src/plugin/nsfw_backend.c +++ b/src/plugin/nsfw_backend.c @@ -28,6 +28,7 @@ # include "nsfw_filter_vaapi.h" #endif #include "frame_processor.h" +#include "platform_abstraction.h" /* ------------------------------------------------------------------ */ /* D3D11 backend wrappers */ @@ -203,11 +204,59 @@ static vlc_fourcc_t PreferredOpaqueChroma(vlc_fourcc_t opaque_chroma) } } +static bool IsCvpxChroma(vlc_fourcc_t chroma) +{ + switch (chroma) { +#ifdef VLC_CODEC_CVPX_NV12 + case VLC_CODEC_CVPX_NV12: +#endif +#ifdef VLC_CODEC_CVPX_UYVY + case VLC_CODEC_CVPX_UYVY: +#endif +#ifdef VLC_CODEC_CVPX_I420 + case VLC_CODEC_CVPX_I420: +#endif +#ifdef VLC_CODEC_CVPX_BGRA + case VLC_CODEC_CVPX_BGRA: +#endif +#ifdef VLC_CODEC_CVPX_P010 + case VLC_CODEC_CVPX_P010: +#endif + return true; + default: + return false; + } +} + +/* VideoToolbox frames can be neither blurred nor drawn on in place, and + * the opaque fallback analyses them synchronously while holding decoder + * buffers, which stalls VideoToolbox. Declining them makes VLC's "chain" + * filter insert its cvpx converter and reopen icop with a software chroma, + * so the normal CPU path (async worker, block styles, overlay) runs. Only + * decline when that converter exists: otherwise VLC would drop icop from + * the chain and play the video unfiltered. */ +static bool CvpxSoftwareConverterAvailable(void) +{ + typedef bool (*module_exists_fn)(const char *); + module_exists_fn exists = + (module_exists_fn)nsfw_plat_lookup_vlc_sym("module_exists"); + + return exists != NULL && exists("cvpx"); +} + int nsfw_backend_open(filter_t *filter) { filter_sys_t *sys = filter->p_sys; vlc_fourcc_t chroma = filter->fmt_in.video.i_chroma; + if (IsCvpxChroma(chroma) && CvpxSoftwareConverterAvailable()) { + fprintf(stderr, + "icop: declining VideoToolbox chroma %4.4s;" + " VLC will convert frames to a software chroma\n", + (const char *)&chroma); + return VLC_EGENERIC; + } + if (nsfw_d3d11_is_opaque(chroma)) { nsfw_d3d11_backend_t *d3d11 = NULL; if (nsfw_d3d11_open(filter, &d3d11) != VLC_SUCCESS) { diff --git a/tools/macos_vlc_smoke_test.sh b/tools/macos_vlc_smoke_test.sh index 8fa4e35..f62dc83 100755 --- a/tools/macos_vlc_smoke_test.sh +++ b/tools/macos_vlc_smoke_test.sh @@ -51,7 +51,8 @@ for failure in \ 'unable to load the packaged ONNX Runtime' \ 'ONNX detector unavailable' \ 'detector initialization failed' \ - 'does not support C API version'; do + 'does not support C API version' \ + 'blocked frames will be dropped fail-closed'; do if grep -q "$failure" "$log_file"; then die "VLC reported: $failure (see $log_file)" fi From 446b611338292c15cfba340a683fce86560f431d Mon Sep 17 00:00:00 2001 From: asayed18 <33804215+asayed18@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:26:12 +0200 Subject: [PATCH 2/2] Use several ONNX threads when icop runs a single detector worker Every ONNX session was pinned to one intra-op thread, which assumes one CPU worker per core. GPU providers (CoreML on macOS) and hardware backends run a single worker, so marqo took ~120 ms per inference on one thread. With the automatic stride (every 3rd frame) a 720p30 video needed more than a second of inference per second and fell steadily behind: 380 late frames, median 3.5 s, in 30 s. The plugin now exports NSFW_ONNX_INTRA_OP_THREADS = cores / workers (1-4) unless the user set it, and icop_core applies it to the session. Same file afterwards: 5 late frames, median 37 ms, no decoder stalls. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +++++ src/nsfw_onnx_providers.cpp | 17 ++++++++++++++++- src/plugin/nsfw_filter_worker.c | 25 +++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3112a62..3c23e81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,11 @@ first, and short hashes identify the commit that introduced each change. (`pic_holder_wait timed out`). The opaque fallback is kept for VLC builds without the `cvpx` module, so icop never ends up playing video unfiltered. The macOS smoke test now fails if blocked frames would be dropped. +- Give each ONNX session several threads when icop runs fewer detector + workers than CPU cores (cores / workers, at most 4). GPU providers such as + CoreML run one worker, and with one thread marqo needed ~120 ms per frame, + so automatic strides fell behind real time (a 720p30 file ended up seconds + late). `NSFW_ONNX_INTRA_OP_THREADS` overrides the value. - Bump the version to 0.1.7 so the fixed macOS packages do not reuse the broken v0.1.6 draft. diff --git a/src/nsfw_onnx_providers.cpp b/src/nsfw_onnx_providers.cpp index 81907f7..5b462f0 100644 --- a/src/nsfw_onnx_providers.cpp +++ b/src/nsfw_onnx_providers.cpp @@ -231,6 +231,21 @@ static std::string nsfw_get_env_string(const char *name) return std::string(value); } +/* Threads per ONNX session. Defaults to 1 because the plugin usually runs + * one CPU worker per core; when it runs a single worker (GPU providers, + * hardware backends) it sets NSFW_ONNX_INTRA_OP_THREADS so that one session + * can use several cores instead of falling behind real time. */ +static int nsfw_get_intra_op_threads() +{ + std::string value = nsfw_get_env_string("NSFW_ONNX_INTRA_OP_THREADS"); + char *end = nullptr; + long threads = value.empty() ? 1 : std::strtol(value.c_str(), &end, 10); + + if (value.empty() || end == value.c_str() || *end != '\0' || threads < 1) + return 1; + return threads > 16 ? 16 : static_cast(threads); +} + static std::string nsfw_ascii_lower(std::string value) { std::transform(value.begin(), value.end(), value.begin(), @@ -601,7 +616,7 @@ int nsfw_onnx_load_model(void *ctx, const char *model_path) } Ort::SessionOptions opts; - opts.SetIntraOpNumThreads(1); + opts.SetIntraOpNumThreads(nsfw_get_intra_op_threads()); opts.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); oc->provider_name = onnx_configure_providers(&opts); if (oc->provider_name == "unavailable") diff --git a/src/plugin/nsfw_filter_worker.c b/src/plugin/nsfw_filter_worker.c index 490ab4e..82de0fd 100644 --- a/src/plugin/nsfw_filter_worker.c +++ b/src/plugin/nsfw_filter_worker.c @@ -1071,6 +1071,11 @@ static void *DetectorWorkerThreadPthread(void *data) return NULL; } #endif +/* The thread count travels to icop_core through the process environment. + * Recompute it for every filter instance unless the user exported it. */ +static bool intra_op_threads_set_by_icop; +static bool intra_op_threads_user_set; + int StartDetectorWorker(filter_sys_t *sys, const nsfw_config_t *cfg) { unsigned i; @@ -1086,6 +1091,26 @@ int StartDetectorWorker(filter_sys_t *sys, const nsfw_config_t *cfg) desired_workers = sys->backend_ops != NULL ? 1 : ResolveWorkerCount(); if (desired_workers == 0) desired_workers = 1; + /* Each ONNX session is single-threaded by default. With fewer workers + * than cores (GPU providers and hardware backends run one), spread the + * spare cores over the sessions; a lone single-threaded worker cannot + * keep up with dense analysis strides. Capped at 4: beyond that ORT + * gains little and competes with VLC's decode and render threads. */ + if (!intra_op_threads_set_by_icop && + getenv("NSFW_ONNX_INTRA_OP_THREADS") != NULL) + intra_op_threads_user_set = true; + if (!intra_op_threads_user_set) { + unsigned cpus = nsfw_plat_cpu_count(); + unsigned threads = cpus > desired_workers ? cpus / desired_workers : 1; + + if (threads > 4) + threads = 4; + nsfw_plat_set_env_unsigned("NSFW_ONNX_INTRA_OP_THREADS", threads); + intra_op_threads_set_by_icop = true; + fprintf(stderr, + "icop: %u detector worker(s), %u ONNX thread(s) each\n", + desired_workers, threads); + } create_detector_on_thread = ProviderEnvWantsGpu(); #ifdef _WIN32 use_cuda_host = ProviderEnvWantsGpu();