diff --git a/CHANGELOG.md b/CHANGELOG.md index abcee25..3c23e81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,18 @@ first, and short hashes identify the commit that introduced each change. - Add `install_icop_plugin.sh --uninstall`, and publish releases to the `asayed18/homebrew-icop` tap (`brew install --cask icop`). A new workflow notifies the tap when a release is published. +- Decline VideoToolbox (CVPX) frames so VLC inserts its `cvpx` converter + and icop runs on software frames. Previously blocked frames were dropped + (a black screen instead of blur or warning), the debug overlay never showed, + and synchronous analysis held decoder buffers until VideoToolbox stalled + (`pic_holder_wait timed out`). The opaque fallback is kept for VLC builds + without the `cvpx` module, so icop never ends up playing video unfiltered. + The macOS smoke test now fails if blocked frames would be dropped. +- Give each ONNX session several threads when icop runs fewer detector + workers than CPU cores (cores / workers, at most 4). GPU providers such as + CoreML run one worker, and with one thread marqo needed ~120 ms per frame, + so automatic strides fell behind real time (a 720p30 file ended up seconds + late). `NSFW_ONNX_INTRA_OP_THREADS` overrides the value. - Bump the version to 0.1.7 so the fixed macOS packages do not reuse the broken v0.1.6 draft. diff --git a/src/nsfw_onnx_providers.cpp b/src/nsfw_onnx_providers.cpp index 81907f7..5b462f0 100644 --- a/src/nsfw_onnx_providers.cpp +++ b/src/nsfw_onnx_providers.cpp @@ -231,6 +231,21 @@ static std::string nsfw_get_env_string(const char *name) return std::string(value); } +/* Threads per ONNX session. Defaults to 1 because the plugin usually runs + * one CPU worker per core; when it runs a single worker (GPU providers, + * hardware backends) it sets NSFW_ONNX_INTRA_OP_THREADS so that one session + * can use several cores instead of falling behind real time. */ +static int nsfw_get_intra_op_threads() +{ + std::string value = nsfw_get_env_string("NSFW_ONNX_INTRA_OP_THREADS"); + char *end = nullptr; + long threads = value.empty() ? 1 : std::strtol(value.c_str(), &end, 10); + + if (value.empty() || end == value.c_str() || *end != '\0' || threads < 1) + return 1; + return threads > 16 ? 16 : static_cast(threads); +} + static std::string nsfw_ascii_lower(std::string value) { std::transform(value.begin(), value.end(), value.begin(), @@ -601,7 +616,7 @@ int nsfw_onnx_load_model(void *ctx, const char *model_path) } Ort::SessionOptions opts; - opts.SetIntraOpNumThreads(1); + opts.SetIntraOpNumThreads(nsfw_get_intra_op_threads()); opts.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); oc->provider_name = onnx_configure_providers(&opts); if (oc->provider_name == "unavailable") diff --git a/src/plugin/nsfw_backend.c b/src/plugin/nsfw_backend.c index d60c7a8..dd039e0 100644 --- a/src/plugin/nsfw_backend.c +++ b/src/plugin/nsfw_backend.c @@ -28,6 +28,7 @@ # include "nsfw_filter_vaapi.h" #endif #include "frame_processor.h" +#include "platform_abstraction.h" /* ------------------------------------------------------------------ */ /* D3D11 backend wrappers */ @@ -203,11 +204,59 @@ static vlc_fourcc_t PreferredOpaqueChroma(vlc_fourcc_t opaque_chroma) } } +static bool IsCvpxChroma(vlc_fourcc_t chroma) +{ + switch (chroma) { +#ifdef VLC_CODEC_CVPX_NV12 + case VLC_CODEC_CVPX_NV12: +#endif +#ifdef VLC_CODEC_CVPX_UYVY + case VLC_CODEC_CVPX_UYVY: +#endif +#ifdef VLC_CODEC_CVPX_I420 + case VLC_CODEC_CVPX_I420: +#endif +#ifdef VLC_CODEC_CVPX_BGRA + case VLC_CODEC_CVPX_BGRA: +#endif +#ifdef VLC_CODEC_CVPX_P010 + case VLC_CODEC_CVPX_P010: +#endif + return true; + default: + return false; + } +} + +/* VideoToolbox frames can be neither blurred nor drawn on in place, and + * the opaque fallback analyses them synchronously while holding decoder + * buffers, which stalls VideoToolbox. Declining them makes VLC's "chain" + * filter insert its cvpx converter and reopen icop with a software chroma, + * so the normal CPU path (async worker, block styles, overlay) runs. Only + * decline when that converter exists: otherwise VLC would drop icop from + * the chain and play the video unfiltered. */ +static bool CvpxSoftwareConverterAvailable(void) +{ + typedef bool (*module_exists_fn)(const char *); + module_exists_fn exists = + (module_exists_fn)nsfw_plat_lookup_vlc_sym("module_exists"); + + return exists != NULL && exists("cvpx"); +} + int nsfw_backend_open(filter_t *filter) { filter_sys_t *sys = filter->p_sys; vlc_fourcc_t chroma = filter->fmt_in.video.i_chroma; + if (IsCvpxChroma(chroma) && CvpxSoftwareConverterAvailable()) { + fprintf(stderr, + "icop: declining VideoToolbox chroma %4.4s;" + " VLC will convert frames to a software chroma\n", + (const char *)&chroma); + return VLC_EGENERIC; + } + if (nsfw_d3d11_is_opaque(chroma)) { nsfw_d3d11_backend_t *d3d11 = NULL; if (nsfw_d3d11_open(filter, &d3d11) != VLC_SUCCESS) { diff --git a/src/plugin/nsfw_filter_worker.c b/src/plugin/nsfw_filter_worker.c index 490ab4e..82de0fd 100644 --- a/src/plugin/nsfw_filter_worker.c +++ b/src/plugin/nsfw_filter_worker.c @@ -1071,6 +1071,11 @@ static void *DetectorWorkerThreadPthread(void *data) return NULL; } #endif +/* The thread count travels to icop_core through the process environment. + * Recompute it for every filter instance unless the user exported it. */ +static bool intra_op_threads_set_by_icop; +static bool intra_op_threads_user_set; + int StartDetectorWorker(filter_sys_t *sys, const nsfw_config_t *cfg) { unsigned i; @@ -1086,6 +1091,26 @@ int StartDetectorWorker(filter_sys_t *sys, const nsfw_config_t *cfg) desired_workers = sys->backend_ops != NULL ? 1 : ResolveWorkerCount(); if (desired_workers == 0) desired_workers = 1; + /* Each ONNX session is single-threaded by default. With fewer workers + * than cores (GPU providers and hardware backends run one), spread the + * spare cores over the sessions; a lone single-threaded worker cannot + * keep up with dense analysis strides. Capped at 4: beyond that ORT + * gains little and competes with VLC's decode and render threads. */ + if (!intra_op_threads_set_by_icop && + getenv("NSFW_ONNX_INTRA_OP_THREADS") != NULL) + intra_op_threads_user_set = true; + if (!intra_op_threads_user_set) { + unsigned cpus = nsfw_plat_cpu_count(); + unsigned threads = cpus > desired_workers ? cpus / desired_workers : 1; + + if (threads > 4) + threads = 4; + nsfw_plat_set_env_unsigned("NSFW_ONNX_INTRA_OP_THREADS", threads); + intra_op_threads_set_by_icop = true; + fprintf(stderr, + "icop: %u detector worker(s), %u ONNX thread(s) each\n", + desired_workers, threads); + } create_detector_on_thread = ProviderEnvWantsGpu(); #ifdef _WIN32 use_cuda_host = ProviderEnvWantsGpu(); diff --git a/tools/macos_vlc_smoke_test.sh b/tools/macos_vlc_smoke_test.sh index 8fa4e35..f62dc83 100755 --- a/tools/macos_vlc_smoke_test.sh +++ b/tools/macos_vlc_smoke_test.sh @@ -51,7 +51,8 @@ for failure in \ 'unable to load the packaged ONNX Runtime' \ 'ONNX detector unavailable' \ 'detector initialization failed' \ - 'does not support C API version'; do + 'does not support C API version' \ + 'blocked frames will be dropped fail-closed'; do if grep -q "$failure" "$log_file"; then die "VLC reported: $failure (see $log_file)" fi