diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b62a286..32a6ccb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,7 +71,7 @@ jobs: - name: Install ffmpeg (macOS) if: runner.os == 'macOS' - run: brew install ffmpeg + run: brew install ffmpeg@7 - name: Fetch bundled ffmpeg (Windows) if: runner.os == 'Windows' diff --git a/README.md b/README.md index 661f57d..8c3b530 100644 --- a/README.md +++ b/README.md @@ -132,8 +132,12 @@ a later release. `/dev/dri/renderD128` and your user in the `render` (or `video`) group. Credentials use `secret-tool` against the GNOME/KDE keyring, with an AES-GCM file fallback if D-Bus is unavailable. -- **macOS** — Homebrew FFmpeg (`brew install ffmpeg`). VideoToolbox HW decode - works on any Mac 12+. Credentials live in the Keychain via `security`. +- **macOS** — Homebrew FFmpeg 7 (`brew install ffmpeg@7`). The app binds the + FFmpeg 7.x ABI (`libavcodec.61`), and Homebrew's plain `ffmpeg` formula is + newer than that, so it won't load. `ffmpeg@7` is keg-only; the app finds it + under the Homebrew prefix by itself, no `DYLD_LIBRARY_PATH` needed. + VideoToolbox HW decode works on any Mac 12+; a stream it can't decode falls + back to software. Credentials live in the Keychain via `security`. > **Validation caveat.** Linux / macOS / Android / iOS code paths build + link > in CI but aren't yet end-to-end tested on real devices for every commit. @@ -178,7 +182,7 @@ Node the build still succeeds and the server runs API-only; pass Run the FFmpeg fetch script for your OS once — it downloads the bundled shared-build (`n7.1` ABI) from `BtbN/FFmpeg-Builds` into `runtimes//native/`: `tools/fetch-ffmpeg.ps1` (Windows → `win-x64`) or `tools/fetch-ffmpeg-linux.sh` -(Linux → `linux-x64`). macOS uses the system Homebrew FFmpeg (see +(Linux → `linux-x64`). macOS uses Homebrew's `ffmpeg@7` (see [Runtime requirements](#runtime-requirements)). ### Android diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index b04ecb4..192aed0 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -45,6 +45,20 @@ strictly weaker than a real keyring. .app → *Open*. macOS remembers the choice after one confirm. Signing + notarization land in a later release-polish phase. +**`FFmpeg native libraries failed to load for runtime osx-…`.** The app +needs the FFmpeg **7.x** ABI (`libavcodec.61`). Homebrew's plain `ffmpeg` +formula is newer (8.x+) and won't load — install `brew install ffmpeg@7`. +It's keg-only, but the app looks under `/opt/homebrew/opt/ffmpeg@7/lib` +(Apple Silicon) and `/usr/local/opt/ffmpeg@7/lib` (Intel) on its own, so +no `DYLD_LIBRARY_PATH` is needed. + +**Camera connects but the tile stays black, log full of `vt decoder cb` +/ `hardware accelerator failed to decode picture`.** VideoToolbox rejects +HEVC frames whose reference picture is missing (typical when joining a +smart-GOP stream mid-GOP). After ~2.5s of such errors with no frame the app +switches that camera to software decode until restart; if it still stays +black, attach the log to an issue. + **Keychain prompt on every credential read.** Means the app's codesign identity changed between builds. Trust the keychain access permanently once and it goes away (or re-add the camera). diff --git a/docs/web-server.md b/docs/web-server.md index 8bf587a..36f7cfe 100644 --- a/docs/web-server.md +++ b/docs/web-server.md @@ -138,11 +138,13 @@ lost, so keep its password safe. ## ffmpeg Live video needs ffmpeg. Resolution order: a bundled binary next to the app -(`runtimes//native/ffmpeg`) first, then `ffmpeg` on `PATH`. +(`runtimes//native/ffmpeg`) first, then (macOS) the Homebrew prefixes, +then `ffmpeg` on `PATH`. * **Windows** — bundled in the release archive. Nothing to install. * **Linux** — bundled in the release archive; or `sudo apt install ffmpeg`. -* **macOS** — install via `brew install ffmpeg` (not bundled). +* **macOS** — not bundled. Install `brew install ffmpeg@7`; the app finds + it under the Homebrew prefix itself (keg-only, no `PATH` change needed). --- diff --git a/docs/web-server.ru.md b/docs/web-server.ru.md index 8fc87cb..61184bb 100644 --- a/docs/web-server.ru.md +++ b/docs/web-server.ru.md @@ -135,11 +135,14 @@ OpenIPC-камерам по локальной сети. Без облака и ## ffmpeg Живому видео нужен ffmpeg. Порядок поиска: сначала бандл рядом с приложением -(`runtimes//native/ffmpeg`), затем `ffmpeg` в `PATH`. +(`runtimes//native/ffmpeg`), затем (macOS) префиксы Homebrew, затем +`ffmpeg` в `PATH`. * **Windows** — в комплекте с релиз-архивом. Ничего ставить не нужно. * **Linux** — в комплекте с релиз-архивом; либо `sudo apt install ffmpeg`. -* **macOS** — поставьте через `brew install ffmpeg` (в комплект не входит). +* **macOS** — в комплект не входит. Поставьте `brew install ffmpeg@7`; + приложение само найдёт его в префиксе Homebrew (keg-only, `PATH` править + не нужно). --- diff --git a/src/OpenIPC.Viewer.Core/Video/FfmpegExecutablePaths.cs b/src/OpenIPC.Viewer.Core/Video/FfmpegExecutablePaths.cs new file mode 100644 index 0000000..00e579a --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Video/FfmpegExecutablePaths.cs @@ -0,0 +1,16 @@ +namespace OpenIPC.Viewer.Core.Video; + +// Where to look for the ffmpeg CLI on macOS, which ships no bundled binary. +// Homebrew's keg-only ffmpeg@7 isn't on PATH, and a Finder-launched app +// doesn't see /opt/homebrew/bin either. Shared by the Video subprocess +// recorder/exporter and the web live remuxer, which don't reference each other. +public static class FfmpegExecutablePaths +{ + public static readonly string[] MacOs = + { + "/opt/homebrew/opt/ffmpeg@7/bin/ffmpeg", // Apple Silicon Homebrew + "/usr/local/opt/ffmpeg@7/bin/ffmpeg", // Intel Homebrew + "/opt/homebrew/bin/ffmpeg", + "/usr/local/bin/ffmpeg", + }; +} diff --git a/src/OpenIPC.Viewer.Video/Pipeline/FfmpegRuntime.cs b/src/OpenIPC.Viewer.Video/Pipeline/FfmpegRuntime.cs index a9e63c2..992672d 100644 --- a/src/OpenIPC.Viewer.Video/Pipeline/FfmpegRuntime.cs +++ b/src/OpenIPC.Viewer.Video/Pipeline/FfmpegRuntime.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Linq; using System.Runtime.InteropServices; using System.Threading; using FFmpeg.AutoGen.Abstractions; @@ -121,8 +122,9 @@ public static void EnsureInitialized() $"(run tools/build-ffmpeg-android.sh via WSL or pull .so from a CI APK artifact). " + $"Desktop: keep the runtimes/ folder from the release archive next to the exe, " + $"or run tools/fetch-ffmpeg.ps1 (Windows) / tools/fetch-ffmpeg-linux.sh (Linux) / " + - $"brew install ffmpeg (macOS). NOTE: the app needs FFmpeg 7.x (libavcodec.so.61); " + - $"distro packages like Ubuntu 24.04's are FFmpeg 6.1 (libavcodec.so.60) and will not load. " + + $"brew install ffmpeg@7 (macOS). NOTE: the app needs FFmpeg 7.x (libavcodec 61); " + + $"distro packages like Ubuntu 24.04's are FFmpeg 6.1 (libavcodec.so.60) and will not load, " + + $"and neither does Homebrew's current `ffmpeg` formula (8.x+). " + $"Underlying: {DescribeChain(ex)}", ex); _ready = true; throw _initFailure; @@ -199,11 +201,37 @@ private static string ResolveNativeDir() return candidate; } - // Fall back to "" — FFmpeg.AutoGen then uses the platform loader path + if (OperatingSystem.IsMacOS() && FindMacFfmpegDir() is { } macDir) + return macDir; + + // Fall back to ""— FFmpeg.AutoGen then uses the platform loader path // (apt/brew-installed libs on Linux/macOS, DLLs next to the exe on Windows, // and the bundled .so files added via on Android). return ""; } + + // macOS has no bundled FFmpeg. Homebrew's plain `ffmpeg` formula has moved + // past 7.x, and `ffmpeg@7` is keg-only — its libs aren't on any default + // dlopen path, so without this users had to export DYLD_LIBRARY_PATH (#94). + // DYLD_LIBRARY_PATH entries go first so an explicit choice still wins. + private static string? FindMacFfmpegDir() + { + // Must match the libavcodec major FFmpeg.AutoGen 7.1 binds against. + const string probeLib = "libavcodec.61.dylib"; + + var dirs = (Environment.GetEnvironmentVariable("DYLD_LIBRARY_PATH") ?? "") + .Split(':', StringSplitOptions.RemoveEmptyEntries) + .Concat(new[] + { + "/opt/homebrew/opt/ffmpeg@7/lib", // Apple Silicon Homebrew + "/usr/local/opt/ffmpeg@7/lib", // Intel Homebrew + "/opt/homebrew/lib", + "/usr/local/lib", + "/opt/local/lib", // MacPorts + }); + + return dirs.FirstOrDefault(d => File.Exists(Path.Combine(d, probeLib))); + } } public sealed class FfmpegNativeLibsMissingException : Exception diff --git a/src/OpenIPC.Viewer.Video/Pipeline/FfmpegVideoSession.cs b/src/OpenIPC.Viewer.Video/Pipeline/FfmpegVideoSession.cs index 9450d1d..83fb083 100644 --- a/src/OpenIPC.Viewer.Video/Pipeline/FfmpegVideoSession.cs +++ b/src/OpenIPC.Viewer.Video/Pipeline/FfmpegVideoSession.cs @@ -1,5 +1,6 @@ using System; using System.Buffers; +using System.Collections.Concurrent; using System.Reactive.Subjects; using System.Runtime.InteropServices; using System.Threading; @@ -71,6 +72,18 @@ internal sealed class FfmpegVideoSession : IVideoSession private AVCodecContext_get_format? _getFormatDelegate; private AVPixelFormat _selectedHwPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; + // How long HW decode may keep failing (send/receive/transfer errors, no + // frame out) before the session drops to software. Time-based, not a + // count: a 1 fps camera and a 30 fps one get the same grace for the next + // keyframe, and both switch well inside the reconnect watchdog's 5s + // no-frame window, which would otherwise restart HW from scratch. + private const long HwFallbackAfterMs = 2_500; + // Streams whose HW decode broke during this run. Static because every + // reconnect builds a new session; keyed without credentials. 2.5s of + // nothing but errors means HW can't handle this stream, not a blip. + private static readonly ConcurrentDictionary HwFailedStreams = new(); + private string HwStreamKey => _options.RtspUri.GetComponents(UriComponents.HttpRequestUrl, UriFormat.UriEscaped); + // Snapshot of the most recently decoded frame, kept around for SnapshotAsync. private byte[]? _snapshotBgra; private int _snapshotWidth; @@ -197,6 +210,8 @@ private unsafe void Run() var audioProbedNoTrack = false; var swsSrcPixFmt = AVPixelFormat.AV_PIX_FMT_NONE; var hwActive = false; + var hwErrors = 0; + var lastFrameMs = 0L; try { @@ -240,33 +255,17 @@ private unsafe void Run() if (codec == null) throw new InvalidOperationException($"No decoder available for codec id {codecpar->codec_id}"); - codecCtx = ffmpeg.avcodec_alloc_context3(codec); - ret = ffmpeg.avcodec_parameters_to_context(codecCtx, codecpar); - FfmpegError.ThrowIfError(ret, "avcodec_parameters_to_context"); - - // HW accel decision: pure policy in Resolve, native wiring inline. - // Any failure here logs and continues with software decode. + // HW accel decision: pure policy in Resolve, native wiring in + // OpenDecoder. Any failure there logs and continues with software + // decode. A stream that already broke the HW decoder in this run + // goes straight to software, so a reconnect doesn't repeat it. var resolved = HwAccelSelector.Resolve(_options.HwAccel, _hwFactory, _logger); - if (resolved != HwAccelHint.None) - hwActive = TryEnableHw(codecCtx, resolved, &hwDeviceCtx); - - ret = ffmpeg.avcodec_open2(codecCtx, codec, null); - if (ret < 0 && hwActive) + if (resolved != HwAccelHint.None && HwFailedStreams.ContainsKey(HwStreamKey)) { - // HW open failed: tear down HW state and retry as software. - _logger.LogWarning("HW-accelerated avcodec_open2 failed: {Err}; retrying software", FfmpegError.Describe(ret)); - TearDownHw(codecCtx, &hwDeviceCtx); - hwActive = false; - ret = ffmpeg.avcodec_open2(codecCtx, codec, null); + _logger.LogInformation("HW decode skipped for {Host}: it failed on this stream earlier", _options.RtspUri.Host); + resolved = HwAccelHint.None; } - FfmpegError.ThrowIfError(ret, "avcodec_open2"); - - _width = codecCtx->width; - _height = codecCtx->height; - var baseName = Marshal.PtrToStringAnsi((IntPtr)codec->name); - _codecName = hwActive ? $"{baseName} ({resolved})" : baseName; - if (hwActive) - _logger.LogInformation("HW decode active: {Codec} via {Hint}", baseName, resolved); + hwActive = OpenDecoder(&codecCtx, &hwDeviceCtx, codecpar, codec, resolved); packet = ffmpeg.av_packet_alloc(); frame = ffmpeg.av_frame_alloc(); @@ -274,6 +273,7 @@ private unsafe void Run() SetState(SessionState.Playing); _lastFpsTick = DateTime.UtcNow; + lastFrameMs = Environment.TickCount64; var ct = _cts!.Token; while (!ct.IsCancellationRequested) @@ -285,6 +285,8 @@ private unsafe void Run() { try { _decodeGate.Wait(ct); } catch (OperationCanceledException) { break; } + // Time parked isn't time HW spent failing. + lastFrameMs = Environment.TickCount64; } // Lazily spin the audio decoder up/down to track SetAudioEnabled. @@ -335,11 +337,30 @@ private unsafe void Run() // Demux-level byte count → video bitrate in MaybePublishTelemetry. Interlocked.Add(ref _bytesSinceFpsTick, packet->size); + // The HW decoder can open fine and then reject every frame — + // VideoToolbox does this on HEVC when the stream is joined + // mid-GOP and a reference picture is missing (#94), where the + // software decoder would conceal and carry on. Swap to software + // once it has failed for HwFallbackAfterMs without a frame, + // instead of stalling into a reconnect loop. + var sinceFrameMs = Environment.TickCount64 - lastFrameMs; + if (hwActive && hwErrors > 0 && sinceFrameMs >= HwFallbackAfterMs) + { + _logger.LogWarning("HW decode failed {Count} times over {Ms} ms on {Host}; switching to software", + hwErrors, sinceFrameMs, _options.RtspUri.Host); + HwFailedStreams.TryAdd(HwStreamKey, 0); + ffmpeg.avcodec_free_context(&codecCtx); + if (hwDeviceCtx != null) ffmpeg.av_buffer_unref(&hwDeviceCtx); + hwActive = OpenDecoder(&codecCtx, &hwDeviceCtx, codecpar, codec, HwAccelHint.None); + hwErrors = 0; + } + ret = ffmpeg.avcodec_send_packet(codecCtx, packet); ffmpeg.av_packet_unref(packet); if (ret < 0 && ret != ffmpeg.AVERROR(ffmpeg.EAGAIN)) { _logger.LogWarning("avcodec_send_packet failed: {Err}", FfmpegError.Describe(ret)); + if (hwActive) hwErrors++; continue; } @@ -351,6 +372,7 @@ private unsafe void Run() if (ret < 0) { _logger.LogWarning("avcodec_receive_frame failed: {Err}", FfmpegError.Describe(ret)); + if (hwActive) hwErrors++; break; } @@ -361,11 +383,14 @@ private unsafe void Run() if (transferRet < 0) { _logger.LogWarning("av_hwframe_transfer_data failed: {Err}", FfmpegError.Describe(transferRet)); + hwErrors++; ffmpeg.av_frame_unref(frame); continue; } presentable = swFrame; } + hwErrors = 0; + lastFrameMs = Environment.TickCount64; // Lazy sws init — the actual sw pixfmt is only known once // the first frame arrives (matters for HW path where the @@ -432,6 +457,38 @@ private void ArmIoDeadline(long budgetMs) => private unsafe int OnInterrupt(void* opaque) => _abortIo || Environment.TickCount64 > Volatile.Read(ref _ioDeadlineMs) ? 1 : 0; + // Allocates and opens the video decoder into *codecCtx (assigned before + // anything can throw, so the caller's cleanup frees it). Returns whether + // HW decode ended up active; a failed HW open retries as software. + private unsafe bool OpenDecoder(AVCodecContext** codecCtx, AVBufferRef** hwDeviceCtx, + AVCodecParameters* codecpar, AVCodec* codec, HwAccelHint hint) + { + var ctx = *codecCtx = ffmpeg.avcodec_alloc_context3(codec); + var ret = ffmpeg.avcodec_parameters_to_context(ctx, codecpar); + FfmpegError.ThrowIfError(ret, "avcodec_parameters_to_context"); + + var hwActive = hint != HwAccelHint.None && TryEnableHw(ctx, hint, hwDeviceCtx); + + ret = ffmpeg.avcodec_open2(ctx, codec, null); + if (ret < 0 && hwActive) + { + // HW open failed: tear down HW state and retry as software. + _logger.LogWarning("HW-accelerated avcodec_open2 failed: {Err}; retrying software", FfmpegError.Describe(ret)); + TearDownHw(ctx, hwDeviceCtx); + hwActive = false; + ret = ffmpeg.avcodec_open2(ctx, codec, null); + } + FfmpegError.ThrowIfError(ret, "avcodec_open2"); + + _width = ctx->width; + _height = ctx->height; + var baseName = Marshal.PtrToStringAnsi((IntPtr)codec->name); + _codecName = hwActive ? $"{baseName} ({hint})" : baseName; + if (hwActive) + _logger.LogInformation("HW decode active: {Codec} via {Hint}", baseName, hint); + return hwActive; + } + private unsafe bool TryEnableHw(AVCodecContext* ctx, HwAccelHint hint, AVBufferRef** outDeviceCtx) { var (deviceType, hwPixFmt) = HwAccelSelector.MapToFfmpeg(hint); diff --git a/src/OpenIPC.Viewer.Video/Pipeline/RuntimeIds.cs b/src/OpenIPC.Viewer.Video/Pipeline/RuntimeIds.cs index 1792f65..0b50676 100644 --- a/src/OpenIPC.Viewer.Video/Pipeline/RuntimeIds.cs +++ b/src/OpenIPC.Viewer.Video/Pipeline/RuntimeIds.cs @@ -1,5 +1,7 @@ using System; +using System.IO; using System.Runtime.InteropServices; +using OpenIPC.Viewer.Core.Video; namespace OpenIPC.Viewer.Video.Pipeline; @@ -25,4 +27,21 @@ public static (string? Rid, string ExeName) Current() return ("linux-x64", "ffmpeg"); return (null, "ffmpeg"); } + + // The ffmpeg CLI for the subprocess recorder / clip exporter: the bundled + // binary, else (macOS) Homebrew's, else PATH. + public static string ResolveFfmpegExe() + { + var (rid, exe) = Current(); + if (rid is not null) + { + var bundled = Path.Combine(AppContext.BaseDirectory, "runtimes", rid, "native", exe); + if (File.Exists(bundled)) return bundled; + } + + if (OperatingSystem.IsMacOS()) + foreach (var path in FfmpegExecutablePaths.MacOs) + if (File.Exists(path)) return path; + return "ffmpeg"; + } } diff --git a/src/OpenIPC.Viewer.Video/Recording/FfmpegSubprocessClipExporter.cs b/src/OpenIPC.Viewer.Video/Recording/FfmpegSubprocessClipExporter.cs index 2665cd5..cd3ce8c 100644 --- a/src/OpenIPC.Viewer.Video/Recording/FfmpegSubprocessClipExporter.cs +++ b/src/OpenIPC.Viewer.Video/Recording/FfmpegSubprocessClipExporter.cs @@ -114,14 +114,5 @@ public async Task ExportAsync(ClipExportRequest request, IProgress? prog progress?.Report(1.0); } - private static string ResolveDefault() - { - var (rid, exe) = RuntimeIds.Current(); - if (rid is not null) - { - var bundled = Path.Combine(AppContext.BaseDirectory, "runtimes", rid, "native", exe); - if (File.Exists(bundled)) return bundled; - } - return "ffmpeg"; - } + private static string ResolveDefault() => RuntimeIds.ResolveFfmpegExe(); } diff --git a/src/OpenIPC.Viewer.Video/Recording/FfmpegSubprocessRecorder.cs b/src/OpenIPC.Viewer.Video/Recording/FfmpegSubprocessRecorder.cs index 43f56e8..e151d56 100644 --- a/src/OpenIPC.Viewer.Video/Recording/FfmpegSubprocessRecorder.cs +++ b/src/OpenIPC.Viewer.Video/Recording/FfmpegSubprocessRecorder.cs @@ -15,8 +15,9 @@ namespace OpenIPC.Viewer.Video.Recording; // FFmpeg path resolution (first hit wins): // 1. Explicit override (constructor / appsettings "Recording:FfmpegPath") // 2. Per-RID bundled binary in runtimes/{rid}/native/ (win-x64 via -// fetch-ffmpeg.ps1, linux-x64 via fetch-ffmpeg-linux.sh; osx still TODO) -// 3. "ffmpeg" — falls through to OS PATH lookup (apt/brew/PATH on *nix) +// fetch-ffmpeg.ps1, linux-x64 via fetch-ffmpeg-linux.sh) +// 3. macOS: Homebrew ffmpeg@7 / ffmpeg at their fixed prefixes +// 4. "ffmpeg" — falls through to OS PATH lookup (apt/PATH on *nix) public sealed class FfmpegSubprocessRecorder : IRecorder { private readonly ILoggerFactory _loggerFactory; @@ -37,14 +38,5 @@ public Task StartAsync(RecordingOptions options, Cancellation return Task.FromResult(session); } - private static string ResolveDefault() - { - var (rid, exe) = RuntimeIds.Current(); - if (rid is not null) - { - var bundled = Path.Combine(AppContext.BaseDirectory, "runtimes", rid, "native", exe); - if (File.Exists(bundled)) return bundled; - } - return "ffmpeg"; - } + private static string ResolveDefault() => RuntimeIds.ResolveFfmpegExe(); } diff --git a/src/OpenIPC.Viewer.Web/Api/LiveFfmpeg.cs b/src/OpenIPC.Viewer.Web/Api/LiveFfmpeg.cs index 709963f..e13917c 100644 --- a/src/OpenIPC.Viewer.Web/Api/LiveFfmpeg.cs +++ b/src/OpenIPC.Viewer.Web/Api/LiveFfmpeg.cs @@ -3,9 +3,11 @@ using System.Diagnostics; using System.Globalization; using System.IO; +using System.Runtime.InteropServices; using System.Text.RegularExpressions; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using OpenIPC.Viewer.Core.Video; namespace OpenIPC.Viewer.Web.Api; @@ -129,7 +131,8 @@ private static string Resolve() var exe = "ffmpeg"; if (OperatingSystem.IsWindows()) { rid = "win-x64"; exe = "ffmpeg.exe"; } else if (OperatingSystem.IsLinux()) { rid = "linux-x64"; } - else if (OperatingSystem.IsMacOS()) { rid = "osx-x64"; } + else if (OperatingSystem.IsMacOS()) + rid = RuntimeInformation.OSArchitecture == Architecture.Arm64 ? "osx-arm64" : "osx-x64"; if (rid is not null) { @@ -137,6 +140,10 @@ private static string Resolve() if (File.Exists(bundled)) return bundled; } + + if (OperatingSystem.IsMacOS()) + foreach (var path in FfmpegExecutablePaths.MacOs) + if (File.Exists(path)) return path; return "ffmpeg"; } }