From df59aa0af87e9aa697747fb359c354e6bbe9f8a2 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:34:23 +0200 Subject: [PATCH 1/2] fix(audio): let an audio output reload reach the next output mpv's reload left the frame it had handed to the old output in the ao filter, which then never asked the chain for more, and never woke the output chain after resetting it, so the next output never opened. Both patches come from kristofferR/mpv-build (pool 0034 and 0102). --- ...the-frame-held-for-the-old-ao-on-reload.patch | 16 ++++++++++++++++ ...-f-output-chain-wake-after-reset-harder.patch | 14 ++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 Sources/BuildScripts/patch/libmpv/0008-player-drop-the-frame-held-for-the-old-ao-on-reload.patch create mode 100644 Sources/BuildScripts/patch/libmpv/0009-f-output-chain-wake-after-reset-harder.patch diff --git a/Sources/BuildScripts/patch/libmpv/0008-player-drop-the-frame-held-for-the-old-ao-on-reload.patch b/Sources/BuildScripts/patch/libmpv/0008-player-drop-the-frame-held-for-the-old-ao-on-reload.patch new file mode 100644 index 00000000..2b9e4ee4 --- /dev/null +++ b/Sources/BuildScripts/patch/libmpv/0008-player-drop-the-frame-held-for-the-old-ao-on-reload.patch @@ -0,0 +1,16 @@ +diff --git a/player/audio.c b/player/audio.c +index b3e975d..c7e5650 100644 +--- a/player/audio.c ++++ b/player/audio.c +@@ -776,6 +776,11 @@ void reload_audio_output(struct MPContext *mpctx) + if (ao_c) { + reset_audio_state(mpctx); + mp_output_chain_reset_harder(ao_c->filter); ++ // A frame the chain already handed over for the old AO would stay in ++ // the ao filter's input. Without a queue, ao_process() only peeks at ++ // it, so no request reaches the chain and the next AO never opens. ++ mp_filter_reset(ao_c->ao_filter); ++ mp_filter_wakeup(ao_c->ao_filter); + } + + // Whether we can use spdif might have changed. If we failed to use spdif diff --git a/Sources/BuildScripts/patch/libmpv/0009-f-output-chain-wake-after-reset-harder.patch b/Sources/BuildScripts/patch/libmpv/0009-f-output-chain-wake-after-reset-harder.patch new file mode 100644 index 00000000..b1bfa013 --- /dev/null +++ b/Sources/BuildScripts/patch/libmpv/0009-f-output-chain-wake-after-reset-harder.patch @@ -0,0 +1,14 @@ +diff --git a/filters/f_output_chain.c b/filters/f_output_chain.c +index 43ec3db..4bbbdde 100644 +--- a/filters/f_output_chain.c ++++ b/filters/f_output_chain.c +@@ -339,6 +339,9 @@ void mp_output_chain_reset_harder(struct mp_output_chain *c) + p->ao = NULL; + mp_autoconvert_clear(p->convert); + } ++ ++ // Re-propagate the still-asserted downstream request. ++ mp_filter_wakeup(p->f); + } + + static void output_chain_destroy(struct mp_filter *f) From bcf22a7e440ae33a138d550341852e249cbd1f06 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:01:00 +0200 Subject: [PATCH 2/2] feat(audio): hand TrueHD and DTS-HD passthrough to the host as spatial audio mpv decodes TrueHD and DTS-HD to PCM, so the Atmos objects and DTS:X never reach the receiver. Patch 0007 adds a "spatial" audio output that hands the passthrough bursts to a host sink (mpv/spatial_audio.h, sink version 2) and keeps mpv's clock on the AVSampleBufferRenderSynchronizer the host renders into. mpv's volume and mute reach the host through set_volume, and a stream the host fails on falls back to PCM. FFmpeg now builds its spdif muxer, which passthrough needs. The release workflow runs the output's unit test and checks the muxer is in the build. Every other format, and any player that doesn't set the sink, is unaffected. --- .github/workflows/release-combined.yml | 13 + .../BuildScripts/XCFrameworkBuild/main.swift | 2 + .../BuildScripts/create-combined-framework.sh | 1 + .../libmpv/0007-spatial-audio-output.patch | 517 ++++++++++++++++++ scripts/spatial-audio-output-test.m | 310 +++++++++++ scripts/test-spatial-audio-output.sh | 24 + 6 files changed, 867 insertions(+) create mode 100644 Sources/BuildScripts/patch/libmpv/0007-spatial-audio-output.patch create mode 100644 scripts/spatial-audio-output-test.m create mode 100755 scripts/test-spatial-audio-output.sh diff --git a/.github/workflows/release-combined.yml b/.github/workflows/release-combined.yml index 3aa04377..c3a0afeb 100644 --- a/.github/workflows/release-combined.yml +++ b/.github/workflows/release-combined.yml @@ -33,6 +33,9 @@ jobs: fi echo "RELEASE_VERSION=$VERSION" >> "$GITHUB_ENV" + - name: Test the spatial audio output + run: ./scripts/test-spatial-audio-output.sh + - name: Install dependencies run: | brew install autoconf automake libtool ninja @@ -49,6 +52,16 @@ jobs: - name: Create combined MPVKit.xcframework run: ./Sources/BuildScripts/create-combined-framework.sh + # Without FFmpeg's spdif muxer, mpv's passthrough decoder fails on its + # first packet and the audio track dies, so the spatial output never opens. + - name: Check that passthrough is built + run: | + for lib in dist/MPVKit-combined/xcframework/MPVKit.xcframework/*/MPVKit.framework/MPVKit; do + # grep -c reads to the end, so nm never writes into a closed pipe + # (grep -q would, and pipefail would then fail a good build). + [ "$(nm "$lib" | grep -c ff_spdif_muxer || true)" -gt 0 ] || { echo "no spdif muxer in $lib"; exit 1; } + done + - name: Upload binary to GitHub Release if: github.event_name != 'pull_request' uses: softprops/action-gh-release@v2 diff --git a/Sources/BuildScripts/XCFrameworkBuild/main.swift b/Sources/BuildScripts/XCFrameworkBuild/main.swift index 1888591b..b56efe65 100644 --- a/Sources/BuildScripts/XCFrameworkBuild/main.swift +++ b/Sources/BuildScripts/XCFrameworkBuild/main.swift @@ -709,6 +709,8 @@ private class BuildFFMPEG: BaseBuild { "--enable-muxer=flac", "--enable-muxer=dash", "--enable-muxer=hevc", "--enable-muxer=m4v", "--enable-muxer=matroska", "--enable-muxer=mov", "--enable-muxer=mp4", "--enable-muxer=mpegts", "--enable-muxer=webm*", + // mpv's passthrough (--audio-spdif) packs audio into IEC 61937 with it. + "--enable-muxer=spdif", // ./configure --list-encoders "--disable-encoders", "--enable-encoder=aac", "--enable-encoder=alac", "--enable-encoder=flac", "--enable-encoder=pcm*", diff --git a/Sources/BuildScripts/create-combined-framework.sh b/Sources/BuildScripts/create-combined-framework.sh index 71ad7ced..7e8ca927 100755 --- a/Sources/BuildScripts/create-combined-framework.sh +++ b/Sources/BuildScripts/create-combined-framework.sh @@ -240,6 +240,7 @@ for platform in ios isimulator tvos tvsimulator; do #include "mpv/render.h" #include "mpv/render_gl.h" #include "mpv/stream_cb.h" +#include "mpv/spatial_audio.h" #endif /* MPVKit_h */ UMBRELLA diff --git a/Sources/BuildScripts/patch/libmpv/0007-spatial-audio-output.patch b/Sources/BuildScripts/patch/libmpv/0007-spatial-audio-output.patch new file mode 100644 index 00000000..4f750659 --- /dev/null +++ b/Sources/BuildScripts/patch/libmpv/0007-spatial-audio-output.patch @@ -0,0 +1,517 @@ +diff --git a/audio/out/ao.c b/audio/out/ao.c +index 4e5e9e3..ef2d124 100644 +--- a/audio/out/ao.c ++++ b/audio/out/ao.c +@@ -41,6 +41,7 @@ extern const struct ao_driver audio_out_audiounit; + extern const struct ao_driver audio_out_coreaudio; + extern const struct ao_driver audio_out_coreaudio_exclusive; + extern const struct ao_driver audio_out_avfoundation; ++extern const struct ao_driver audio_out_spatial; + extern const struct ao_driver audio_out_rsound; + extern const struct ao_driver audio_out_pipewire; + extern const struct ao_driver audio_out_sndio; +@@ -107,6 +108,9 @@ static const struct ao_driver * const audio_out_drivers[] = { + &audio_out_null, + #if HAVE_COREAUDIO + &audio_out_coreaudio_exclusive, ++#endif ++#if HAVE_AVFOUNDATION ++ &audio_out_spatial, + #endif + &audio_out_pcm, + &audio_out_lavc, +@@ -613,6 +617,8 @@ void ao_print_devices(struct mpv_global *global, struct mp_log *log, + void ao_set_gain(struct ao *ao, float gain) + { + atomic_store(&ao->gain, gain); ++ if (ao->driver->set_gain) ++ ao->driver->set_gain(ao, gain); + } + + #define MUL_GAIN_i(d, num_samples, gain, low, center, high) \ +diff --git a/audio/out/ao_spatial.m b/audio/out/ao_spatial.m +new file mode 100644 +index 0000000..18d3080 +--- /dev/null ++++ b/audio/out/ao_spatial.m +@@ -0,0 +1,312 @@ ++/* ++ * This file is part of mpv. ++ * ++ * mpv is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU Lesser General Public ++ * License as published by the Free Software Foundation; either ++ * version 2.1 of the License, or (at your option) any later version. ++ * ++ * mpv is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU Lesser General Public License for more details. ++ * ++ * You should have received a copy of the GNU Lesser General Public ++ * License along with mpv. If not, see . ++ */ ++ ++// Hands TrueHD and DTS-HD passthrough to the host application, which renders ++// it itself (see include/mpv/spatial_audio.h), and keeps mpv's audio clock on ++// the AVSampleBufferRenderSynchronizer the host renders into. ++// ++// A stream the host fails on falls back to PCM the way avfoundation's ++// compressed sink does: the format is latched and the output reloaded, the ++// reopen refuses it, and mpv decodes the track instead. A later PCM ++// initialization clears the latch, so the next track or file tries again. ++ ++#include ++ ++#include "ao.h" ++#include "audio/format.h" ++#include "common/common.h" ++#include "common/msg.h" ++#include "internal.h" ++#include "mpv/spatial_audio.h" ++#include "options/m_option.h" ++ ++#import ++#import ++ ++struct priv { ++ int64_t sink_address; ++ const mpv_spatial_audio_sink *sink; ++ AVSampleBufferRenderSynchronizer *synchronizer; ++ // Nudges the ao thread while the clock waits for the host, which the ++ // thread would otherwise only notice at its next refill. ++ dispatch_queue_t queue; ++ dispatch_source_t wakeup_timer; ++ atomic_bool waiting; ++ // The current timeline's start on the synchronizer clock, and the samples ++ // written since. ++ CMTime timeline; ++ int64_t written; ++ bool playing; ++ bool paused; ++ bool clock_running; ++ // The host has finished this timeline; the next write begins another. ++ bool ended; ++ bool reload_requested; ++}; ++ ++static _Atomic bool spatial_format_latch[2]; ++ ++static int spatial_latch_index(int format) ++{ ++ return format == AF_FORMAT_S_DTSHD; ++} ++ ++static int64_t ns_from_time(CMTime time) ++{ ++ return CMTimeConvertScale(time, 1000000000, kCMTimeRoundingMethod_Default).value; ++} ++ ++// Arms the timer that nudges the ao thread only while the clock waits. ++static void set_waiting(struct ao *ao, bool waiting) ++{ ++ struct priv *p = ao->priv; ++ ++ atomic_store(&p->waiting, waiting); ++ dispatch_source_set_timer(p->wakeup_timer, ++ waiting ? dispatch_time(DISPATCH_TIME_NOW, 10 * NSEC_PER_MSEC) : DISPATCH_TIME_FOREVER, ++ 10 * NSEC_PER_MSEC, NSEC_PER_MSEC); ++} ++ ++// Audio written but not yet played. While the clock waits, that is all of it, ++// so mpv's audio position, and with it the video, waits too. ++static int64_t queued_ns(struct ao *ao) ++{ ++ struct priv *p = ao->priv; ++ ++ // Split so that days of 192 kHz audio stay within int64. ++ int64_t written = p->written / ao->samplerate * INT64_C(1000000000) + ++ p->written % ao->samplerate * INT64_C(1000000000) / ao->samplerate; ++ int64_t now = ns_from_time(CMTimebaseGetTime(p->synchronizer.timebase)); ++ return MPMAX(ns_from_time(p->timeline) + written - now, 0); ++} ++ ++static void begin_timeline(struct ao *ao) ++{ ++ struct priv *p = ao->priv; ++ ++ [p->synchronizer setRate:0]; ++ p->timeline = CMTimebaseGetTime(p->synchronizer.timebase); ++ p->written = 0; ++ p->playing = false; ++ p->paused = false; ++ p->clock_running = false; ++ p->ended = false; ++ set_waiting(ao, false); ++ p->sink->begin(p->sink->ctx, ns_from_time(p->timeline)); ++} ++ ++static int init(struct ao *ao) ++{ ++ struct priv *p = ao->priv; ++ ++ p->sink = (const mpv_spatial_audio_sink *)(intptr_t)p->sink_address; ++ if (!p->sink || p->sink->version != MPV_SPATIAL_AUDIO_SINK_VERSION) { ++ MP_VERBOSE(ao, "no host sink (--ao-spatial-sink)\n"); ++ return -1; ++ } ++ ++ mpv_spatial_audio_codec codec; ++ if (ao->format == AF_FORMAT_S_TRUEHD) { ++ codec = MPV_SPATIAL_AUDIO_CODEC_TRUEHD; ++ } else if (ao->format == AF_FORMAT_S_DTSHD) { ++ codec = MPV_SPATIAL_AUDIO_CODEC_DTSHD; ++ } else { ++ if (!af_fmt_is_spdif(ao->format)) { ++ atomic_store(&spatial_format_latch[0], false); ++ atomic_store(&spatial_format_latch[1], false); ++ } ++ MP_VERBOSE(ao, "only TrueHD and DTS-HD passthrough are handed to the host\n"); ++ return -1; ++ } ++ if (atomic_load(&spatial_format_latch[spatial_latch_index(ao->format)])) { ++ MP_INFO(ao, "%s failed in the host; decoding it instead\n", ++ af_fmt_to_str(ao->format)); ++ return -1; ++ } ++ ++ p->synchronizer = [[AVSampleBufferRenderSynchronizer alloc] init]; ++ if (!p->sink->open(p->sink->ctx, codec, (void *)p->synchronizer)) { ++ MP_VERBOSE(ao, "the host refused the stream\n"); ++ [p->synchronizer release]; ++ p->synchronizer = nil; ++ return -1; ++ } ++ ++ p->sink->set_volume(p->sink->ctx, atomic_load(&ao->gain)); ++ ++ // The host's encoder holds 1.5 s before its first output, so let mpv ++ // queue well beyond that. ++ ao->device_buffer = ao->samplerate * 3; ++ ++ p->queue = dispatch_queue_create("mpv spatial audio", DISPATCH_QUEUE_SERIAL); ++ p->wakeup_timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, p->queue); ++ dispatch_source_set_timer(p->wakeup_timer, DISPATCH_TIME_FOREVER, 0, 0); ++ dispatch_source_set_event_handler(p->wakeup_timer, ^{ ++ if (atomic_load(&p->waiting)) ++ ao_wakeup(ao); ++ }); ++ dispatch_resume(p->wakeup_timer); ++ ++ begin_timeline(ao); ++ return 0; ++} ++ ++static void uninit(struct ao *ao) ++{ ++ struct priv *p = ao->priv; ++ ++ // Not set when init() failed. ++ if (!p->queue) ++ return; ++ dispatch_source_cancel(p->wakeup_timer); ++ // A handler already running finishes before this returns. ++ dispatch_sync(p->queue, ^{}); ++ dispatch_release(p->wakeup_timer); ++ dispatch_release(p->queue); ++ ++ [p->synchronizer setRate:0]; ++ p->sink->close(p->sink->ctx); ++ [p->synchronizer release]; ++} ++ ++static void reset(struct ao *ao) ++{ ++ begin_timeline(ao); ++} ++ ++static void start(struct ao *ao) ++{ ++ struct priv *p = ao->priv; ++ ++ p->playing = true; ++ set_waiting(ao, !p->paused && !p->clock_running); ++} ++ ++static bool set_pause(struct ao *ao, bool paused) ++{ ++ struct priv *p = ao->priv; ++ ++ p->paused = paused; ++ if (p->clock_running) ++ [p->synchronizer setRate:paused ? 0 : 1]; ++ set_waiting(ao, p->playing && !paused && !p->clock_running); ++ return true; ++} ++ ++static bool audio_write(struct ao *ao, void **data, int samples) ++{ ++ struct priv *p = ao->priv; ++ ++ // After an underrun the clock has run past everything written, so ++ // continuing that timeline would place new audio in the past. After the ++ // end (gapless playback goes on to the next file without a reset), the ++ // host takes no more audio on it. Either way a new timeline begins, and ++ // playback stays active: mpv calls start() only if it saw the underrun. ++ if (p->ended || (p->clock_running && queued_ns(ao) == 0)) { ++ MP_VERBOSE(ao, "beginning a new timeline\n"); ++ bool playing = p->playing; ++ begin_timeline(ao); ++ p->playing = playing; ++ set_waiting(ao, playing); ++ } ++ p->sink->write(p->sink->ctx, data[0], (size_t)samples * ao->sstride); ++ p->written += samples; ++ return true; ++} ++ ++// The host holds back the start of a stream, and its encoder the latest ++// audio, until it has more; at the end it has to let both go. ++static void end_of_stream(struct ao *ao) ++{ ++ struct priv *p = ao->priv; ++ ++ p->ended = true; ++ p->sink->end(p->sink->ctx); ++} ++ ++// mpv's software volume never reaches a bitstream, so the host applies it. ++// mpv calls this whenever it changes, paused or not. ++static void set_gain(struct ao *ao, float gain) ++{ ++ struct priv *p = ao->priv; ++ ++ // ao_alloc() sets the gain before init() has opened the host. ++ if (!p->queue) ++ return; ++ p->sink->set_volume(p->sink->ctx, gain); ++} ++ ++static void get_state(struct ao *ao, struct mp_pcm_state *state) ++{ ++ struct priv *p = ao->priv; ++ ++ if (p->sink->has_failed(p->sink->ctx) && !p->reload_requested) { ++ MP_WARN(ao, "the host failed to render the stream, falling back to PCM\n"); ++ p->reload_requested = true; ++ atomic_store(&spatial_format_latch[spatial_latch_index(ao->format)], true); ++ ao_request_reload(ao); ++ } ++ ++ if (p->playing && !p->paused && !p->clock_running && ++ p->sink->is_ready(p->sink->ctx)) ++ { ++ [p->synchronizer setRate:1 time:p->timeline]; ++ p->clock_running = true; ++ set_waiting(ao, false); ++ } ++ ++ int64_t queued = queued_ns(ao); ++ ++ // After the end mpv writes nothing until this timeline has played out, and ++ // its thread would sleep a quarter of the buffer before noticing, holding ++ // up EOF and a gapless next file. Paused, nothing plays out. ++ bool draining = p->ended && !p->paused && queued > 0; ++ if (p->ended && atomic_load(&p->waiting) != draining) ++ set_waiting(ao, draining); ++ ++ state->queued_samples = queued * ao->samplerate / INT64_C(1000000000); ++ int period = ao->samplerate / 100; ++ state->free_samples = MPMAX(ao->device_buffer - state->queued_samples, 0) ++ / period * period; ++ // The next file's audio waits until this one has played out. ++ if (p->ended && queued > 0) ++ state->free_samples = 0; ++ state->delay = queued / 1e9; ++ state->playing = p->playing && queued > 0; ++} ++ ++#define OPT_BASE_STRUCT struct priv ++ ++const struct ao_driver audio_out_spatial = { ++ .description = "TrueHD and DTS-HD passthrough to the host application", ++ .name = "spatial", ++ .init = init, ++ .uninit = uninit, ++ .reset = reset, ++ .start = start, ++ .set_pause = set_pause, ++ .write = audio_write, ++ .get_state = get_state, ++ .end_of_stream = end_of_stream, ++ .set_gain = set_gain, ++ .priv_size = sizeof(struct priv), ++ .options = (const struct m_option[]) { ++ {"sink", OPT_INT64(sink_address)}, ++ {0} ++ }, ++ .options_prefix = "ao-spatial", ++}; +diff --git a/audio/out/buffer.c b/audio/out/buffer.c +index 277b524..a7e38f9 100644 +--- a/audio/out/buffer.c ++++ b/audio/out/buffer.c +@@ -685,8 +685,11 @@ static bool ao_play_data(struct ao *ao) + MP_TRACE(ao, "in=%d space=%d(%d) pl=%d, eof=%d\n", + samples, space, state.free_samples, p->playing, got_eof); + +- if (got_eof) ++ if (got_eof) { ++ if (ao->driver->end_of_stream) ++ ao->driver->end_of_stream(ao); + goto eof; ++ } + + return samples > 0 && (samples < space || ao->untimed); + +diff --git a/audio/out/internal.h b/audio/out/internal.h +index 85fa777..7b64ea3 100644 +--- a/audio/out/internal.h ++++ b/audio/out/internal.h +@@ -173,6 +173,13 @@ struct ao_driver { + bool (*write)(struct ao *ao, void **data, int samples); + // push based: return mandatory stream information + void (*get_state)(struct ao *ao, struct mp_pcm_state *state); ++ // push based, optional: the audio has ended, and nothing more is written ++ // until reset(). For drivers that hold audio back until they get more. ++ void (*end_of_stream)(struct ao *ao); ++ ++ // optional: mpv's software volume changed (ao_set_gain()). For drivers ++ // whose output it never reaches, such as a bitstream the host renders. ++ void (*set_gain)(struct ao *ao, float gain); + + // Return the list of devices currently available in the system. Use + // ao_device_list_add() to add entries. The selected device will be set as +diff --git a/include/mpv/spatial_audio.h b/include/mpv/spatial_audio.h +new file mode 100644 +index 0000000..6733096 +--- /dev/null ++++ b/include/mpv/spatial_audio.h +@@ -0,0 +1,105 @@ ++/* Copyright (C) 2026 the mpv developers ++ * ++ * Permission to use, copy, modify, and/or distribute this software for any ++ * purpose with or without fee is hereby granted, provided that the above ++ * copyright notice and this permission notice appear in all copies. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES ++ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF ++ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ++ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES ++ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ++ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF ++ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ++ */ ++ ++#ifndef MPV_CLIENT_API_SPATIAL_AUDIO_H_ ++#define MPV_CLIENT_API_SPATIAL_AUDIO_H_ ++ ++#include ++#include ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/** ++ * The "spatial" audio output hands TrueHD and DTS-HD passthrough to the host ++ * application, which renders it itself, for example as Apple spatial audio ++ * through an AVSampleBufferAudioRenderer. Apple platforms only. ++ * ++ * Use it with --audio-spdif=truehd,dts-hd and --ao=spatial, (or ++ * --ao=spatial, to autoprobe the rest), and ++ * pass the address of a filled-in mpv_spatial_audio_sink, which must outlive ++ * the player, as the int64 option --ao-spatial-sink. The output refuses every ++ * other format, so PCM goes to the next output in the list. ++ * ++ * Load each file with loadfile replace: gapless handoff across files with a ++ * queued next file is not supported. ++ * ++ * mpv keeps its clock on an AVSampleBufferRenderSynchronizer that the output ++ * owns and the host renders into. It holds the clock until is_ready() returns ++ * true, then runs it from the timeline's start, so video waits for the host. ++ */ ++ ++#define MPV_SPATIAL_AUDIO_SINK_VERSION 2 ++ ++typedef enum mpv_spatial_audio_codec { ++ MPV_SPATIAL_AUDIO_CODEC_TRUEHD = 1, ++ MPV_SPATIAL_AUDIO_CODEC_DTSHD = 2, ++} mpv_spatial_audio_codec; ++ ++/** ++ * Callbacks from the output. They are never called concurrently, except ++ * set_volume, which mpv calls from its core thread. Times are on the ++ * synchronizer's clock, in nanoseconds. ++ */ ++typedef struct mpv_spatial_audio_sink { ++ /** MPV_SPATIAL_AUDIO_SINK_VERSION. */ ++ int version; ++ void *ctx; ++ /** ++ * Prepares to render `codec` into `synchronizer`, an ++ * AVSampleBufferRenderSynchronizer that stays valid until close(). Return ++ * false to refuse; mpv then decodes the track to PCM. ++ */ ++ bool (*open)(void *ctx, mpv_spatial_audio_codec codec, void *synchronizer); ++ /** Starts a new timeline at `time`, dropping everything queued. */ ++ void (*begin)(void *ctx, int64_t time); ++ /** ++ * IEC 61937 bursts exactly as mpv's passthrough produces them, split ++ * anywhere, continuing the timeline from begin(). ++ */ ++ void (*write)(void *ctx, const void *data, size_t size); ++ /** Whether enough audio is queued to start the clock. */ ++ bool (*is_ready)(void *ctx); ++ /** ++ * Whether the host failed, for example on a stream it cannot render. The ++ * output then reopens without the host for that codec, so mpv decodes the ++ * track to PCM. Once that PCM output has opened, the next track or audio ++ * reinitialization offers the codec to the host again. ++ */ ++ bool (*has_failed)(void *ctx); ++ /** Releases what open() prepared. */ ++ void (*close)(void *ctx); ++ /** ++ * Nothing more is written until the next begin(): render everything ++ * written, including what a startup buffer holds back, and become ready. ++ */ ++ void (*end)(void *ctx); ++ /** ++ * mpv's software volume as a linear gain: --volume, --volume-gain, ++ * replaygain and --mute together, 0 when muted. mpv cannot scale a ++ * bitstream, so the host applies it to what it renders. Called after ++ * open() and whenever it changes; above 1 is amplification the host may ++ * clamp. ++ */ ++ void (*set_volume)(void *ctx, float volume); ++} mpv_spatial_audio_sink; ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/meson.build b/meson.build +index 56cb290..c2d7a81 100644 +--- a/meson.build ++++ b/meson.build +@@ -821,6 +821,7 @@ features += {'avfoundation': avfoundation.found()} + if features['avfoundation'] + dependencies += avfoundation + sources += files('audio/out/ao_avfoundation.m') ++ sources += files('audio/out/ao_spatial.m') + endif + + vo_avfoundation = dependency('appleframeworks', modules: ['AVFoundation', 'CoreMedia', 'CoreVideo'], +@@ -1815,7 +1816,8 @@ if get_option('libmpv') + description: 'mpv media player client library') + + headers = ['include/mpv/client.h', 'include/mpv/render.h', +- 'include/mpv/render_gl.h', 'include/mpv/stream_cb.h'] ++ 'include/mpv/render_gl.h', 'include/mpv/stream_cb.h', ++ 'include/mpv/spatial_audio.h'] + install_headers(headers, subdir: 'mpv') + + # Allow projects to build with libmpv by cloning into ./subprojects/mpv diff --git a/scripts/spatial-audio-output-test.m b/scripts/spatial-audio-output-test.m new file mode 100644 index 00000000..fde9824f --- /dev/null +++ b/scripts/spatial-audio-output-test.m @@ -0,0 +1,310 @@ +// Unit test for the "spatial" audio output (patch 0007). It drives the driver +// the way mpv's push thread does, against a real synchronizer and a fake host +// sink. Run scripts/test-spatial-audio-output.sh, which builds it against a +// patched mpv checkout. + +#include +#include +#include +#include +#include +#include + +#include "audio/out/ao_spatial.m" + +#import + +// The parts of mpv the driver calls. +static atomic_int wakeups; +static int reloads; +void mp_msg(struct mp_log *log, int lev, const char *format, ...) {} +void ao_wakeup(struct ao *ao) { atomic_fetch_add(&wakeups, 1); } +void ao_request_reload(struct ao *ao) { reloads++; } +bool af_fmt_is_spdif(int format) { return format >= AF_FORMAT_S_AAC && format < AF_FORMAT_COUNT; } +const char *af_fmt_to_str(int format) { return "spdif"; } +const m_option_type_t m_option_type_int64; + +struct fake_sink { + mpv_spatial_audio_sink sink; + bool refuse, ready, failed; + mpv_spatial_audio_codec codec; + AVSampleBufferRenderSynchronizer *synchronizer; + int begins, closes, ends; + int volumes; + float volume; + int64_t begin_time; + NSMutableData *received; +}; + +static bool sink_open(void *ctx, mpv_spatial_audio_codec codec, void *synchronizer) +{ + struct fake_sink *s = ctx; + s->codec = codec; + s->synchronizer = synchronizer; + return !s->refuse; +} +static void sink_begin(void *ctx, int64_t time) +{ + struct fake_sink *s = ctx; + s->begins++; + s->begin_time = time; + [s->received setLength:0]; +} +static void sink_write(void *ctx, const void *data, size_t size) +{ + [((struct fake_sink *)ctx)->received appendBytes:data length:size]; +} +static bool sink_is_ready(void *ctx) { return ((struct fake_sink *)ctx)->ready; } +static bool sink_has_failed(void *ctx) { return ((struct fake_sink *)ctx)->failed; } +static void sink_close(void *ctx) { ((struct fake_sink *)ctx)->closes++; } +static void sink_set_volume(void *ctx, float volume) +{ + struct fake_sink *s = ctx; + s->volumes++; + s->volume = volume; +} +// The real host flushes its renderer here, then reports ready. +static void sink_end(void *ctx) +{ + struct fake_sink *s = ctx; + s->ends++; + s->ready = true; +} + +static int failures; +#define CHECK(cond, ...) do { \ + if (!(cond)) { failures++; printf("FAIL %s:%d: ", __FILE__, __LINE__); printf(__VA_ARGS__); printf("\n"); } \ +} while (0) + +static struct fake_sink *new_sink(void) +{ + struct fake_sink *s = calloc(1, sizeof(*s)); + s->sink = (mpv_spatial_audio_sink){ + .version = MPV_SPATIAL_AUDIO_SINK_VERSION, .ctx = s, + .open = sink_open, .begin = sink_begin, .write = sink_write, + .is_ready = sink_is_ready, .has_failed = sink_has_failed, .close = sink_close, + .end = sink_end, .set_volume = sink_set_volume, + }; + s->received = [[NSMutableData alloc] init]; + return s; +} + +// TrueHD passthrough as ad_spdif sets it up: 192 kHz, 8 channels of 16 bits. +static struct ao *new_ao(struct fake_sink *s, int format) +{ + struct ao *ao = calloc(1, sizeof(*ao)); + struct priv *p = calloc(1, sizeof(*p)); + p->sink_address = s ? (int64_t)(intptr_t)&s->sink : 0; + ao->priv = p; + ao->format = format; + ao->samplerate = 192000; + ao->sstride = 16; + // As ao_alloc() does: ao_set_gain() reaches the driver before init(). + atomic_store(&ao->gain, 1.0f); + audio_out_spatial.set_gain(ao, 1.0f); + return ao; +} + +static struct mp_pcm_state state_of(struct ao *ao) +{ + struct mp_pcm_state state = {0}; + get_state(ao, &state); + return state; +} + +// One MAT burst lasts 20 ms: 3840 samples at 192 kHz. +static void write_burst(struct ao *ao, uint8_t fill) +{ + uint8_t burst[3840 * 16]; + memset(burst, fill, sizeof(burst)); + void *planes[] = {burst}; + audio_write(ao, planes, 3840); +} + +int main(void) +{ + @autoreleasepool { + // Without a sink, or for PCM, the output refuses, so mpv uses the next one. + struct ao *none = new_ao(NULL, AF_FORMAT_S_TRUEHD); + CHECK(init(none) < 0, "init without a sink must fail"); + + struct fake_sink *s = new_sink(); + // ao_alloc() sets the gain before init(); nothing is open to tell. + struct ao *unopened = new_ao(s, AF_FORMAT_S_TRUEHD); + audio_out_spatial.set_gain(unopened, 1.0f); + CHECK(s->volumes == 0, "a gain set before init() reaches no host"); + struct ao *pcm = new_ao(s, AF_FORMAT_S16); + CHECK(init(pcm) < 0, "init for PCM must fail"); + CHECK(s->codec == 0, "open must not be called for PCM"); + + s->refuse = true; + struct ao *refused = new_ao(s, AF_FORMAT_S_TRUEHD); + CHECK(init(refused) < 0, "init must fail when the host refuses"); + CHECK(s->codec == MPV_SPATIAL_AUDIO_CODEC_TRUEHD, "open gets the codec"); + CHECK(s->volumes == 0, "a refused stream gets no volume"); + s->refuse = false; + + struct ao *dts = new_ao(s, AF_FORMAT_S_DTSHD); + CHECK(init(dts) == 0 && s->codec == MPV_SPATIAL_AUDIO_CODEC_DTSHD, "DTS-HD opens as DTS-HD"); + uninit(dts); + + // TrueHD: opened with the synchronizer the clock runs on, at rate 0. + struct ao *ao = new_ao(s, AF_FORMAT_S_TRUEHD); + struct priv *p = ao->priv; + s->begins = 0; + s->volumes = 0; + CHECK(init(ao) == 0, "init for TrueHD succeeds"); + CHECK(s->synchronizer == p->synchronizer && s->synchronizer != nil, "open gets the synchronizer"); + CHECK(s->begins == 1, "init begins a timeline"); + CHECK(ao->device_buffer == 192000 * 3, "mpv may queue 3 s"); + CHECK(p->synchronizer.rate == 0, "the clock waits at rate 0"); + + // mpv cannot scale a bitstream, so its volume and mute go to the host + // as mpv sets them (ao_set_gain() calls the driver), paused or not. + CHECK(s->volumes == 1 && s->volume == 1.0f, "open hands the host mpv's volume"); + set_pause(ao, true); + audio_out_spatial.set_gain(ao, 0.0f); + CHECK(s->volumes == 2 && s->volume == 0.0f, "a change reaches the host at once, paused too, mute as 0"); + state_of(ao); + CHECK(s->volumes == 2, "reading the state sends nothing more"); + set_pause(ao, false); + audio_out_spatial.set_gain(ao, 1.0f); + + // Bursts reach the host unchanged, before and after start(). + write_burst(ao, 0x5a); + start(ao); + for (int n = 0; n < 9; n++) + write_burst(ao, (uint8_t)n); + CHECK(s->received.length == 10 * 3840 * 16, "every byte reaches the host (%lu)", (unsigned long)s->received.length); + CHECK(((const uint8_t *)s->received.bytes)[0] == 0x5a, "bytes arrive unchanged"); + + // While the host is not ready, the clock holds and everything counts as queued. + struct mp_pcm_state st = state_of(ao); + CHECK(st.playing, "reports playing while the clock waits, or mpv sees an underrun"); + CHECK(fabs(st.delay - 0.2) < 0.001, "delay is all written audio while waiting (%f)", st.delay); + CHECK(st.free_samples == 192000 * 3 - 38400, "free space is the rest (%d)", st.free_samples); + int before = atomic_load(&wakeups); + usleep(60 * 1000); + CHECK(fabs(state_of(ao).delay - 0.2) < 0.001, "the clock does not move while waiting"); + CHECK(atomic_load(&wakeups) > before, "the ao thread is nudged while waiting"); + + // Once ready, the clock runs from the timeline's start. Sleeps on a + // busy machine overshoot, so the delay is checked against the time + // that actually passed. + s->ready = true; + st = state_of(ao); + CFTimeInterval started = CACurrentMediaTime(); + CHECK(p->synchronizer.rate == 1, "the clock starts when the host is ready"); + usleep(50 * 1000); + double delay = state_of(ao).delay; + double expected = MPMAX(0.2 - (CACurrentMediaTime() - started), 0); + CHECK(fabs(delay - expected) < 0.02, "delay falls in real time (%f, expected %f)", delay, expected); + usleep(15 * 1000); // lets a timer tick already in flight land + before = atomic_load(&wakeups); + usleep(40 * 1000); + CHECK(atomic_load(&wakeups) == before, "no nudging once the clock runs"); + + // Pause stops the clock, resume restarts it. + set_pause(ao, true); + CHECK(p->synchronizer.rate == 0, "pause stops the clock"); + delay = state_of(ao).delay; + usleep(50 * 1000); + CHECK(fabs(state_of(ao).delay - delay) < 0.002, "the clock holds while paused"); + set_pause(ao, false); + CHECK(p->synchronizer.rate == 1, "resume restarts the clock"); + + // Playing out everything ends the stream: mpv reads that as EOF. + usleep(150 * 1000); + st = state_of(ao); + CHECK(!st.playing && st.delay == 0, "all audio played ends playback (%f)", st.delay); + + // Audio after an underrun would land in the past on the old timeline, + // so it starts a new one, which waits for the host again. + int begins = s->begins; + s->ready = false; + write_burst(ao, 2); + CHECK(s->begins == begins + 1, "a write after an underrun begins a new timeline"); + CHECK(!p->clock_running && p->synchronizer.rate == 0, "the new timeline waits for the host"); + // mpv may not have seen the underrun, and then calls no start(). + CHECK(p->playing && atomic_load(&p->waiting), "playback stays active across the new timeline"); + CHECK(fabs(state_of(ao).delay - 0.02) < 0.001, "only the new audio is queued"); + + // A seek begins a new timeline at the current time and waits for the host again. + begins = s->begins; + reset(ao); + CHECK(s->begins == begins + 1 && s->received.length == 0, "reset begins a new timeline"); + CHECK(p->synchronizer.rate == 0 && !p->clock_running, "reset stops the clock"); + CHECK(s->begin_time == ns_from_time(CMTimebaseGetTime(p->synchronizer.timebase)), "the timeline starts at the clock's time"); + for (int i = 0; i < 3; i++) + write_burst(ao, 1); + start(ao); + CHECK(fabs(state_of(ao).delay - 0.06) < 0.001, "delay restarts from the new timeline"); + + // At the end the host lets go of what it holds back, so audio shorter + // than its startup buffer still plays. + end_of_stream(ao); + CHECK(s->ends == 1, "the host hears the end"); + state_of(ao); + CHECK(p->clock_running && p->synchronizer.rate == 1, "the clock starts once the host has flushed"); + CHECK(state_of(ao).free_samples == 0, "the next file waits until this one has played out"); + set_pause(ao, true); + state_of(ao); + CHECK(!atomic_load(&p->waiting), "no nudging while paused at the end, as nothing plays out"); + set_pause(ao, false); + CHECK(state_of(ao).delay > 0 && atomic_load(&p->waiting), "nudging resumes with playback"); + int nudged = atomic_load(&wakeups); + usleep(100 * 1000); + CHECK(atomic_load(&wakeups) > nudged, "the ao thread is nudged until the ended timeline has played out"); + state_of(ao); + CHECK(!atomic_load(&p->waiting), "and no longer once it has"); + begins = s->begins; + write_burst(ao, 3); + CHECK(s->begins == begins + 1 && !p->ended, "the next file begins a new timeline"); + CHECK(p->playing, "and keeps playing, as mpv sends no start() between gapless files"); + + // A long run stays representable: 14 h at 192 kHz overflows written * 1e9. + reset(ao); + CMTime now = CMTimebaseGetTime(p->synchronizer.timebase); + p->timeline = CMTimeSubtract(now, CMTimeMake(14 * 3600, 1)); + p->written = INT64_C(192000) * 14 * 3600 + 19200; // and 0.1 s still to play + CHECK(fabs(state_of(ao).delay - 0.1) < 0.001, "a 14 h timeline keeps its delay (%f)", state_of(ao).delay); + + // A host failure asks mpv to reopen the output, once. + s->failed = true; + state_of(ao); + state_of(ao); + CHECK(reloads == 1, "a failure requests one reload (%d)", reloads); + + uninit(ao); + CHECK(s->closes == 2, "uninit closes the sink (%d)", s->closes); + before = atomic_load(&wakeups); + usleep(30 * 1000); + CHECK(atomic_load(&wakeups) == before, "no wakeups after uninit"); + + // The reopen refuses the failed format without asking the host, so + // mpv falls back to decoding it. + s->failed = false; + s->codec = 0; + struct ao *reopened = new_ao(s, AF_FORMAT_S_TRUEHD); + CHECK(init(reopened) < 0, "the reopen after a failure refuses TrueHD"); + CHECK(s->codec == 0, "and does not offer it to the host"); + struct ao *other = new_ao(s, AF_FORMAT_S_DTSHD); + CHECK(init(other) == 0, "DTS-HD is not latched by a TrueHD failure"); + uninit(other); + struct ao *ac3 = new_ao(s, AF_FORMAT_S_AC3); + CHECK(init(ac3) < 0, "AC-3 goes to the next output"); + s->codec = 0; + CHECK(init(new_ao(s, AF_FORMAT_S_TRUEHD)) < 0, "another passthrough format leaves the latch"); + CHECK(s->codec == 0, "and the host still is not asked"); + + // Once the decoded track opens a PCM output, the next track tries the + // host again. + CHECK(init(new_ao(s, AF_FORMAT_FLOAT)) < 0, "PCM goes to the next output"); + struct ao *retried = new_ao(s, AF_FORMAT_S_TRUEHD); + CHECK(init(retried) == 0 && s->codec == MPV_SPATIAL_AUDIO_CODEC_TRUEHD, + "a PCM initialization clears the latch"); + uninit(retried); + } + printf(failures ? "%d FAILED\n" : "all passed\n", failures); + return failures ? 1 : 0; +} diff --git a/scripts/test-spatial-audio-output.sh b/scripts/test-spatial-audio-output.sh new file mode 100755 index 00000000..ca12f4be --- /dev/null +++ b/scripts/test-spatial-audio-output.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Builds and runs the "spatial" audio output's unit test (patch 0007) against +# the pinned mpv with this repository's patches applied. macOS only. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VERSION=$(sed -n '/case .libmpv:/{n;s/.*"\(.*\)".*/\1/p;q;}' "$ROOT/Sources/BuildScripts/XCFrameworkBuild/main.swift") +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +git -c advice.detachedHead=false clone -q --depth 1 --branch "$VERSION" \ + https://github.com/mpv-player/mpv.git "$WORK/mpv" +for patch in "$ROOT"/Sources/BuildScripts/patch/libmpv/*.patch; do + git -C "$WORK/mpv" apply "$patch" +done + +# The driver reads one build switch; the test supplies the mpv functions it calls. +printf '#pragma once\n#define HAVE_AVFOUNDATION 1\n' > "$WORK/config.h" +xcrun --sdk macosx clang -x objective-c -std=c11 -Wall -Wno-unused-parameter \ + -I"$WORK" -I"$WORK/mpv" -I"$WORK/mpv/include" \ + "$ROOT/scripts/spatial-audio-output-test.m" \ + -framework AVFoundation -framework CoreMedia -framework Foundation -framework QuartzCore \ + -o "$WORK/spatial-audio-output-test" +"$WORK/spatial-audio-output-test"