From 9e37fe291f462e083116f031781b246a541e5081 Mon Sep 17 00:00:00 2001 From: Chrizz Gelbach Date: Tue, 3 Nov 2026 15:11:07 +0100 Subject: [PATCH] AudioPlayer: add sample buffer renderer output for AirPlay 2 multi-room New output mode `.sampleBufferRenderer(maxBufferedSeconds:)`: instead of an output audio unit, a timer pulls the engine's manually rendered audio and enqueues it into an AVSampleBufferAudioRenderer driven by an AVSampleBufferRenderSynchronizer. On Mac Catalyst only this output follows the AirPlay devices picked in the system route picker, including several rooms at once (with the long-form audio route sharing policy). The audio is enqueued ahead of what is heard, so progress, duration and the item change callbacks are compensated by the unheard audio content: the previous item is reported until its end is heard, and the callbacks are delivered in order once the change is audible. Seek and item changes flush the renderer, the end of the queue drains, automatic flushes re-enqueue from a history, volume and mute act on the renderer directly. The default output mode stays `.audioUnit`. --- .../Streaming/AudioPlayer/AudioPlayer.swift | 159 +++++- .../AudioPlayerConfiguration.swift | 23 +- .../AudioPlayerRenderProcessor.swift | 22 + .../SampleBufferRendererOutput.swift | 510 ++++++++++++++++++ 4 files changed, 687 insertions(+), 27 deletions(-) create mode 100644 AudioStreaming/Streaming/AudioPlayer/SampleBufferRendererOutput.swift diff --git a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift index e08ac94..f2f7ee8 100644 --- a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift +++ b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift @@ -11,8 +11,16 @@ open class AudioPlayer { public weak var delegate: AudioPlayerDelegate? public var muted: Bool { - get { playerContext.muted.value } - set { playerContext.muted.write { $0 = newValue } } + get { rendererOutput?.isMuted ?? playerContext.muted.value } + set { + if let rendererOutput { + // the renderer output plays ahead of the engine: mute only the renderer, so it applies + // at once and no silenced audio stays enqueued after unmuting + rendererOutput.isMuted = newValue + } else { + playerContext.muted.write { $0 = newValue } + } + } } /// The volume of the audio @@ -20,8 +28,16 @@ open class AudioPlayer { /// Defaults to 1.0. Valid ranges are 0.0 to 1.0 /// The value is restricted from 0.0 to 1.0 public var volume: Float { - get { audioEngine.mainMixerNode.outputVolume } - set { audioEngine.mainMixerNode.outputVolume = min(1.0, max(0.0, newValue)) } + get { rendererOutput?.volume ?? audioEngine.mainMixerNode.outputVolume } + set { + let volume = min(1.0, max(0.0, newValue)) + if let rendererOutput { + // the renderer output plays ahead of the engine, so set the volume on the renderer + rendererOutput.volume = volume + } else { + audioEngine.mainMixerNode.outputVolume = volume + } + } } /// The playback rate of the player. @@ -55,13 +71,9 @@ open class AudioPlayer { /// - Returns: A `Double` value indicating the total duration. public var duration: Double { guard playerContext.internalState != .pendingNext else { return 0 } - playerContext.entriesLock.lock() - let playingEntry = playerContext.audioPlayingEntry - playerContext.entriesLock.unlock() - guard let entry = playingEntry else { return 0 } + guard let (entry, progress) = audiblePosition() else { return 0 } let entryDuration = entry.duration() - let progress = self.progress if entryDuration < progress, entryDuration > 0 { return progress } @@ -71,18 +83,36 @@ open class AudioPlayer { /// The progress of the audio playback, in seconds. public var progress: Double { guard playerContext.internalState != .pendingNext else { return 0 } - playerContext.entriesLock.lock() - let playingEntry = playerContext.audioPlayingEntry - playerContext.entriesLock.unlock() - guard let entry = playingEntry else { return 0 } + return audiblePosition()?.progress ?? 0 + } + + /// The entry that is heard right now and its progress. + /// + /// With the renderer output, audio is rendered ahead of time: the playing entry can already be + /// the next item while the end of the previous one is still heard (or the queue drains). + private func audiblePosition() -> (entry: AudioEntry, progress: Double)? { + let (playingEntry, previousEntry) = playerContext.entriesLock.withLock { + (playerContext.audioPlayingEntry, previousPlayingEntry) + } + let unheard = rendererOutput?.unheardContentSeconds ?? 0 + guard let entry = playingEntry else { + if let previousEntry, unheard > 0 { + return (previousEntry, max(0, previousEntry.progress - unheard)) + } + return nil + } entry.seekRequest.lock.lock() let seekRequested = entry.seekRequest.requested let seekTime = entry.seekRequest.time entry.seekRequest.lock.unlock() if seekRequested { - return seekTime + return (entry, seekTime) } - return entry.progress + let renderedOfEntry = Double(entry.framesPlayed) / outputAudioFormat.sampleRate + if unheard > renderedOfEntry, let previousEntry { + return (previousEntry, max(0, previousEntry.progress - (unheard - renderedOfEntry))) + } + return (entry, max(0, entry.progress - unheard)) } /// The number of audio frames that have been played @@ -92,7 +122,8 @@ open class AudioPlayer { let playingEntry = playerContext.audioPlayingEntry playerContext.entriesLock.unlock() guard let entry = playingEntry else { return 0 } - return entry.framesPlayed + let unheardFrames = Int((rendererOutput?.unheardContentSeconds ?? 0) * outputAudioFormat.sampleRate) + return max(0, entry.framesPlayed - unheardFrames) } /// Indicates whether seeking is supported for the currently playing audio @@ -167,6 +198,13 @@ open class AudioPlayer { private let fileStreamProcessor: AudioFileStreamProcessor private let playerRenderProcessor: AudioPlayerRenderProcessor + /// Set when `configuration.outputMode` is `.sampleBufferRenderer`, replaces the output audio unit `player` + private let rendererOutput: SampleBufferRendererOutput? + /// Delays item change callbacks by the renderer output latency + private let delegateNotifier = OrderedDelayedNotifier() + /// Renderer output only: the item before `audioPlayingEntry`, still heard for a moment after the + /// next one started rendering. Guarded by `playerContext.entriesLock`. + private var previousPlayingEntry: AudioEntry? private let frameFilterProcessor: FrameFilterProcessor private let serializationQueue: DispatchQueue @@ -213,6 +251,17 @@ open class AudioPlayer { outputAudioFormat: outputAudioFormat.basicStreamDescription ) + if case let .sampleBufferRenderer(maxBufferedSeconds) = self.configuration.outputMode { + rendererOutput = SampleBufferRendererOutput( + format: outputAudioFormat, + maxBufferedSeconds: maxBufferedSeconds + ) { [playerRenderProcessor] frameCount, ioData in + playerRenderProcessor.renderOutput(inNumberFrames: frameCount, ioData: ioData) + } + } else { + rendererOutput = nil + } + frameFilterProcessor = FrameFilterProcessor( mixerNodeProvider: { engine.mainMixerNode @@ -227,7 +276,11 @@ open class AudioPlayer { deinit { // The IO thread keeps pulling from the render callback until the hardware is stopped, // so stop it before the buffers it renders into are freed. - player.auAudioUnit.stopHardware() + if let rendererOutput { + rendererOutput.shutdown() + } else { + player.auAudioUnit.stopHardware() + } audioEngine.stop() playerContext.audioPlayingEntry?.close() clearQueue() @@ -426,6 +479,7 @@ open class AudioPlayer { self.playerContext.entriesLock.lock() self.playerContext.audioReadingEntry = nil self.playerContext.audioPlayingEntry = nil + self.previousPlayingEntry = nil self.playerContext.entriesLock.unlock() self.processSource() @@ -434,6 +488,11 @@ open class AudioPlayer { /// Pauses the audio playback public func pause() { + if let rendererOutput, playerContext.internalState == .stopped { + // the end of the queue may still be playing out of the renderer output + rendererOutput.stop(drain: false) + return + } if playerContext.internalState != .paused, playerContext.internalState.contains(.running) { stateBeforePaused = playerContext.internalState playerContext.setInternalState(to: .paused) @@ -579,6 +638,8 @@ open class AudioPlayer { /// Creates and configures an `AVAudioUnit` with an output configuration /// and assigns it to the `player` variable. private func configPlayerNode() { + // the renderer output pulls from the engine itself, no output unit needed + guard rendererOutput == nil else { return } AVAudioUnit.createAudioUnit(with: UnitDescriptions.output) { [weak self] result in guard let self = self else { return } switch result { @@ -683,9 +744,16 @@ open class AudioPlayer { /// Pauses the audio engine and stops the player's hardware private func pauseEngine() { guard isEngineRunning else { return } + if let rendererOutput { + // no rendering while the engine is reset below + rendererOutput.pause() + rendererOutput.waitForRenderToFinish() + } audioEngine.reset() audioEngine.pause() - player.auAudioUnit.stopHardware() + if rendererOutput == nil { + player.auAudioUnit.stopHardware() + } Logger.debug("engine paused ⏸", category: .generic) } @@ -693,8 +761,16 @@ open class AudioPlayer { /// /// - parameter reason: A value of `AudioPlayerStopReason` indicating the reason the engine stopped. private func stopEngine(reason: AudioPlayerStopReason) { + if let rendererOutput { + // at the end of the queue the enqueued audio still has to be heard; + // no rendering while the engine is stopped below + rendererOutput.stop(drain: reason == .eof) + rendererOutput.waitForRenderToFinish() + } audioEngine.stop() - player.auAudioUnit.stopHardware() + if rendererOutput == nil { + player.auAudioUnit.stopHardware() + } rendererContext.resetBuffers() playerContext.setInternalState(to: .stopped) playerContext.stopReason.write { $0 = reason } @@ -706,12 +782,17 @@ open class AudioPlayer { /// - parameter resetBuffers: A `Bool` value indicating if the buffers should be reset, prior starting the player. private func startPlayer(resetBuffers: Bool) { if resetBuffers { + // the renderer output was already told to drop its audio (setCurrentReading) rendererContext.resetBuffers() } do { try startEngineIfNeeded() - try player.auAudioUnit.allocateRenderResources() - try player.auAudioUnit.startHardware() + if let rendererOutput { + rendererOutput.start() + } else { + try player.auAudioUnit.allocateRenderResources() + try player.auAudioUnit.startHardware() + } } catch { stopEngine(reason: .error) raiseUnexpected(error: .audioSystemError(.playerStartError)) @@ -793,6 +874,8 @@ open class AudioPlayer { assert(playerContext.audioReadingEntry === playerContext.audioPlayingEntry, "reading and playing entry must be the same") fileStreamProcessor.processSeek() + // the audio enqueued for the old position must not be heard anymore + discardRendererOutput() } private func setCurrentReading(entry: AudioEntry?, startPlaying: Bool, shouldClearQueue: Bool) { @@ -800,6 +883,9 @@ open class AudioPlayer { Logger.debug("Setting current reading entry to: \(entry.debugDescription)", category: .generic) if startPlaying { rendererContext.fillSilenceAudioBuffer() + // new item or seek: what is still enqueued must not be heard, and the item change + // callbacks below must not wait for it + discardRendererOutput() } fileStreamProcessor.closeFileStreamIfNeeded() @@ -858,6 +944,9 @@ open class AudioPlayer { } playerContext.entriesLock.lock() + if rendererOutput != nil, !isPlayingSameItemProbablySeek { + previousPlayingEntry = entry + } playerContext.audioPlayingEntry = nextEntry let playingQueueEntryId = playerContext.audioPlayingEntry?.id ?? AudioEntryId(id: "") playerContext.entriesLock.unlock() @@ -867,7 +956,7 @@ open class AudioPlayer { let progressInFrames = entry.progressInFrames() let progress = Double(progressInFrames) / outputAudioFormat.basicStreamDescription.mSampleRate let duration = entry.duration() - asyncOnMain { [weak self] in + notifyDelegateWhenHeard { [weak self] in guard let self else { return } self.delegate?.audioPlayerDidFinishPlaying( player: self, @@ -890,7 +979,7 @@ open class AudioPlayer { } } - asyncOnMain { [weak self] in + notifyDelegateWhenHeard { [weak self] in guard let self = self else { return } self.delegate?.audioPlayerDidStartPlaying(player: self, with: playingQueueEntryId) } @@ -910,6 +999,9 @@ open class AudioPlayer { } playerContext.entriesLock.lock() + if rendererOutput != nil, !isPlayingSameItemProbablySeek { + previousPlayingEntry = entry + } playerContext.audioPlayingEntry = nil playerContext.entriesLock.unlock() if let entry = entry, !isPlayingSameItemProbablySeek { @@ -921,7 +1013,7 @@ open class AudioPlayer { sourceQueue.async { [weak self] in guard let self else { return } self.processSource() - asyncOnMain { + self.notifyDelegateWhenHeard { self.delegate?.audioPlayerDidFinishPlaying( player: self, entryId: entryId, @@ -939,6 +1031,25 @@ open class AudioPlayer { checkRenderWaitingAndNotifyIfNeeded() } + /// Drops the audio enqueued in the renderer output (seek, new item). + private func discardRendererOutput() { + guard let rendererOutput else { return } + rendererOutput.flush() + playerContext.entriesLock.withLock { previousPlayingEntry = nil } + delegateNotifier.deliverAllPending() + } + + /// Calls `block` on the main queue once the audio rendered so far is heard. + /// + /// Without the renderer output this is immediate, as before. + private func notifyDelegateWhenHeard(_ block: @escaping () -> Void) { + guard rendererOutput != nil else { + asyncOnMain(block) + return + } + delegateNotifier.notify(after: rendererOutput?.bufferedSeconds ?? 0, block) + } + /// Clears pending queues and informs the delegate private func clearQueue() { let pendingItems = entriesQueue.pendingEntriesId() diff --git a/AudioStreaming/Streaming/AudioPlayer/AudioPlayerConfiguration.swift b/AudioStreaming/Streaming/AudioPlayer/AudioPlayerConfiguration.swift index 949a823..cf78df8 100644 --- a/AudioStreaming/Streaming/AudioPlayer/AudioPlayerConfiguration.swift +++ b/AudioStreaming/Streaming/AudioPlayer/AudioPlayerConfiguration.swift @@ -6,6 +6,15 @@ import Foundation import OSLog +/// Where the rendered audio is played. +public enum AudioPlayerOutputMode: Equatable, Sendable { + /// An output audio unit (RemoteIO / default output). Lowest latency. + case audioUnit + /// An `AVSampleBufferAudioRenderer`, which takes part in AirPlay 2 (multi-room playback). + /// Audio is enqueued up to `maxBufferedSeconds` ahead of what is heard. + case sampleBufferRenderer(maxBufferedSeconds: Double) +} + public struct AudioPlayerConfiguration: Equatable { /// All pending items will be flushed when seeking a track if this is set to `true` let flushQueueOnSeek: Bool @@ -23,12 +32,16 @@ public struct AudioPlayerConfiguration: Equatable { /// Enables the internal logs let enableLogs: Bool + /// Where the rendered audio is played + let outputMode: AudioPlayerOutputMode + public static let `default` = AudioPlayerConfiguration(flushQueueOnSeek: false, bufferSizeInSeconds: 10, secondsRequiredToStartPlaying: 1, gracePeriodAfterSeekInSeconds: 0.5, secondsRequiredToStartPlayingAfterBufferUnderrun: 7, - enableLogs: false) + enableLogs: false, + outputMode: .audioUnit) /// Initializes the configuration for the `AudioPlayer` /// /// Parameters are pre set for convenience @@ -39,13 +52,15 @@ public struct AudioPlayerConfiguration: Equatable { /// - parameter gracePeriodAfterSeekInSeconds: Number of seconds of audio required after seek occurs. /// - parameter secondsRequiredToStartPlayingAfterBufferUnderrun: Number of seconds of audio required to before playback resumes after a buffer underrun /// - parameter enableLogs: Enables the internal logs + /// - parameter outputMode: Where the rendered audio is played /// public init(flushQueueOnSeek: Bool = true, bufferSizeInSeconds: Double = 10, secondsRequiredToStartPlaying: Double = 1, gracePeriodAfterSeekInSeconds: Double = 0.5, secondsRequiredToStartPlayingAfterBufferUnderrun: Int = 1, - enableLogs: Bool = true) + enableLogs: Bool = true, + outputMode: AudioPlayerOutputMode = .audioUnit) { self.flushQueueOnSeek = flushQueueOnSeek self.bufferSizeInSeconds = bufferSizeInSeconds @@ -53,6 +68,7 @@ public struct AudioPlayerConfiguration: Equatable { self.gracePeriodAfterSeekInSeconds = gracePeriodAfterSeekInSeconds self.secondsRequiredToStartPlayingAfterBufferUnderrun = secondsRequiredToStartPlayingAfterBufferUnderrun self.enableLogs = enableLogs + self.outputMode = outputMode Logger.isEnabled = enableLogs } @@ -81,6 +97,7 @@ public struct AudioPlayerConfiguration: Equatable { secondsRequiredToStartPlaying: secondsRequiredToStartPlaying, gracePeriodAfterSeekInSeconds: gracePeriodAfterSeekInSeconds, secondsRequiredToStartPlayingAfterBufferUnderrun: secondsRequiredToStartPlayingAfterBufferUnderrun, - enableLogs: enableLogs) + enableLogs: enableLogs, + outputMode: outputMode) } } diff --git a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioPlayerRenderProcessor.swift b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioPlayerRenderProcessor.swift index 23f8ee4..d474c4b 100644 --- a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioPlayerRenderProcessor.swift +++ b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioPlayerRenderProcessor.swift @@ -17,6 +17,8 @@ final class AudioPlayerRenderProcessor: NSObject { private let playerContext: AudioPlayerContext private let rendererContext: AudioRendererContext + /// Frames copied from the decoded buffer (not silence padding), only touched on the render thread + private var copiedFramesCount: Int64 = 0 private let outputAudioFormat: AudioStreamBasicDescription init(playerContext: AudioPlayerContext, @@ -210,6 +212,8 @@ final class AudioPlayerRenderProcessor: NSObject { } } + copiedFramesCount += Int64(totalFramesCopied) + if totalFramesCopied < inNumberFrames { let delta = inNumberFrames - totalFramesCopied if let mData = bufferList.mBuffers.mData { @@ -344,6 +348,24 @@ final class AudioPlayerRenderProcessor: NSObject { return status } + /// Renders the engine output into `ioData` without an output audio unit + /// (used by `SampleBufferRendererOutput`). + /// + /// - returns: The number of frames rendered into `ioData`, and how many of them are audio + /// content (counted as played by the entries) rather than silence padding. + func renderOutput( + inNumberFrames: UInt32, + ioData: UnsafeMutablePointer + ) -> (rendered: UInt32, content: UInt32) { + let copiedBefore = copiedFramesCount + var flags = AudioUnitRenderActionFlags() + _ = render(inNumberFrames: inNumberFrames, ioData: ioData, flags: &flags) + let content = UInt32(clamping: copiedFramesCount - copiedBefore) + let bytesRendered = rendererContext.inOutAudioBufferList[0].mBuffers.mDataByteSize + guard outputAudioFormat.mBytesPerFrame > 0 else { return (0, content) } + return (min(inNumberFrames, bytesRendered / outputAudioFormat.mBytesPerFrame), content) + } + func renderProvider( flags: UnsafeMutablePointer, timeStamp _: UnsafePointer, diff --git a/AudioStreaming/Streaming/AudioPlayer/SampleBufferRendererOutput.swift b/AudioStreaming/Streaming/AudioPlayer/SampleBufferRendererOutput.swift new file mode 100644 index 0000000..9f3c8a6 --- /dev/null +++ b/AudioStreaming/Streaming/AudioPlayer/SampleBufferRendererOutput.swift @@ -0,0 +1,510 @@ +// +// SampleBufferRendererOutput.swift +// AudioStreaming +// + +import AVFoundation +import CoreMedia +import OSLog + +/// Plays the manually rendered output of the audio engine through an `AVSampleBufferAudioRenderer` +/// driven by an `AVSampleBufferRenderSynchronizer`, instead of an output audio unit. +/// +/// Audio played this way takes part in AirPlay 2, including multi-room playback chosen in the +/// system route picker (together with the `.longFormAudio` route sharing policy). An output audio +/// unit is not routed to AirPlay devices picked there on Mac Catalyst. +/// +/// The renderer is fed ahead of time, so everything rendered by the engine is heard +/// `bufferedSeconds` later. `AudioPlayer` compensates its progress and its delegate callbacks. +/// +/// Threading: all control methods are non-blocking. They record the request under a lock and +/// apply it on the internal queue, which is also the queue that renders. The render function may +/// call back into `AudioPlayer` (which synchronously uses its serialization queue), so this type +/// must never wait for the internal queue from the caller's thread. +final class SampleBufferRendererOutput { + /// Renders `frameCount` frames into `ioData`; returns the frames rendered and how many of them are + /// audio content (counted as played by `AudioPlayer`) rather than silence padding. + typealias RenderFunction = (_ frameCount: UInt32, _ ioData: UnsafeMutablePointer) + -> (rendered: UInt32, content: UInt32) + + private let renderer = AVSampleBufferAudioRenderer() + private let synchronizer = AVSampleBufferRenderSynchronizer() + private let queue = DispatchQueue(label: "streaming.renderer.output", qos: .userInitiated) + private let queueKey = DispatchSpecificKey() + /// Held while the engine renders, see `waitForRenderToFinish()` + private let renderLock = NSLock() + + private let render: RenderFunction + private let format: AVAudioFormat + private let formatDescription: CMAudioFormatDescription + private let sampleRate: Int32 + private let bytesPerFrame: UInt32 + private let framesPerBuffer: AVAudioFrameCount = 4096 + private let renderBuffer: AVAudioPCMBuffer + /// Upper bound of audio enqueued ahead of what is heard. + private let maxBufferedSeconds: Double + /// Audio enqueued before the synchronizer clock starts after a flush. + private let prerollSeconds: Double = 0.1 + /// Silence (no audio content yet, e.g. waiting for data) is enqueued only this far ahead, so the + /// audio that arrives later is not queued behind seconds of silence. + private let maxBufferedSilenceSeconds: Double = 0.2 + + /// Guards the requested state below, `enqueuedEndSeconds` and `contentTimeline`. + private let lock = NSLock() + private var wantsRunning = false + private var pendingFlush = false + private var pendingDrainStop = false + private var enqueuedEndSeconds: Double = 0 + /// Enqueued buffers not completely heard yet, with the content frames each one holds + private var contentTimeline = [(start: Double, end: Double, contentFrames: Int64)]() + private var isShutDown = false + + // Only accessed on `queue` + private var timer: DispatchSourceTimer? + private var nextFrame: Int64 = 0 + private var startFrame: Int64 = 0 + private var clockStarted = false + private var isRateRunning = false + private var drainGeneration = 0 + private var lastClockRestart = Date.distantPast + /// Enqueued buffers that might not have been heard yet, re-enqueued after an automatic flush. + private var history = [CMSampleBuffer]() + private var observers = [NSObjectProtocol]() + + init?(format: AVAudioFormat, maxBufferedSeconds: Double, render: @escaping RenderFunction) { + var description: CMAudioFormatDescription? + let status = CMAudioFormatDescriptionCreate( + allocator: kCFAllocatorDefault, + asbd: format.streamDescription, + layoutSize: 0, + layout: nil, + magicCookieSize: 0, + magicCookie: nil, + extensions: nil, + formatDescriptionOut: &description + ) + guard status == noErr, let description, + let renderBuffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: framesPerBuffer) + else { + Logger.audioRendering.error("renderer output: cannot create format description (\(status))") + return nil + } + self.format = format + self.formatDescription = description + self.renderBuffer = renderBuffer + self.sampleRate = Int32(format.sampleRate) + self.bytesPerFrame = format.streamDescription.pointee.mBytesPerFrame + self.maxBufferedSeconds = max(0.5, maxBufferedSeconds) + self.render = render + + queue.setSpecific(key: queueKey, value: ()) + synchronizer.addRenderer(renderer) + observeRenderer() + } + + deinit { + shutdown() + } + + // MARK: Public controls + + var volume: Float { + get { renderer.volume } + set { renderer.volume = newValue } + } + + var isMuted: Bool { + get { renderer.isMuted } + set { renderer.isMuted = newValue } + } + + /// Seconds of audio content that were rendered (and counted as played by `AudioPlayer`) + /// but not heard yet. Silence padding (waiting for data) is not included. + var unheardContentSeconds: Double { + let current = synchronizer.currentTime().seconds + guard current.isFinite else { return 0 } + let frames: Double = lock.withLock { + var frames: Double = 0 + for chunk in contentTimeline where chunk.end > current { + if chunk.start >= current { + frames += Double(chunk.contentFrames) + } else { + let unheardPart = (chunk.end - current) / (chunk.end - chunk.start) + frames += Double(chunk.contentFrames) * unheardPart + } + } + return frames + } + return frames / Double(sampleRate) + } + + /// Seconds until the audio enqueued so far is heard (including silence padding). + var bufferedSeconds: Double { + let enqueuedEnd = lock.withLock { enqueuedEndSeconds } + let current = synchronizer.currentTime().seconds + guard current.isFinite else { return 0 } + return max(0, enqueuedEnd - current) + } + + /// Starts or resumes rendering and playback. + func start() { + lock.withLock { + wantsRunning = true + pendingDrainStop = false + } + queue.async { [weak self] in self?.applyStart() } + } + + /// Pauses playback, keeping the enqueued audio for resuming. + func pause() { + lock.withLock { wantsRunning = false } + queue.async { [weak self] in self?.applyPause() } + } + + /// Drops all enqueued audio (seek, new item). The clock restarts with the next `start()`. + func flush() { + lock.withLock { + pendingFlush = true + enqueuedEndSeconds = 0 + contentTimeline.removeAll() + } + queue.async { [weak self] in self?.applyFlushIfNeeded() } + } + + /// Stops playback. With `drain`, the audio already enqueued plays out first (end of queue). + func stop(drain: Bool) { + if drain { + lock.withLock { + wantsRunning = false + pendingDrainStop = true + } + queue.async { [weak self] in self?.applyDrainStop() } + } else { + lock.withLock { wantsRunning = false } + flush() + queue.async { [weak self] in self?.applyPause() } + } + } + + /// Stops everything. Called on deinit of the player. + func shutdown() { + let alreadyShutDown = lock.withLock { + let value = isShutDown + isShutDown = true + wantsRunning = false + return value + } + guard !alreadyShutDown else { return } + let stopOnQueue = { [self] in + observers.forEach { NotificationCenter.default.removeObserver($0) } + observers.removeAll() + stopTimer() + synchronizer.setRate(0, time: .zero) + renderer.flush() + } + // wait for a render in progress: the player frees the render buffers right after this + if DispatchQueue.getSpecific(key: queueKey) != nil { + stopOnQueue() + } else { + queue.sync(execute: stopOnQueue) + } + } + + /// Waits until a render that is already running has finished, at most a short moment. + /// + /// Call it after `pause()`/`stop(drain:)`, which prevent new renders, before resetting or stopping + /// the engine. The wait is bounded: the render can call back into the player, which may be + /// waiting here, and must not deadlock. + func waitForRenderToFinish() { + if renderLock.lock(before: Date().addingTimeInterval(0.25)) { + renderLock.unlock() + } + } + + // MARK: Applying requests (on queue) + + private func applyStart() { + dispatchPrecondition(condition: .onQueue(queue)) + applyFlushIfNeeded() + guard lock.withLock({ wantsRunning && !isShutDown }) else { return } + drainGeneration += 1 + if clockStarted, !isRateRunning { + synchronizer.rate = 1 + isRateRunning = true + } + startTimerIfNeeded() + feed() + } + + private func applyPause() { + dispatchPrecondition(condition: .onQueue(queue)) + guard !lock.withLock({ wantsRunning }) else { return } + stopTimer() + if isRateRunning { + synchronizer.rate = 0 + isRateRunning = false + } + } + + private func applyFlushIfNeeded() { + dispatchPrecondition(condition: .onQueue(queue)) + let flush = lock.withLock { + let value = pendingFlush + pendingFlush = false + return value + } + guard flush else { return } + drainGeneration += 1 + synchronizer.setRate(0, time: .zero) + isRateRunning = false + clockStarted = false + renderer.flush() + history.removeAll() + nextFrame = 0 + startFrame = 0 + lock.withLock { + enqueuedEndSeconds = 0 + contentTimeline.removeAll() + } + } + + private func applyDrainStop() { + dispatchPrecondition(condition: .onQueue(queue)) + guard lock.withLock({ pendingDrainStop }) else { return } + stopTimer() + drainGeneration += 1 + let generation = drainGeneration + let remaining = bufferedSeconds + Logger.audioRendering.info("renderer output: draining \(remaining, format: .fixed(precision: 2)) s before stopping") + queue.asyncAfter(deadline: .now() + remaining + 0.05) { [weak self] in + guard let self, self.drainGeneration == generation else { return } + guard self.lock.withLock({ self.pendingDrainStop }) else { return } + self.lock.withLock { self.pendingDrainStop = false } + self.synchronizer.rate = 0 + self.isRateRunning = false + } + } + + // MARK: Feeding (on queue) + + private func startTimerIfNeeded() { + guard timer == nil else { return } + let timer = DispatchSource.makeTimerSource(queue: queue) + timer.schedule(deadline: .now(), repeating: .milliseconds(10), leeway: .milliseconds(5)) + timer.setEventHandler { [weak self] in self?.feed() } + timer.resume() + self.timer = timer + } + + private func stopTimer() { + timer?.cancel() + timer = nil + } + + private func feed() { + dispatchPrecondition(condition: .onQueue(queue)) + applyFlushIfNeeded() + guard lock.withLock({ wantsRunning && !isShutDown && !pendingFlush }) else { return } + + while renderer.isReadyForMoreMediaData, bufferedSeconds < maxBufferedSeconds { + guard lock.withLock({ wantsRunning && !pendingFlush }) else { break } + guard let (sampleBuffer, contentFrames) = renderNextSampleBuffer() else { break } + renderer.enqueue(sampleBuffer) + history.append(sampleBuffer) + if contentFrames == 0, bufferedSeconds >= maxBufferedSilenceSeconds { + // no audio yet: keep the silence close to real time + break + } + if !clockStarted { + let enqueuedSeconds = Double(nextFrame - startFrame) / Double(sampleRate) + if enqueuedSeconds >= prerollSeconds { + synchronizer.setRate(1, time: CMTime(value: startFrame, timescale: sampleRate)) + clockStarted = true + isRateRunning = true + } + } + } + pruneHistory() + } + + private func renderNextSampleBuffer() -> (CMSampleBuffer, contentFrames: UInt32)? { + renderBuffer.frameLength = framesPerBuffer + let audioBufferList = renderBuffer.mutableAudioBufferList + audioBufferList.pointee.mBuffers.mDataByteSize = framesPerBuffer * bytesPerFrame + guard lock.withLock({ wantsRunning && !pendingFlush }) else { return nil } + renderLock.lock() + let (rendered, contentFrames) = render(framesPerBuffer, audioBufferList) + renderLock.unlock() + if rendered < framesPerBuffer, let data = audioBufferList.pointee.mBuffers.mData { + // keep the timeline contiguous: pad with silence + let offset = Int(rendered * bytesPerFrame) + memset(data + offset, 0, Int((framesPerBuffer - rendered) * bytesPerFrame)) + } + audioBufferList.pointee.mBuffers.mDataByteSize = framesPerBuffer * bytesPerFrame + + var timing = CMSampleTimingInfo( + duration: CMTime(value: 1, timescale: sampleRate), + presentationTimeStamp: CMTime(value: nextFrame, timescale: sampleRate), + decodeTimeStamp: .invalid + ) + var sampleBuffer: CMSampleBuffer? + var status = CMSampleBufferCreate( + allocator: kCFAllocatorDefault, + dataBuffer: nil, + dataReady: false, + makeDataReadyCallback: nil, + refcon: nil, + formatDescription: formatDescription, + sampleCount: CMItemCount(framesPerBuffer), + sampleTimingEntryCount: 1, + sampleTimingArray: &timing, + sampleSizeEntryCount: 0, + sampleSizeArray: nil, + sampleBufferOut: &sampleBuffer + ) + guard status == noErr, let sampleBuffer else { + Logger.audioRendering.error("renderer output: CMSampleBufferCreate failed (\(status))") + return nil + } + status = CMSampleBufferSetDataBufferFromAudioBufferList( + sampleBuffer, + blockBufferAllocator: kCFAllocatorDefault, + blockBufferMemoryAllocator: kCFAllocatorDefault, + flags: 0, + bufferList: audioBufferList + ) + guard status == noErr else { + Logger.audioRendering.error("renderer output: setting sample data failed (\(status))") + return nil + } + let startSeconds = Double(nextFrame) / Double(sampleRate) + nextFrame += Int64(framesPerBuffer) + let endSeconds = Double(nextFrame) / Double(sampleRate) + lock.withLock { + enqueuedEndSeconds = endSeconds + contentTimeline.append((startSeconds, endSeconds, Int64(contentFrames))) + } + return (sampleBuffer, contentFrames) + } + + private func pruneHistory() { + let current = synchronizer.currentTime() + guard current.isValid else { return } + let currentSeconds = current.seconds + lock.withLock { + let heard = contentTimeline.prefix(while: { $0.end <= currentSeconds }).count + contentTimeline.removeFirst(heard) + } + while let first = history.first { + let end = CMSampleBufferGetPresentationTimeStamp(first) + CMSampleBufferGetDuration(first) + guard end < current else { break } + history.removeFirst() + } + } + + // MARK: Renderer notifications + + private func observeRenderer() { + let center = NotificationCenter.default + observers.append(center.addObserver( + forName: .AVSampleBufferAudioRendererWasFlushedAutomatically, + object: renderer, + queue: nil + ) { [weak self] notification in + let flushTime = (notification.userInfo?[AVSampleBufferAudioRendererFlushTimeKey] as? NSValue)?.timeValue + self?.queue.async { self?.reenqueueAfterAutomaticFlush(at: flushTime) } + }) + observers.append(center.addObserver( + forName: .AVSampleBufferAudioRendererOutputConfigurationDidChange, + object: renderer, + queue: nil + ) { _ in + Logger.audioRendering.info("renderer output: output configuration changed") + }) + observers.append(center.addObserver( + forName: AVSampleBufferRenderSynchronizer.rateDidChangeNotification, + object: synchronizer, + queue: nil + ) { [weak self] _ in + self?.queue.async { self?.handleRateChange() } + }) + } + + /// The renderer dropped its enqueued audio (e.g. the output route changed). The engine already + /// consumed that audio, so enqueue it again from the history. + private func reenqueueAfterAutomaticFlush(at flushTime: CMTime?) { + dispatchPrecondition(condition: .onQueue(queue)) + let time = flushTime ?? synchronizer.currentTime() + var count = 0 + for sampleBuffer in history { + let end = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + CMSampleBufferGetDuration(sampleBuffer) + guard end > time else { continue } + renderer.enqueue(sampleBuffer) + count += 1 + } + Logger.audioRendering.info( + "renderer output: flushed automatically at \(time.seconds, format: .fixed(precision: 2)) s, re-enqueued \(count) buffers" + ) + } + + private func handleRateChange() { + dispatchPrecondition(condition: .onQueue(queue)) + let rate = synchronizer.rate + Logger.audioRendering.info("renderer output: synchronizer rate changed to \(rate)") + // The system can stop the clock on its own (e.g. when the route changes back to the Mac). + // Restart it if playback is wanted and nothing on our side paused it. + if rate == 0, isRateRunning, clockStarted, lock.withLock({ wantsRunning || pendingDrainStop }) { + // at most once per second, so a clock the system keeps stopping cannot loop + let now = Date() + guard now.timeIntervalSince(lastClockRestart) > 1 else { return } + lastClockRestart = now + Logger.audioRendering.info("renderer output: restarting stopped clock") + synchronizer.rate = 1 + } + } +} + +/// Delivers delegate callbacks on the main queue after a delay, strictly in the order they were +/// scheduled (a later callback never overtakes an earlier one). +/// +/// Used with `SampleBufferRendererOutput` so item changes are reported when they are heard, +/// not when they are rendered. +final class OrderedDelayedNotifier { + private let lock = NSLock() + private var pending = [(deadline: DispatchTime, block: () -> Void)]() + private var lastDeadline = DispatchTime.now() + + func notify(after seconds: Double, _ block: @escaping () -> Void) { + let deadline: DispatchTime = lock.withLock { + var deadline = DispatchTime.now() + max(0, seconds) + if deadline < lastDeadline { + deadline = lastDeadline + } + lastDeadline = deadline + pending.append((deadline, block)) + return deadline + } + DispatchQueue.main.asyncAfter(deadline: deadline) { [weak self] in + self?.deliver(upTo: .now()) + } + } + + /// Delivers everything still pending right away (keeping the order), e.g. on seek or item change. + func deliverAllPending() { + DispatchQueue.main.async { [weak self] in + self?.deliver(upTo: .distantFuture) + } + lock.withLock { lastDeadline = .now() } + } + + private func deliver(upTo time: DispatchTime) { + dispatchPrecondition(condition: .onQueue(.main)) + let due: [() -> Void] = lock.withLock { + let count = pending.prefix(while: { $0.deadline <= time }).count + let due = pending.prefix(count).map(\.block) + pending.removeFirst(count) + return due + } + due.forEach { $0() } + } +}