diff --git a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift index e08ac94..f2f7ee8 100644 --- a/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift +++ b/AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift @@ -11,8 +11,16 @@ open class AudioPlayer { public weak var delegate: AudioPlayerDelegate? public var muted: Bool { - get { playerContext.muted.value } - set { playerContext.muted.write { $0 = newValue } } + get { rendererOutput?.isMuted ?? playerContext.muted.value } + set { + if let rendererOutput { + // the renderer output plays ahead of the engine: mute only the renderer, so it applies + // at once and no silenced audio stays enqueued after unmuting + rendererOutput.isMuted = newValue + } else { + playerContext.muted.write { $0 = newValue } + } + } } /// The volume of the audio @@ -20,8 +28,16 @@ open class AudioPlayer { /// Defaults to 1.0. Valid ranges are 0.0 to 1.0 /// The value is restricted from 0.0 to 1.0 public var volume: Float { - get { audioEngine.mainMixerNode.outputVolume } - set { audioEngine.mainMixerNode.outputVolume = min(1.0, max(0.0, newValue)) } + get { rendererOutput?.volume ?? audioEngine.mainMixerNode.outputVolume } + set { + let volume = min(1.0, max(0.0, newValue)) + if let rendererOutput { + // the renderer output plays ahead of the engine, so set the volume on the renderer + rendererOutput.volume = volume + } else { + audioEngine.mainMixerNode.outputVolume = volume + } + } } /// The playback rate of the player. @@ -55,13 +71,9 @@ open class AudioPlayer { /// - Returns: A `Double` value indicating the total duration. public var duration: Double { guard playerContext.internalState != .pendingNext else { return 0 } - playerContext.entriesLock.lock() - let playingEntry = playerContext.audioPlayingEntry - playerContext.entriesLock.unlock() - guard let entry = playingEntry else { return 0 } + guard let (entry, progress) = audiblePosition() else { return 0 } let entryDuration = entry.duration() - let progress = self.progress if entryDuration < progress, entryDuration > 0 { return progress } @@ -71,18 +83,36 @@ open class AudioPlayer { /// The progress of the audio playback, in seconds. public var progress: Double { guard playerContext.internalState != .pendingNext else { return 0 } - playerContext.entriesLock.lock() - let playingEntry = playerContext.audioPlayingEntry - playerContext.entriesLock.unlock() - guard let entry = playingEntry else { return 0 } + return audiblePosition()?.progress ?? 0 + } + + /// The entry that is heard right now and its progress. + /// + /// With the renderer output, audio is rendered ahead of time: the playing entry can already be + /// the next item while the end of the previous one is still heard (or the queue drains). + private func audiblePosition() -> (entry: AudioEntry, progress: Double)? { + let (playingEntry, previousEntry) = playerContext.entriesLock.withLock { + (playerContext.audioPlayingEntry, previousPlayingEntry) + } + let unheard = rendererOutput?.unheardContentSeconds ?? 0 + guard let entry = playingEntry else { + if let previousEntry, unheard > 0 { + return (previousEntry, max(0, previousEntry.progress - unheard)) + } + return nil + } entry.seekRequest.lock.lock() let seekRequested = entry.seekRequest.requested let seekTime = entry.seekRequest.time entry.seekRequest.lock.unlock() if seekRequested { - return seekTime + return (entry, seekTime) } - return entry.progress + let renderedOfEntry = Double(entry.framesPlayed) / outputAudioFormat.sampleRate + if unheard > renderedOfEntry, let previousEntry { + return (previousEntry, max(0, previousEntry.progress - (unheard - renderedOfEntry))) + } + return (entry, max(0, entry.progress - unheard)) } /// The number of audio frames that have been played @@ -92,7 +122,8 @@ open class AudioPlayer { let playingEntry = playerContext.audioPlayingEntry playerContext.entriesLock.unlock() guard let entry = playingEntry else { return 0 } - return entry.framesPlayed + let unheardFrames = Int((rendererOutput?.unheardContentSeconds ?? 0) * outputAudioFormat.sampleRate) + return max(0, entry.framesPlayed - unheardFrames) } /// Indicates whether seeking is supported for the currently playing audio @@ -167,6 +198,13 @@ open class AudioPlayer { private let fileStreamProcessor: AudioFileStreamProcessor private let playerRenderProcessor: AudioPlayerRenderProcessor + /// Set when `configuration.outputMode` is `.sampleBufferRenderer`, replaces the output audio unit `player` + private let rendererOutput: SampleBufferRendererOutput? + /// Delays item change callbacks by the renderer output latency + private let delegateNotifier = OrderedDelayedNotifier() + /// Renderer output only: the item before `audioPlayingEntry`, still heard for a moment after the + /// next one started rendering. Guarded by `playerContext.entriesLock`. + private var previousPlayingEntry: AudioEntry? private let frameFilterProcessor: FrameFilterProcessor private let serializationQueue: DispatchQueue @@ -213,6 +251,17 @@ open class AudioPlayer { outputAudioFormat: outputAudioFormat.basicStreamDescription ) + if case let .sampleBufferRenderer(maxBufferedSeconds) = self.configuration.outputMode { + rendererOutput = SampleBufferRendererOutput( + format: outputAudioFormat, + maxBufferedSeconds: maxBufferedSeconds + ) { [playerRenderProcessor] frameCount, ioData in + playerRenderProcessor.renderOutput(inNumberFrames: frameCount, ioData: ioData) + } + } else { + rendererOutput = nil + } + frameFilterProcessor = FrameFilterProcessor( mixerNodeProvider: { engine.mainMixerNode @@ -227,7 +276,11 @@ open class AudioPlayer { deinit { // The IO thread keeps pulling from the render callback until the hardware is stopped, // so stop it before the buffers it renders into are freed. - player.auAudioUnit.stopHardware() + if let rendererOutput { + rendererOutput.shutdown() + } else { + player.auAudioUnit.stopHardware() + } audioEngine.stop() playerContext.audioPlayingEntry?.close() clearQueue() @@ -426,6 +479,7 @@ open class AudioPlayer { self.playerContext.entriesLock.lock() self.playerContext.audioReadingEntry = nil self.playerContext.audioPlayingEntry = nil + self.previousPlayingEntry = nil self.playerContext.entriesLock.unlock() self.processSource() @@ -434,6 +488,11 @@ open class AudioPlayer { /// Pauses the audio playback public func pause() { + if let rendererOutput, playerContext.internalState == .stopped { + // the end of the queue may still be playing out of the renderer output + rendererOutput.stop(drain: false) + return + } if playerContext.internalState != .paused, playerContext.internalState.contains(.running) { stateBeforePaused = playerContext.internalState playerContext.setInternalState(to: .paused) @@ -579,6 +638,8 @@ open class AudioPlayer { /// Creates and configures an `AVAudioUnit` with an output configuration /// and assigns it to the `player` variable. private func configPlayerNode() { + // the renderer output pulls from the engine itself, no output unit needed + guard rendererOutput == nil else { return } AVAudioUnit.createAudioUnit(with: UnitDescriptions.output) { [weak self] result in guard let self = self else { return } switch result { @@ -683,9 +744,16 @@ open class AudioPlayer { /// Pauses the audio engine and stops the player's hardware private func pauseEngine() { guard isEngineRunning else { return } + if let rendererOutput { + // no rendering while the engine is reset below + rendererOutput.pause() + rendererOutput.waitForRenderToFinish() + } audioEngine.reset() audioEngine.pause() - player.auAudioUnit.stopHardware() + if rendererOutput == nil { + player.auAudioUnit.stopHardware() + } Logger.debug("engine paused ⏸", category: .generic) } @@ -693,8 +761,16 @@ open class AudioPlayer { /// /// - parameter reason: A value of `AudioPlayerStopReason` indicating the reason the engine stopped. private func stopEngine(reason: AudioPlayerStopReason) { + if let rendererOutput { + // at the end of the queue the enqueued audio still has to be heard; + // no rendering while the engine is stopped below + rendererOutput.stop(drain: reason == .eof) + rendererOutput.waitForRenderToFinish() + } audioEngine.stop() - player.auAudioUnit.stopHardware() + if rendererOutput == nil { + player.auAudioUnit.stopHardware() + } rendererContext.resetBuffers() playerContext.setInternalState(to: .stopped) playerContext.stopReason.write { $0 = reason } @@ -706,12 +782,17 @@ open class AudioPlayer { /// - parameter resetBuffers: A `Bool` value indicating if the buffers should be reset, prior starting the player. private func startPlayer(resetBuffers: Bool) { if resetBuffers { + // the renderer output was already told to drop its audio (setCurrentReading) rendererContext.resetBuffers() } do { try startEngineIfNeeded() - try player.auAudioUnit.allocateRenderResources() - try player.auAudioUnit.startHardware() + if let rendererOutput { + rendererOutput.start() + } else { + try player.auAudioUnit.allocateRenderResources() + try player.auAudioUnit.startHardware() + } } catch { stopEngine(reason: .error) raiseUnexpected(error: .audioSystemError(.playerStartError)) @@ -793,6 +874,8 @@ open class AudioPlayer { assert(playerContext.audioReadingEntry === playerContext.audioPlayingEntry, "reading and playing entry must be the same") fileStreamProcessor.processSeek() + // the audio enqueued for the old position must not be heard anymore + discardRendererOutput() } private func setCurrentReading(entry: AudioEntry?, startPlaying: Bool, shouldClearQueue: Bool) { @@ -800,6 +883,9 @@ open class AudioPlayer { Logger.debug("Setting current reading entry to: \(entry.debugDescription)", category: .generic) if startPlaying { rendererContext.fillSilenceAudioBuffer() + // new item or seek: what is still enqueued must not be heard, and the item change + // callbacks below must not wait for it + discardRendererOutput() } fileStreamProcessor.closeFileStreamIfNeeded() @@ -858,6 +944,9 @@ open class AudioPlayer { } playerContext.entriesLock.lock() + if rendererOutput != nil, !isPlayingSameItemProbablySeek { + previousPlayingEntry = entry + } playerContext.audioPlayingEntry = nextEntry let playingQueueEntryId = playerContext.audioPlayingEntry?.id ?? AudioEntryId(id: "") playerContext.entriesLock.unlock() @@ -867,7 +956,7 @@ open class AudioPlayer { let progressInFrames = entry.progressInFrames() let progress = Double(progressInFrames) / outputAudioFormat.basicStreamDescription.mSampleRate let duration = entry.duration() - asyncOnMain { [weak self] in + notifyDelegateWhenHeard { [weak self] in guard let self else { return } self.delegate?.audioPlayerDidFinishPlaying( player: self, @@ -890,7 +979,7 @@ open class AudioPlayer { } } - asyncOnMain { [weak self] in + notifyDelegateWhenHeard { [weak self] in guard let self = self else { return } self.delegate?.audioPlayerDidStartPlaying(player: self, with: playingQueueEntryId) } @@ -910,6 +999,9 @@ open class AudioPlayer { } playerContext.entriesLock.lock() + if rendererOutput != nil, !isPlayingSameItemProbablySeek { + previousPlayingEntry = entry + } playerContext.audioPlayingEntry = nil playerContext.entriesLock.unlock() if let entry = entry, !isPlayingSameItemProbablySeek { @@ -921,7 +1013,7 @@ open class AudioPlayer { sourceQueue.async { [weak self] in guard let self else { return } self.processSource() - asyncOnMain { + self.notifyDelegateWhenHeard { self.delegate?.audioPlayerDidFinishPlaying( player: self, entryId: entryId, @@ -939,6 +1031,25 @@ open class AudioPlayer { checkRenderWaitingAndNotifyIfNeeded() } + /// Drops the audio enqueued in the renderer output (seek, new item). + private func discardRendererOutput() { + guard let rendererOutput else { return } + rendererOutput.flush() + playerContext.entriesLock.withLock { previousPlayingEntry = nil } + delegateNotifier.deliverAllPending() + } + + /// Calls `block` on the main queue once the audio rendered so far is heard. + /// + /// Without the renderer output this is immediate, as before. + private func notifyDelegateWhenHeard(_ block: @escaping () -> Void) { + guard rendererOutput != nil else { + asyncOnMain(block) + return + } + delegateNotifier.notify(after: rendererOutput?.bufferedSeconds ?? 0, block) + } + /// Clears pending queues and informs the delegate private func clearQueue() { let pendingItems = entriesQueue.pendingEntriesId() diff --git a/AudioStreaming/Streaming/AudioPlayer/AudioPlayerConfiguration.swift b/AudioStreaming/Streaming/AudioPlayer/AudioPlayerConfiguration.swift index 949a823..cf78df8 100644 --- a/AudioStreaming/Streaming/AudioPlayer/AudioPlayerConfiguration.swift +++ b/AudioStreaming/Streaming/AudioPlayer/AudioPlayerConfiguration.swift @@ -6,6 +6,15 @@ import Foundation import OSLog +/// Where the rendered audio is played. +public enum AudioPlayerOutputMode: Equatable, Sendable { + /// An output audio unit (RemoteIO / default output). Lowest latency. + case audioUnit + /// An `AVSampleBufferAudioRenderer`, which takes part in AirPlay 2 (multi-room playback). + /// Audio is enqueued up to `maxBufferedSeconds` ahead of what is heard. + case sampleBufferRenderer(maxBufferedSeconds: Double) +} + public struct AudioPlayerConfiguration: Equatable { /// All pending items will be flushed when seeking a track if this is set to `true` let flushQueueOnSeek: Bool @@ -23,12 +32,16 @@ public struct AudioPlayerConfiguration: Equatable { /// Enables the internal logs let enableLogs: Bool + /// Where the rendered audio is played + let outputMode: AudioPlayerOutputMode + public static let `default` = AudioPlayerConfiguration(flushQueueOnSeek: false, bufferSizeInSeconds: 10, secondsRequiredToStartPlaying: 1, gracePeriodAfterSeekInSeconds: 0.5, secondsRequiredToStartPlayingAfterBufferUnderrun: 7, - enableLogs: false) + enableLogs: false, + outputMode: .audioUnit) /// Initializes the configuration for the `AudioPlayer` /// /// Parameters are pre set for convenience @@ -39,13 +52,15 @@ public struct AudioPlayerConfiguration: Equatable { /// - parameter gracePeriodAfterSeekInSeconds: Number of seconds of audio required after seek occurs. /// - parameter secondsRequiredToStartPlayingAfterBufferUnderrun: Number of seconds of audio required to before playback resumes after a buffer underrun /// - parameter enableLogs: Enables the internal logs + /// - parameter outputMode: Where the rendered audio is played /// public init(flushQueueOnSeek: Bool = true, bufferSizeInSeconds: Double = 10, secondsRequiredToStartPlaying: Double = 1, gracePeriodAfterSeekInSeconds: Double = 0.5, secondsRequiredToStartPlayingAfterBufferUnderrun: Int = 1, - enableLogs: Bool = true) + enableLogs: Bool = true, + outputMode: AudioPlayerOutputMode = .audioUnit) { self.flushQueueOnSeek = flushQueueOnSeek self.bufferSizeInSeconds = bufferSizeInSeconds @@ -53,6 +68,7 @@ public struct AudioPlayerConfiguration: Equatable { self.gracePeriodAfterSeekInSeconds = gracePeriodAfterSeekInSeconds self.secondsRequiredToStartPlayingAfterBufferUnderrun = secondsRequiredToStartPlayingAfterBufferUnderrun self.enableLogs = enableLogs + self.outputMode = outputMode Logger.isEnabled = enableLogs } @@ -81,6 +97,7 @@ public struct AudioPlayerConfiguration: Equatable { secondsRequiredToStartPlaying: secondsRequiredToStartPlaying, gracePeriodAfterSeekInSeconds: gracePeriodAfterSeekInSeconds, secondsRequiredToStartPlayingAfterBufferUnderrun: secondsRequiredToStartPlayingAfterBufferUnderrun, - enableLogs: enableLogs) + enableLogs: enableLogs, + outputMode: outputMode) } } diff --git a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioPlayerRenderProcessor.swift b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioPlayerRenderProcessor.swift index 23f8ee4..d474c4b 100644 --- a/AudioStreaming/Streaming/AudioPlayer/Processors/AudioPlayerRenderProcessor.swift +++ b/AudioStreaming/Streaming/AudioPlayer/Processors/AudioPlayerRenderProcessor.swift @@ -17,6 +17,8 @@ final class AudioPlayerRenderProcessor: NSObject { private let playerContext: AudioPlayerContext private let rendererContext: AudioRendererContext + /// Frames copied from the decoded buffer (not silence padding), only touched on the render thread + private var copiedFramesCount: Int64 = 0 private let outputAudioFormat: AudioStreamBasicDescription init(playerContext: AudioPlayerContext, @@ -210,6 +212,8 @@ final class AudioPlayerRenderProcessor: NSObject { } } + copiedFramesCount += Int64(totalFramesCopied) + if totalFramesCopied < inNumberFrames { let delta = inNumberFrames - totalFramesCopied if let mData = bufferList.mBuffers.mData { @@ -344,6 +348,24 @@ final class AudioPlayerRenderProcessor: NSObject { return status } + /// Renders the engine output into `ioData` without an output audio unit + /// (used by `SampleBufferRendererOutput`). + /// + /// - returns: The number of frames rendered into `ioData`, and how many of them are audio + /// content (counted as played by the entries) rather than silence padding. + func renderOutput( + inNumberFrames: UInt32, + ioData: UnsafeMutablePointer + ) -> (rendered: UInt32, content: UInt32) { + let copiedBefore = copiedFramesCount + var flags = AudioUnitRenderActionFlags() + _ = render(inNumberFrames: inNumberFrames, ioData: ioData, flags: &flags) + let content = UInt32(clamping: copiedFramesCount - copiedBefore) + let bytesRendered = rendererContext.inOutAudioBufferList[0].mBuffers.mDataByteSize + guard outputAudioFormat.mBytesPerFrame > 0 else { return (0, content) } + return (min(inNumberFrames, bytesRendered / outputAudioFormat.mBytesPerFrame), content) + } + func renderProvider( flags: UnsafeMutablePointer, timeStamp _: UnsafePointer, diff --git a/AudioStreaming/Streaming/AudioPlayer/SampleBufferRendererOutput.swift b/AudioStreaming/Streaming/AudioPlayer/SampleBufferRendererOutput.swift new file mode 100644 index 0000000..9f3c8a6 --- /dev/null +++ b/AudioStreaming/Streaming/AudioPlayer/SampleBufferRendererOutput.swift @@ -0,0 +1,510 @@ +// +// SampleBufferRendererOutput.swift +// AudioStreaming +// + +import AVFoundation +import CoreMedia +import OSLog + +/// Plays the manually rendered output of the audio engine through an `AVSampleBufferAudioRenderer` +/// driven by an `AVSampleBufferRenderSynchronizer`, instead of an output audio unit. +/// +/// Audio played this way takes part in AirPlay 2, including multi-room playback chosen in the +/// system route picker (together with the `.longFormAudio` route sharing policy). An output audio +/// unit is not routed to AirPlay devices picked there on Mac Catalyst. +/// +/// The renderer is fed ahead of time, so everything rendered by the engine is heard +/// `bufferedSeconds` later. `AudioPlayer` compensates its progress and its delegate callbacks. +/// +/// Threading: all control methods are non-blocking. They record the request under a lock and +/// apply it on the internal queue, which is also the queue that renders. The render function may +/// call back into `AudioPlayer` (which synchronously uses its serialization queue), so this type +/// must never wait for the internal queue from the caller's thread. +final class SampleBufferRendererOutput { + /// Renders `frameCount` frames into `ioData`; returns the frames rendered and how many of them are + /// audio content (counted as played by `AudioPlayer`) rather than silence padding. + typealias RenderFunction = (_ frameCount: UInt32, _ ioData: UnsafeMutablePointer) + -> (rendered: UInt32, content: UInt32) + + private let renderer = AVSampleBufferAudioRenderer() + private let synchronizer = AVSampleBufferRenderSynchronizer() + private let queue = DispatchQueue(label: "streaming.renderer.output", qos: .userInitiated) + private let queueKey = DispatchSpecificKey() + /// Held while the engine renders, see `waitForRenderToFinish()` + private let renderLock = NSLock() + + private let render: RenderFunction + private let format: AVAudioFormat + private let formatDescription: CMAudioFormatDescription + private let sampleRate: Int32 + private let bytesPerFrame: UInt32 + private let framesPerBuffer: AVAudioFrameCount = 4096 + private let renderBuffer: AVAudioPCMBuffer + /// Upper bound of audio enqueued ahead of what is heard. + private let maxBufferedSeconds: Double + /// Audio enqueued before the synchronizer clock starts after a flush. + private let prerollSeconds: Double = 0.1 + /// Silence (no audio content yet, e.g. waiting for data) is enqueued only this far ahead, so the + /// audio that arrives later is not queued behind seconds of silence. + private let maxBufferedSilenceSeconds: Double = 0.2 + + /// Guards the requested state below, `enqueuedEndSeconds` and `contentTimeline`. + private let lock = NSLock() + private var wantsRunning = false + private var pendingFlush = false + private var pendingDrainStop = false + private var enqueuedEndSeconds: Double = 0 + /// Enqueued buffers not completely heard yet, with the content frames each one holds + private var contentTimeline = [(start: Double, end: Double, contentFrames: Int64)]() + private var isShutDown = false + + // Only accessed on `queue` + private var timer: DispatchSourceTimer? + private var nextFrame: Int64 = 0 + private var startFrame: Int64 = 0 + private var clockStarted = false + private var isRateRunning = false + private var drainGeneration = 0 + private var lastClockRestart = Date.distantPast + /// Enqueued buffers that might not have been heard yet, re-enqueued after an automatic flush. + private var history = [CMSampleBuffer]() + private var observers = [NSObjectProtocol]() + + init?(format: AVAudioFormat, maxBufferedSeconds: Double, render: @escaping RenderFunction) { + var description: CMAudioFormatDescription? + let status = CMAudioFormatDescriptionCreate( + allocator: kCFAllocatorDefault, + asbd: format.streamDescription, + layoutSize: 0, + layout: nil, + magicCookieSize: 0, + magicCookie: nil, + extensions: nil, + formatDescriptionOut: &description + ) + guard status == noErr, let description, + let renderBuffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: framesPerBuffer) + else { + Logger.audioRendering.error("renderer output: cannot create format description (\(status))") + return nil + } + self.format = format + self.formatDescription = description + self.renderBuffer = renderBuffer + self.sampleRate = Int32(format.sampleRate) + self.bytesPerFrame = format.streamDescription.pointee.mBytesPerFrame + self.maxBufferedSeconds = max(0.5, maxBufferedSeconds) + self.render = render + + queue.setSpecific(key: queueKey, value: ()) + synchronizer.addRenderer(renderer) + observeRenderer() + } + + deinit { + shutdown() + } + + // MARK: Public controls + + var volume: Float { + get { renderer.volume } + set { renderer.volume = newValue } + } + + var isMuted: Bool { + get { renderer.isMuted } + set { renderer.isMuted = newValue } + } + + /// Seconds of audio content that were rendered (and counted as played by `AudioPlayer`) + /// but not heard yet. Silence padding (waiting for data) is not included. + var unheardContentSeconds: Double { + let current = synchronizer.currentTime().seconds + guard current.isFinite else { return 0 } + let frames: Double = lock.withLock { + var frames: Double = 0 + for chunk in contentTimeline where chunk.end > current { + if chunk.start >= current { + frames += Double(chunk.contentFrames) + } else { + let unheardPart = (chunk.end - current) / (chunk.end - chunk.start) + frames += Double(chunk.contentFrames) * unheardPart + } + } + return frames + } + return frames / Double(sampleRate) + } + + /// Seconds until the audio enqueued so far is heard (including silence padding). + var bufferedSeconds: Double { + let enqueuedEnd = lock.withLock { enqueuedEndSeconds } + let current = synchronizer.currentTime().seconds + guard current.isFinite else { return 0 } + return max(0, enqueuedEnd - current) + } + + /// Starts or resumes rendering and playback. + func start() { + lock.withLock { + wantsRunning = true + pendingDrainStop = false + } + queue.async { [weak self] in self?.applyStart() } + } + + /// Pauses playback, keeping the enqueued audio for resuming. + func pause() { + lock.withLock { wantsRunning = false } + queue.async { [weak self] in self?.applyPause() } + } + + /// Drops all enqueued audio (seek, new item). The clock restarts with the next `start()`. + func flush() { + lock.withLock { + pendingFlush = true + enqueuedEndSeconds = 0 + contentTimeline.removeAll() + } + queue.async { [weak self] in self?.applyFlushIfNeeded() } + } + + /// Stops playback. With `drain`, the audio already enqueued plays out first (end of queue). + func stop(drain: Bool) { + if drain { + lock.withLock { + wantsRunning = false + pendingDrainStop = true + } + queue.async { [weak self] in self?.applyDrainStop() } + } else { + lock.withLock { wantsRunning = false } + flush() + queue.async { [weak self] in self?.applyPause() } + } + } + + /// Stops everything. Called on deinit of the player. + func shutdown() { + let alreadyShutDown = lock.withLock { + let value = isShutDown + isShutDown = true + wantsRunning = false + return value + } + guard !alreadyShutDown else { return } + let stopOnQueue = { [self] in + observers.forEach { NotificationCenter.default.removeObserver($0) } + observers.removeAll() + stopTimer() + synchronizer.setRate(0, time: .zero) + renderer.flush() + } + // wait for a render in progress: the player frees the render buffers right after this + if DispatchQueue.getSpecific(key: queueKey) != nil { + stopOnQueue() + } else { + queue.sync(execute: stopOnQueue) + } + } + + /// Waits until a render that is already running has finished, at most a short moment. + /// + /// Call it after `pause()`/`stop(drain:)`, which prevent new renders, before resetting or stopping + /// the engine. The wait is bounded: the render can call back into the player, which may be + /// waiting here, and must not deadlock. + func waitForRenderToFinish() { + if renderLock.lock(before: Date().addingTimeInterval(0.25)) { + renderLock.unlock() + } + } + + // MARK: Applying requests (on queue) + + private func applyStart() { + dispatchPrecondition(condition: .onQueue(queue)) + applyFlushIfNeeded() + guard lock.withLock({ wantsRunning && !isShutDown }) else { return } + drainGeneration += 1 + if clockStarted, !isRateRunning { + synchronizer.rate = 1 + isRateRunning = true + } + startTimerIfNeeded() + feed() + } + + private func applyPause() { + dispatchPrecondition(condition: .onQueue(queue)) + guard !lock.withLock({ wantsRunning }) else { return } + stopTimer() + if isRateRunning { + synchronizer.rate = 0 + isRateRunning = false + } + } + + private func applyFlushIfNeeded() { + dispatchPrecondition(condition: .onQueue(queue)) + let flush = lock.withLock { + let value = pendingFlush + pendingFlush = false + return value + } + guard flush else { return } + drainGeneration += 1 + synchronizer.setRate(0, time: .zero) + isRateRunning = false + clockStarted = false + renderer.flush() + history.removeAll() + nextFrame = 0 + startFrame = 0 + lock.withLock { + enqueuedEndSeconds = 0 + contentTimeline.removeAll() + } + } + + private func applyDrainStop() { + dispatchPrecondition(condition: .onQueue(queue)) + guard lock.withLock({ pendingDrainStop }) else { return } + stopTimer() + drainGeneration += 1 + let generation = drainGeneration + let remaining = bufferedSeconds + Logger.audioRendering.info("renderer output: draining \(remaining, format: .fixed(precision: 2)) s before stopping") + queue.asyncAfter(deadline: .now() + remaining + 0.05) { [weak self] in + guard let self, self.drainGeneration == generation else { return } + guard self.lock.withLock({ self.pendingDrainStop }) else { return } + self.lock.withLock { self.pendingDrainStop = false } + self.synchronizer.rate = 0 + self.isRateRunning = false + } + } + + // MARK: Feeding (on queue) + + private func startTimerIfNeeded() { + guard timer == nil else { return } + let timer = DispatchSource.makeTimerSource(queue: queue) + timer.schedule(deadline: .now(), repeating: .milliseconds(10), leeway: .milliseconds(5)) + timer.setEventHandler { [weak self] in self?.feed() } + timer.resume() + self.timer = timer + } + + private func stopTimer() { + timer?.cancel() + timer = nil + } + + private func feed() { + dispatchPrecondition(condition: .onQueue(queue)) + applyFlushIfNeeded() + guard lock.withLock({ wantsRunning && !isShutDown && !pendingFlush }) else { return } + + while renderer.isReadyForMoreMediaData, bufferedSeconds < maxBufferedSeconds { + guard lock.withLock({ wantsRunning && !pendingFlush }) else { break } + guard let (sampleBuffer, contentFrames) = renderNextSampleBuffer() else { break } + renderer.enqueue(sampleBuffer) + history.append(sampleBuffer) + if contentFrames == 0, bufferedSeconds >= maxBufferedSilenceSeconds { + // no audio yet: keep the silence close to real time + break + } + if !clockStarted { + let enqueuedSeconds = Double(nextFrame - startFrame) / Double(sampleRate) + if enqueuedSeconds >= prerollSeconds { + synchronizer.setRate(1, time: CMTime(value: startFrame, timescale: sampleRate)) + clockStarted = true + isRateRunning = true + } + } + } + pruneHistory() + } + + private func renderNextSampleBuffer() -> (CMSampleBuffer, contentFrames: UInt32)? { + renderBuffer.frameLength = framesPerBuffer + let audioBufferList = renderBuffer.mutableAudioBufferList + audioBufferList.pointee.mBuffers.mDataByteSize = framesPerBuffer * bytesPerFrame + guard lock.withLock({ wantsRunning && !pendingFlush }) else { return nil } + renderLock.lock() + let (rendered, contentFrames) = render(framesPerBuffer, audioBufferList) + renderLock.unlock() + if rendered < framesPerBuffer, let data = audioBufferList.pointee.mBuffers.mData { + // keep the timeline contiguous: pad with silence + let offset = Int(rendered * bytesPerFrame) + memset(data + offset, 0, Int((framesPerBuffer - rendered) * bytesPerFrame)) + } + audioBufferList.pointee.mBuffers.mDataByteSize = framesPerBuffer * bytesPerFrame + + var timing = CMSampleTimingInfo( + duration: CMTime(value: 1, timescale: sampleRate), + presentationTimeStamp: CMTime(value: nextFrame, timescale: sampleRate), + decodeTimeStamp: .invalid + ) + var sampleBuffer: CMSampleBuffer? + var status = CMSampleBufferCreate( + allocator: kCFAllocatorDefault, + dataBuffer: nil, + dataReady: false, + makeDataReadyCallback: nil, + refcon: nil, + formatDescription: formatDescription, + sampleCount: CMItemCount(framesPerBuffer), + sampleTimingEntryCount: 1, + sampleTimingArray: &timing, + sampleSizeEntryCount: 0, + sampleSizeArray: nil, + sampleBufferOut: &sampleBuffer + ) + guard status == noErr, let sampleBuffer else { + Logger.audioRendering.error("renderer output: CMSampleBufferCreate failed (\(status))") + return nil + } + status = CMSampleBufferSetDataBufferFromAudioBufferList( + sampleBuffer, + blockBufferAllocator: kCFAllocatorDefault, + blockBufferMemoryAllocator: kCFAllocatorDefault, + flags: 0, + bufferList: audioBufferList + ) + guard status == noErr else { + Logger.audioRendering.error("renderer output: setting sample data failed (\(status))") + return nil + } + let startSeconds = Double(nextFrame) / Double(sampleRate) + nextFrame += Int64(framesPerBuffer) + let endSeconds = Double(nextFrame) / Double(sampleRate) + lock.withLock { + enqueuedEndSeconds = endSeconds + contentTimeline.append((startSeconds, endSeconds, Int64(contentFrames))) + } + return (sampleBuffer, contentFrames) + } + + private func pruneHistory() { + let current = synchronizer.currentTime() + guard current.isValid else { return } + let currentSeconds = current.seconds + lock.withLock { + let heard = contentTimeline.prefix(while: { $0.end <= currentSeconds }).count + contentTimeline.removeFirst(heard) + } + while let first = history.first { + let end = CMSampleBufferGetPresentationTimeStamp(first) + CMSampleBufferGetDuration(first) + guard end < current else { break } + history.removeFirst() + } + } + + // MARK: Renderer notifications + + private func observeRenderer() { + let center = NotificationCenter.default + observers.append(center.addObserver( + forName: .AVSampleBufferAudioRendererWasFlushedAutomatically, + object: renderer, + queue: nil + ) { [weak self] notification in + let flushTime = (notification.userInfo?[AVSampleBufferAudioRendererFlushTimeKey] as? NSValue)?.timeValue + self?.queue.async { self?.reenqueueAfterAutomaticFlush(at: flushTime) } + }) + observers.append(center.addObserver( + forName: .AVSampleBufferAudioRendererOutputConfigurationDidChange, + object: renderer, + queue: nil + ) { _ in + Logger.audioRendering.info("renderer output: output configuration changed") + }) + observers.append(center.addObserver( + forName: AVSampleBufferRenderSynchronizer.rateDidChangeNotification, + object: synchronizer, + queue: nil + ) { [weak self] _ in + self?.queue.async { self?.handleRateChange() } + }) + } + + /// The renderer dropped its enqueued audio (e.g. the output route changed). The engine already + /// consumed that audio, so enqueue it again from the history. + private func reenqueueAfterAutomaticFlush(at flushTime: CMTime?) { + dispatchPrecondition(condition: .onQueue(queue)) + let time = flushTime ?? synchronizer.currentTime() + var count = 0 + for sampleBuffer in history { + let end = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + CMSampleBufferGetDuration(sampleBuffer) + guard end > time else { continue } + renderer.enqueue(sampleBuffer) + count += 1 + } + Logger.audioRendering.info( + "renderer output: flushed automatically at \(time.seconds, format: .fixed(precision: 2)) s, re-enqueued \(count) buffers" + ) + } + + private func handleRateChange() { + dispatchPrecondition(condition: .onQueue(queue)) + let rate = synchronizer.rate + Logger.audioRendering.info("renderer output: synchronizer rate changed to \(rate)") + // The system can stop the clock on its own (e.g. when the route changes back to the Mac). + // Restart it if playback is wanted and nothing on our side paused it. + if rate == 0, isRateRunning, clockStarted, lock.withLock({ wantsRunning || pendingDrainStop }) { + // at most once per second, so a clock the system keeps stopping cannot loop + let now = Date() + guard now.timeIntervalSince(lastClockRestart) > 1 else { return } + lastClockRestart = now + Logger.audioRendering.info("renderer output: restarting stopped clock") + synchronizer.rate = 1 + } + } +} + +/// Delivers delegate callbacks on the main queue after a delay, strictly in the order they were +/// scheduled (a later callback never overtakes an earlier one). +/// +/// Used with `SampleBufferRendererOutput` so item changes are reported when they are heard, +/// not when they are rendered. +final class OrderedDelayedNotifier { + private let lock = NSLock() + private var pending = [(deadline: DispatchTime, block: () -> Void)]() + private var lastDeadline = DispatchTime.now() + + func notify(after seconds: Double, _ block: @escaping () -> Void) { + let deadline: DispatchTime = lock.withLock { + var deadline = DispatchTime.now() + max(0, seconds) + if deadline < lastDeadline { + deadline = lastDeadline + } + lastDeadline = deadline + pending.append((deadline, block)) + return deadline + } + DispatchQueue.main.asyncAfter(deadline: deadline) { [weak self] in + self?.deliver(upTo: .now()) + } + } + + /// Delivers everything still pending right away (keeping the order), e.g. on seek or item change. + func deliverAllPending() { + DispatchQueue.main.async { [weak self] in + self?.deliver(upTo: .distantFuture) + } + lock.withLock { lastDeadline = .now() } + } + + private func deliver(upTo time: DispatchTime) { + dispatchPrecondition(condition: .onQueue(.main)) + let due: [() -> Void] = lock.withLock { + let count = pending.prefix(while: { $0.deadline <= time }).count + let due = pending.prefix(count).map(\.block) + pending.removeFirst(count) + return due + } + due.forEach { $0() } + } +}