diff --git a/Sources/AetherEngine/AetherEngine+Loading.swift b/Sources/AetherEngine/AetherEngine+Loading.swift index ca0986f2..832b1b59 100644 --- a/Sources/AetherEngine/AetherEngine+Loading.swift +++ b/Sources/AetherEngine/AetherEngine+Loading.swift @@ -415,6 +415,29 @@ extension AetherEngine { self.applyRemoteHLSDisplayCriteria(format: fmt, options: options) } .store(in: &nativeCancellables) + // What was DELIVERED, which on a capped transcode is not what the host's library holds: without it + // a stats panel fell back to the original file's 3840x2160 for a 1280x720 stream. Its own sink, + // because a later read can refine the description without changing the dynamic range. + // dropFirst on both: a reused host replays the outgoing item's reading on subscribe, before load resets it. + host.$detectedVideoDescription + .dropFirst() + .compactMap { $0 } + .sink { [weak self] video in + guard let self else { return } + self.publishRemoteHLSVideoDescription(video) + } + .store(in: &nativeCancellables) + // No probe lists this route's audio, so AVPlayer's own tracks are the list. Informational: AVPlayer + // owns the audio selection here, and `selectAudioTrack` refuses the route rather than reload it. + host.$detectedAudioTracks + .dropFirst() + .sink { [weak self] readings in + guard let self else { return } + let (tracks, active) = RemoteHLSStreamDescription.audioTracks(readings) + self.audioTracks = tracks + self.activeAudioTrackIndex = active + } + .store(in: &nativeCancellables) // #168 follow-up: an advertised video rendition that never builds an item track means HEVC carried // in MPEG-TS segments, which AVFoundation's HLS demuxer does not support (audio-only, black). The // loopback ingest remuxes TS to fMP4 and plays the same stream, so reroute there transparently. @@ -546,7 +569,9 @@ extension AetherEngine { // #334: the ceiling on silence this path never had. AVPlayer's "gave up" covers an // origin that stops answering; it does not cover one that answers everything while // AVFoundation builds no track, where nothing terminal is ever published. - readinessDeadline: RemoteHLSReadinessDeadline.defaultBudgetSeconds)) + readinessDeadline: RemoteHLSReadinessDeadline.defaultBudgetSeconds, + // No probe lists this route's audio, so the item's own tracks are the list. + readsBackAudioTracks: true)) attachRemoteHLSCueClock(host: host, expectedGeneration: bypassGeneration) @@ -562,7 +587,9 @@ extension AetherEngine { host.play() } startMemoryProbe() - // No startLiveTelemetrySampler: all sampler counters read the loopback pipeline (demuxer / producer / cache / server), none of which exists on this bypass. + // The sampler reads AVPlayer's access log on this route: both bitrates, network rate and transfer, + // dropped frames and forward buffer. The loopback counters (producer, muxer, server) read zero. + startLiveTelemetrySampler() } /// AE#616: on this bypass `sourceTime` would otherwise be item time, which an origin that restarts diff --git a/Sources/AetherEngine/AetherEngine.swift b/Sources/AetherEngine/AetherEngine.swift index 8980b7aa..7cad009f 100644 --- a/Sources/AetherEngine/AetherEngine.swift +++ b/Sources/AetherEngine/AetherEngine.swift @@ -597,9 +597,13 @@ public final class AetherEngine: ObservableObject { @Published public internal(set) var sourceVideoCodecName: String? = nil /// AE#658: the source video stream's pixel format, bit depth, colour description and profile as the - /// container and the probe's decoder declared them, nil before load, on sources without video, and on - /// the probe-free native HLS bypass. Companion to `decodedVideoFormat`, which is what the engine's own - /// decoder actually produced. + /// container and the probe's decoder declared them, nil before load and on sources without video. + /// Companion to `decodedVideoFormat`, which is what the engine's own decoder actually produced. + /// + /// On the probe-free native HLS bypass it is read back from AVPlayer's item video track once that + /// resolves: colour from the format description's extensions, profile, bit depth and pixel format from + /// its avcC / hvcC record where it carries one (nil otherwise, AV1 and VP9 included). That is the + /// DELIVERED stream, which under a server-side transcode is not the file the host's library holds. @Published public internal(set) var sourceVideoStreamFormat: VideoStreamFormat? = nil /// AE#658: the format the engine's software decoder produced and the display buffer it went into, @@ -2162,8 +2166,20 @@ public final class AetherEngine: ObservableObject { /// Source video dimensions from the probe. Used as a bitmap-subtitle canvas fallback before the first PCS /// is parsed. 0 before load or when source has no video (AetherEngine#28). Also available in SourceProbe. + /// On the probe-free native HLS bypass they are the delivered stream's, read back from AVPlayer's item + /// track once it resolves, so a capped transcode reports the resolution it is actually playing. @Published public private(set) var sourceVideoWidth: Int32 = 0 @Published public private(set) var sourceVideoHeight: Int32 = 0 + + /// The remote-HLS bypass's stand-in for the probe: what AVPlayer parsed of the delivered video. + func publishRemoteHLSVideoDescription(_ video: RemoteHLSStreamDescription.Video) { + if video.width > 0, video.height > 0 { + sourceVideoWidth = video.width + sourceVideoHeight = video.height + } + if let codec = video.codecName { sourceVideoCodecName = codec } + sourceVideoStreamFormat = video.format + } /// Display-width multiplier for non-square source pixels: `sourceVideoWidth * this` is the width /// the picture presents at. 1 before load, on square-pixel sources, and whenever the declared /// ratio is one the engine refuses to believe (#290), so it is never a number the picture @@ -6653,7 +6669,20 @@ public final class AetherEngine: ObservableObject { /// expects ~0.5-1 s black frame (AVPlayer.replaceCurrentItem tears the surface). Display-criteria handshake /// is suppressed (video unchanged). `index` is the container stream index (TrackInfo.id). No-op if /// out-of-range, pointing at a non-audio stream, or already active. + /// + /// Not available on `VideoRoute.remoteBypass`: there `audioTracks` lists what AVPlayer built and is + /// informational, the selection belongs to AVFoundation, and a pick is logged and ignored rather than + /// turned into a reload the bypass has no audio override for. A host that wants another language on + /// that route loads a URL that carries it. public func selectAudioTrack(index: Int) { + if videoRoute == .remoteBypass { + EngineLog.emit( + "[AetherEngine] selectAudioTrack(\(index)) ignored: AVPlayer owns the audio selection on " + + "the remote-HLS bypass; audioTracks is informational there", + category: .engine + ) + return + } // Forward-only custom sources (incl. live HLS-ingest) can't rewind; rebuilding would re-consume a // drained FIFO and stall silently. Logged so a picker that does nothing is explainable. if isCustomSource && !customSourceIsSeekable { diff --git a/Sources/AetherEngine/Diagnostics/LiveTelemetry.swift b/Sources/AetherEngine/Diagnostics/LiveTelemetry.swift index a075d578..35616425 100644 --- a/Sources/AetherEngine/Diagnostics/LiveTelemetry.swift +++ b/Sources/AetherEngine/Diagnostics/LiveTelemetry.swift @@ -9,9 +9,13 @@ import Foundation /// interpret a sub-second decoded queue as the size of the compressed packet cache. public struct LiveTelemetry: Equatable, Sendable { // Enthusiast section + /// Mean rate over the last 10 s of the bytes the session pulled from its source, from the demuxer's + /// count on the loopback and software paths. nil until the window spans two ticks. The remote-HLS + /// bypass has no demuxer, and what AVPlayer transferred there is buffer fill at link speed rather than + /// the stream's rate, so on that route this is the playing variant's declared BANDWIDTH. public let instantBitrateMbps: Double? /// Lifetime mean rate of the session, over the seconds it spent consuming media rather than over - /// wall-clock seconds since it started (AE#514). A pause therefore leaves this value standing + /// wall-clock seconds since it started (AE#514). Metered from the same counter as `instantBitrateMbps`; on the remote-HLS bypass it is the variant's declared AVERAGE-BANDWIDTH, or BANDWIDTH where the master omits it. A pause therefore leaves this value standing /// still instead of dragging it toward zero for as long as the pause lasts, and so does the tail /// after end-of-media. nil until the session has both accrued active time and fetched something: /// like `networkThroughputMbps`, a value that cannot be measured yet is a gap, never a zero. diff --git a/Sources/AetherEngine/Diagnostics/LiveTelemetrySampler.swift b/Sources/AetherEngine/Diagnostics/LiveTelemetrySampler.swift index 45a12178..7bc50ad6 100644 --- a/Sources/AetherEngine/Diagnostics/LiveTelemetrySampler.swift +++ b/Sources/AetherEngine/Diagnostics/LiveTelemetrySampler.swift @@ -66,6 +66,10 @@ struct NativeAVFReadings: Sendable { var contiguousForwardBufferSeconds: Double? = nil /// Sum over all access-log events, for the [LagDiag] tick-over-tick drop delta. var droppedFramesLifetimeSum: Int = 0 + /// The current variant's declared BANDWIDTH and AVERAGE-BANDWIDTH, in bits per second, from the + /// latest access-log entry. The log reports an undeclared value as negative. + var indicatedBitrate: Double = -1 + var indicatedAverageBitrate: Double = -1 var currentTimeSeconds: Double = .nan var timeControlStatus: AVPlayer.TimeControlStatus = .paused var rate: Float = 0 @@ -196,6 +200,37 @@ final class LiveTelemetrySampler { return Double(windowBytes) * 8.0 / Double(activeSeconds) / 1_000_000.0 } + /// Which cumulative counter the two bitrate fields are metered from. + enum BitrateCounter: Equatable { + /// Bytes the engine's demuxer pulled from the source (loopback and software). + case demuxer + /// The bitrate the playing variant declares, from AVPlayer's access log. The remote-HLS bypass has no + /// demuxer, and what AVPlayer transferred is not the stream's rate: a buffer filling after a start or + /// a seek pulls at link speed (22.6 Mbps measured for a 3.7 Mbps transcode). The transfer stays in the + /// network fields. + case declaredVariant + } + + nonisolated static func bitrateCounter(for route: VideoRoute) -> BitrateCounter { + route == .remoteBypass ? .declaredVariant : .demuxer + } + + /// Both bitrate fields from the access log's declarations, in Mbps: instant is BANDWIDTH, average is + /// AVERAGE-BANDWIDTH, falling back to BANDWIDTH where the master omits it. Undeclared reads nil. + nonisolated static func declaredBitrates(indicated: Double, indicatedAverage: Double) + -> (instant: Double?, average: Double?) { + func mbps(_ bps: Double) -> Double? { bps.isFinite && bps > 0 ? bps / 1_000_000.0 : nil } + let instant = mbps(indicated) + return (instant, mbps(indicatedAverage) ?? instant) + } + + /// Whether the native branch has the loopback pipeline behind it. On the bypass the producer's A/V gap + /// does not exist (its getter reads 0, which would publish as a measured zero), and the #169 tail-park + /// synthesis stays with the route whose EXTINF overshoot it was built for. + nonisolated static func readsLoopbackPipeline(_ route: VideoRoute) -> Bool { + route != .remoteBypass + } + /// AE#514: whether a tick's second belongs in the lifetime average's divisor. /// /// That average used to divide by wall-clock time since the session started, which made a pause @@ -241,14 +276,16 @@ final class LiveTelemetrySampler { } lastTickTime = tickTime - // Instant + average bitrate from demuxer byte counters (both native and SW paths) + let route = engine.videoRoute + + // Instant + average bitrate from demuxer byte counters (loopback and SW; the bypass overrides below) let demuxerBytes = engine.demuxerBytesFetched let bytesThisTick = max(0, demuxerBytes - lastDemuxerBytes) lastDemuxerBytes = demuxerBytes if sessionStartBytes == 0 { sessionStartBytes = demuxerBytes } byteWindow.push(bytesThisTick) - let instantBitrateMbps: Double? + var instantBitrateMbps: Double? if byteWindow.count >= 2 { let totalBytes = byteWindow.sum let seconds = Double(byteWindow.count) @@ -262,7 +299,7 @@ final class LiveTelemetrySampler { activeSeconds: byteWindow.activeCount, samples: byteWindow.count) - let averageBitrateMbps = Self.averageBitrateMbps( + var averageBitrateMbps = Self.averageBitrateMbps( lifetimeBytes: max(0, demuxerBytes - sessionStartBytes), activeSeconds: activeSeconds) @@ -298,7 +335,8 @@ final class LiveTelemetrySampler { observedFps = nil displayCushionSeconds = nil accumulatedFrameDelaySeconds = nil - avSyncGapMs = engine.lastAVGapMs // HLSSegmentProducer audio-gate-open vs video-gate-open (native path only) + // HLSSegmentProducer audio-gate-open vs video-gate-open, loopback only. + avSyncGapMs = Self.readsLoopbackPipeline(route) ? engine.lastAVGapMs : nil if let player = engine.currentAVPlayer, let item = player.currentItem { var readings = await readNativeOffMain(player: player, item: item) // stop() may have cancelled this tick, or a reload seam may have swapped the @@ -382,13 +420,24 @@ final class LiveTelemetrySampler { accumulatedFrameDelaySeconds = nil } + // Remote-HLS bypass: no demuxer, so both rates are what the playing variant declares. + if Self.bitrateCounter(for: route) == .declaredVariant { + let declared = Self.declaredBitrates( + indicated: nativeReadings?.indicatedBitrate ?? -1, + indicatedAverage: nativeReadings?.indicatedAverageBitrate ?? -1) + instantBitrateMbps = declared.instant + averageBitrateMbps = declared.average + } + // Feed the extractor yield gate (#93 startup): nil on non-native paths keeps the // gate conservative there, but those paths have no active session to gate anyway. engine.extractorYieldState.setForwardBuffer(forwardBufferSeconds) if let readings = nativeReadings { emitLagDiag(engine: engine, readings: readings, netMbps: instantBitrateMbps) - evaluateEndOfMediaPark(engine: engine, readings: readings) + if Self.readsLoopbackPipeline(route) { + evaluateEndOfMediaPark(engine: engine, readings: readings) + } } let softwareCache = engine.softwarePacketCacheSnapshot @@ -554,6 +603,8 @@ final class LiveTelemetrySampler { let observed = event.observedBitrate readings.networkThroughputMbps = observed.isFinite && observed > 0 ? observed / 1_000_000.0 : nil + readings.indicatedBitrate = event.indicatedBitrate + readings.indicatedAverageBitrate = event.indicatedAverageBitrate } // AE#443: the counters are not. They are totals PER ENTRY, and AVFoundation opens a new entry // whenever the playback session changes under it, so reading `.last` publishes a number that diff --git a/Sources/AetherEngine/Native/NativeAVPlayerHost.swift b/Sources/AetherEngine/Native/NativeAVPlayerHost.swift index 61e19284..3a249b6c 100644 --- a/Sources/AetherEngine/Native/NativeAVPlayerHost.swift +++ b/Sources/AetherEngine/Native/NativeAVPlayerHost.swift @@ -163,6 +163,12 @@ final class NativeAVPlayerHost { /// spelling the engine publishes elsewhere. Set beside `detectedVideoFormat`, which the engine's sink /// reads it with; nil while no video track resolves. @Published private(set) var detectedVideoCodecName: String? + /// Dimensions and colour description of the delivered video from the same read, for the bypass's + /// `sourceVideoWidth` / `sourceVideoHeight` / `sourceVideoStreamFormat`. Set before `detectedVideoFormat`. + @Published private(set) var detectedVideoDescription: RemoteHLSStreamDescription.Video? + /// The audio tracks AVPlayer built for the item, read at the same two points as the video format. + /// The engine publishes them as `audioTracks` on the bypass, where no probe lists them. + @Published private(set) var detectedAudioTracks: [RemoteHLSStreamDescription.AudioReading] = [] /// AetherEngine#168 follow-up: fires once when the armed carriage watchdog concludes the master /// advertises a video rendition but AVPlayer never built a video track past the grace window @@ -413,6 +419,9 @@ final class NativeAVPlayerHost { /// then tunnels it through a 2-channel MAT carrier, so the route's channel count is not a /// statement about the audio and the surround-downmix warning below must not read it as one. var audioIsAtmosStreamCopy: Bool = false + /// Read the item's audio tracks back into `detectedAudioTracks`. Only the remote-HLS bypass needs + /// them; the loopback's probe already listed its audio, and the reads are XPC round trips. + var readsBackAudioTracks: Bool = false } /// AE#446 round 5: a fresh item is about to attach, invoked before anything can fetch a playlist @@ -638,6 +647,7 @@ final class NativeAVPlayerHost { } // #168: publish the item's real dynamic range for the probe-free remote-HLS badge. await self.publishDetectedVideoFormat(from: item) + await self.publishDetectedAudioTracks(from: item) guard self.sessionID == sid else { return } // #168 follow-up: watch for an advertised video rendition that never builds a track // (HEVC-in-MPEG-TS carriage); anchored at readyToPlay so dead origins never arm it. @@ -707,6 +717,7 @@ final class NativeAVPlayerHost { // #168: the video track can be absent from item.tracks at readyToPlay for HLS; // re-read once playing so the remote-HLS badge settles on the real dynamic range. await self.publishDetectedVideoFormat(from: item) + await self.publishDetectedAudioTracks(from: item) } } } @@ -2085,6 +2096,8 @@ final class NativeAVPlayerHost { detectedVideoFormat = nil detectedVideoFrameRate = nil detectedVideoCodecName = nil + detectedVideoDescription = nil + detectedAudioTracks = [] // #168 follow-up: the carriage verdict belongs to the outgoing item. carriageWatchdogTask?.cancel() carriageWatchdogTask = nil @@ -2439,9 +2452,11 @@ final class NativeAVPlayerHost { let ext = CMFormatDescriptionGetExtensions(cm) as? [String: Any] ?? [:] let transfer = ext[kCMFormatDescriptionExtension_TransferFunction as String] as? String let fmt = RemoteHLSFormatDetection.videoFormat(transferFunction: transfer, videoSubType: subType) - // Rate and codec before format: the engine's format sink reads both when it fires. + // Rate, codec and description before format: the engine's format sink reads them when it fires. if let rate, rate > 0 { detectedVideoFrameRate = rate } detectedVideoCodecName = RemoteHLSFormatDetection.codecName(videoSubType: subType) + let description = RemoteHLSStreamDescription.video(from: cm) + if detectedVideoDescription != description { detectedVideoDescription = description } if detectedVideoFormat != fmt { detectedVideoFormat = fmt EngineLog.emit( @@ -2455,6 +2470,34 @@ final class NativeAVPlayerHost { } } + /// The audio half of the read above, at the same two points. Language comes from the track, and where + /// the track has none (muxed HLS audio rarely does) from the audible option AVPlayer selected. + @MainActor + private func publishDetectedAudioTracks(from item: AVPlayerItem) async { + let sid = sessionID + guard sid != 0, playerItem === item, sessionContract.readsBackAudioTracks else { return } + var selectedOptionLanguage: String? + if let group = try? await item.asset.loadMediaSelectionGroup(for: .audible), + let option = item.currentMediaSelection.selectedMediaOption(in: group) { + selectedOptionLanguage = option.extendedLanguageTag + } + var readings: [RemoteHLSStreamDescription.AudioReading] = [] + for itemTrack in item.tracks { + guard let assetTrack = itemTrack.assetTrack, assetTrack.mediaType == .audio else { continue } + guard let cm = try? await assetTrack.load(.formatDescriptions).first else { continue } + let extendedTag = try? await assetTrack.load(.extendedLanguageTag) + let languageCode = try? await assetTrack.load(.languageCode) + let language = [extendedTag ?? nil, languageCode ?? nil, selectedOptionLanguage] + .compactMap { $0 }.first { !$0.isEmpty && $0 != "und" } + if let reading = RemoteHLSStreamDescription.audioReading( + from: cm, isEnabled: itemTrack.isEnabled, language: language) { + readings.append(reading) + } + } + guard sessionID == sid, playerItem === item else { return } + if detectedAudioTracks != readings { detectedAudioTracks = readings } + } + /// Compact video track summary: dimensions + color attachments (primaries/transfer/matrix). Mismatch vs source-side codecpar signals DV/HDR signaling didn't survive the muxer. /// Dump item.tracks on .failed (FourCC per track). Async: AVAssetTrack.formatDescriptions is /// load-based; assetTrack access is main-actor. diff --git a/Sources/AetherEngine/Native/RemoteHLSStreamDescription.swift b/Sources/AetherEngine/Native/RemoteHLSStreamDescription.swift new file mode 100644 index 00000000..fa842248 --- /dev/null +++ b/Sources/AetherEngine/Native/RemoteHLSStreamDescription.swift @@ -0,0 +1,296 @@ +import Foundation +import CoreMedia +import AudioToolbox +import AetherLibavutil +import AetherLibavcodec + +/// The `nativeRemoteHLS` bypass runs no libav probe, so the fields a stats panel reads (dimensions, colour +/// description, audio tracks) had no source on it, and a host fell back to metadata of the ORIGINAL file: +/// a capped 1280x720 H.264 transcode read "3840x2160, Main 10". AVPlayer has already parsed the delivered +/// stream into its item tracks' format descriptions, so this maps those onto the probe path's vocabulary +/// (libav names), without a second connection to the origin. Pure so it is unit-testable; the host does +/// the async track reads. +enum RemoteHLSStreamDescription { + + /// First id of the audio tracks published on the bypass. Synthetic, like the legible range at + /// `RemoteHLSMediaSelection.subtitleTrackIDBase`: there is no AVStream index on this route, and a + /// small ordinal would collide with a host's own source stream numbering. + static let audioTrackIDBase = 400_000 + + // MARK: - Video + + struct Video: Equatable, Sendable { + let width: Int32 + let height: Int32 + let codecName: String? + let format: VideoStreamFormat + } + + static func video(from description: CMFormatDescription) -> Video? { + guard CMFormatDescriptionGetMediaType(description) == kCMMediaType_Video else { return nil } + let subType = CMFormatDescriptionGetMediaSubType(description) + let dims = CMVideoFormatDescriptionGetDimensions(description) + let ext = CMFormatDescriptionGetExtensions(description) as? [String: Any] ?? [:] + return Video( + width: dims.width, height: dims.height, + codecName: RemoteHLSFormatDetection.codecName(videoSubType: subType), + format: videoStreamFormat(extensions: ext)) + } + + static func videoStreamFormat(extensions ext: [String: Any]) -> VideoStreamFormat { + let atoms = ext[kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms as String] as? [String: Any] ?? [:] + let config: CodecConfiguration? = + (atoms["avcC"] as? Data).flatMap(avcConfiguration) + ?? (atoms["hvcC"] as? Data).flatMap(hevcConfiguration) + let declaredDepth = (ext[kCMFormatDescriptionExtension_BitsPerComponent as String] as? NSNumber)?.intValue + let range: String? + if let full = ext[kCMFormatDescriptionExtension_FullRangeVideo as String] as? Bool { + range = full ? "pc" : "tv" + } else { + range = nil + } + return VideoStreamFormat( + pixelFormat: config?.pixelFormat, + bitDepth: config?.bitDepth ?? declaredDepth, + colorPrimaries: (ext[kCMFormatDescriptionExtension_ColorPrimaries as String] as? String).map(primariesName), + transfer: (ext[kCMFormatDescriptionExtension_TransferFunction as String] as? String).map(transferName), + matrix: (ext[kCMFormatDescriptionExtension_YCbCrMatrix as String] as? String).map(matrixName), + range: range, + profile: config?.profileName) + } + + /// CoreMedia's colour names onto libav's. An unknown name passes through: a value CoreMedia adds later + /// is still the stream's own declaration, and dropping it would read as "untagged". + static func primariesName(_ name: String) -> String { primariesNames[name] ?? name } + static func transferName(_ name: String) -> String { transferNames[name] ?? name } + static func matrixName(_ name: String) -> String { matrixNames[name] ?? name } + + private static let primariesNames: [String: String] = [ + kCMFormatDescriptionColorPrimaries_ITU_R_709_2 as String: "bt709", + kCMFormatDescriptionColorPrimaries_ITU_R_2020 as String: "bt2020", + kCMFormatDescriptionColorPrimaries_EBU_3213 as String: "bt470bg", + kCMFormatDescriptionColorPrimaries_SMPTE_C as String: "smpte170m", + kCMFormatDescriptionColorPrimaries_P3_D65 as String: "smpte432", + kCMFormatDescriptionColorPrimaries_DCI_P3 as String: "smpte431", + ] + + private static let transferNames: [String: String] = [ + kCMFormatDescriptionTransferFunction_ITU_R_709_2 as String: "bt709", + kCMFormatDescriptionTransferFunction_SMPTE_ST_2084_PQ as String: "smpte2084", + kCMFormatDescriptionTransferFunction_ITU_R_2100_HLG as String: "arib-std-b67", + kCMFormatDescriptionTransferFunction_ITU_R_2020 as String: "bt2020-10", + kCMFormatDescriptionTransferFunction_SMPTE_240M_1995 as String: "smpte240m", + kCMFormatDescriptionTransferFunction_Linear as String: "linear", + kCMFormatDescriptionTransferFunction_sRGB as String: "iec61966-2-1", + ] + + private static let matrixNames: [String: String] = [ + kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2 as String: "bt709", + kCMFormatDescriptionYCbCrMatrix_ITU_R_601_4 as String: "smpte170m", + kCMFormatDescriptionYCbCrMatrix_ITU_R_2020 as String: "bt2020nc", + kCMFormatDescriptionYCbCrMatrix_SMPTE_240M_1995 as String: "smpte240m", + ] + + /// What a decoder configuration record declares: libav's profile name, and where the record pins + /// them, chroma format and luma depth as a libav pixel format. + struct CodecConfiguration: Equatable { + let profileName: String? + let pixelFormat: String? + let bitDepth: Int? + } + + /// ISO/IEC 14496-15 avcC. The chroma/depth extension exists only for the High family, and there only + /// when the muxer wrote it; the lower profiles are 8-bit 4:2:0 by definition, and so is plain High. + static func avcConfiguration(_ record: Data) -> CodecConfiguration? { + let b = [UInt8](record) + guard b.count >= 7, b[0] == 1 else { return nil } + let idc = Int32(b[1]) + let compatibility = b[2] + var profile = idc + if idc == 66, compatibility & 0x40 != 0 { profile |= 1 << 9 } // AV_PROFILE_H264_CONSTRAINED + if [110, 122, 244].contains(idc), compatibility & 0x10 != 0 { profile |= 1 << 11 } // _INTRA + let highFamily: Set = [100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134, 135] + + var chroma: Int? + var depth: Int? + if highFamily.contains(idc) { + // Skip the SPS and PPS arrays to reach the extension. + var i = 5 + let spsCount = Int(b[i] & 0x1F); i += 1 + var ok = true + for _ in 0.. CodecConfiguration? { + let b = [UInt8](record) + guard b.count >= 23, b[0] == 1 else { return nil } + let depth = Int(b[17] & 0x07) + 8 + return CodecConfiguration( + profileName: VideoStreamFormat.profileName(codecID: AV_CODEC_ID_HEVC, profile: Int32(b[1] & 0x1F)), + pixelFormat: pixelFormatName(chroma: Int(b[16] & 0x03), depth: depth), + bitDepth: depth) + } + + /// The pixel format libav's decoder reports for a chroma format and depth, checked against libav's own + /// table so a combination it has no name for stays nil instead of becoming a made-up one. + static func pixelFormatName(chroma: Int?, depth: Int?) -> String? { + guard let chroma, let depth else { return nil } + let base: String + switch chroma { + case 0: base = "gray" + case 1: base = "yuv420p" + case 2: base = "yuv422p" + case 3: base = "yuv444p" + default: return nil + } + let name = depth == 8 ? base : "\(base)\(depth)le" + return av_get_pix_fmt(name) == AV_PIX_FMT_NONE ? nil : name + } + + // MARK: - Audio + + /// One audio track as AVPlayer built it, reduced to values so the mapping below stays pure. + struct AudioReading: Equatable, Sendable { + let formatID: AudioFormatID + let sampleRate: Double + let channels: Int + let bitsPerChannel: Int + let carriesJOC: Bool + let isEnabled: Bool + let language: String? + } + + static func audioReading(from description: CMFormatDescription, isEnabled: Bool, language: String?) -> AudioReading? { + guard CMFormatDescriptionGetMediaType(description) == kCMMediaType_Audio, + let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(description)?.pointee else { return nil } + var joc = false + if asbd.mFormatID == kAudioFormatEnhancedAC3 { + var size = 0 + if let cookie = CMAudioFormatDescriptionGetMagicCookie(description, sizeOut: &size), size > 0 { + joc = eac3CarriesJOC(dec3: Data(bytes: cookie, count: size)) + } + } + return AudioReading( + formatID: asbd.mFormatID, + sampleRate: asbd.mSampleRate, + channels: Int(asbd.mChannelsPerFrame), + bitsPerChannel: Int(asbd.mBitsPerChannel), + carriesJOC: joc, + isEnabled: isEnabled, + language: language) + } + + /// ETSI TS 102 366 Annex F dec3: `flag_ec3_extension_type_a` after the substream loop is the JOC + /// (Atmos) signal, the same one libav reads into E-AC-3 profile 30. Accepts the bare payload or the + /// whole box, since the magic cookie AVFoundation hands back has been seen both ways. + static func eac3CarriesJOC(dec3 record: Data) -> Bool { + var b = [UInt8](record) + if b.count >= 8, Array(b[4..<8]) == Array("dec3".utf8) { b.removeFirst(8) } + var reader = BitReader(bytes: b) + guard reader.skip(13), let indSubs = reader.read(3) else { return false } + for _ in 0...indSubs { + // fscod bsid reserved asvc bsmod acmod lfeon reserved + guard reader.skip(2 + 5 + 1 + 1 + 3 + 3 + 1 + 3), let depSubs = reader.read(4) else { return false } + guard reader.skip(depSubs > 0 ? 9 : 1) else { return false } + } + guard reader.skip(7), let flag = reader.read(1) else { return false } + return flag == 1 + } + + /// Audio format IDs onto the libavcodec name the probe path publishes, AAC's object type onto its profile. + static func audioCodec(formatID: AudioFormatID) -> (name: String, profile: String?) { + switch formatID { + case kAudioFormatMPEG4AAC: + return ("aac", VideoStreamFormat.profileName(codecID: AV_CODEC_ID_AAC, profile: 1)) + case kAudioFormatMPEG4AAC_HE: + return ("aac", VideoStreamFormat.profileName(codecID: AV_CODEC_ID_AAC, profile: 4)) + case kAudioFormatMPEG4AAC_HE_V2: + return ("aac", VideoStreamFormat.profileName(codecID: AV_CODEC_ID_AAC, profile: 28)) + case kAudioFormatMPEG4AAC_LD: return ("aac", "LD") + case kAudioFormatMPEG4AAC_ELD: return ("aac", "ELD") + case kAudioFormatAC3: return ("ac3", nil) + case kAudioFormatEnhancedAC3: return ("eac3", nil) + case kAudioFormatMPEGLayer1: return ("mp1", nil) + case kAudioFormatMPEGLayer2: return ("mp2", nil) + case kAudioFormatMPEGLayer3: return ("mp3", nil) + case kAudioFormatFLAC: return ("flac", nil) + case kAudioFormatOpus: return ("opus", nil) + case kAudioFormatAppleLossless: return ("alac", nil) + case kAudioFormatLinearPCM: return ("pcm", nil) + case 0x61632D34: return ("ac4", nil) // 'ac-4' + default: + let bytes = [UInt8((formatID >> 24) & 0xFF), UInt8((formatID >> 16) & 0xFF), + UInt8((formatID >> 8) & 0xFF), UInt8(formatID & 0xFF)] + return (String(bytes: bytes, encoding: .macOSRoman) ?? "unknown", nil) + } + } + + /// The published list: one entry per audio track AVPlayer built, ids from `audioTrackIDBase` in item + /// order, the enabled one as the active index. Named the way the probe path names an untitled track. + static func audioTracks(_ readings: [AudioReading]) -> (tracks: [TrackInfo], activeID: Int?) { + var tracks: [TrackInfo] = [] + var activeID: Int? + for (i, reading) in readings.enumerated() { + let id = audioTrackIDBase + i + let codec = audioCodec(formatID: reading.formatID) + let language = reading.language.flatMap { $0.isEmpty || $0 == "und" ? nil : $0 } + let name = language.map { "\($0.uppercased()) (\(codec.name))" } ?? "Track \(i + 1) (\(codec.name))" + let profile = reading.carriesJOC + ? VideoStreamFormat.profileName(codecID: AV_CODEC_ID_EAC3, profile: 30) + : codec.profile + tracks.append(TrackInfo( + id: id, name: name, codec: codec.name, language: language, + channels: reading.channels, isDefault: reading.isEnabled, isAtmos: reading.carriesJOC, + sampleRate: reading.sampleRate.isFinite ? Int(reading.sampleRate) : 0, + bitsPerSample: reading.bitsPerChannel, profile: profile)) + if reading.isEnabled, activeID == nil { activeID = id } + } + return (tracks, activeID) + } + + private struct BitReader { + let bytes: [UInt8] + var position = 0 + + init(bytes: [UInt8]) { self.bytes = bytes } + + mutating func read(_ count: Int) -> Int? { + guard position + count <= bytes.count * 8 else { return nil } + var value = 0 + for _ in 0..> (7 - UInt8(position % 8))) & 1 + value = value << 1 | Int(bit) + position += 1 + } + return value + } + + mutating func skip(_ count: Int) -> Bool { read(count) != nil || count == 0 } + } +} diff --git a/Sources/aetherctl/PlaybackCmd.swift b/Sources/aetherctl/PlaybackCmd.swift index a9517dc3..06963d07 100644 --- a/Sources/aetherctl/PlaybackCmd.swift +++ b/Sources/aetherctl/PlaybackCmd.swift @@ -153,6 +153,8 @@ func runPlay(url: URL, seconds: Double, live: Bool, nativeHLS: Bool = false, liv private func networkTelemetryFragment(_ telemetry: LiveTelemetry?) -> String { guard let telemetry else { return "" } var out = "" + if let inst = telemetry.instantBitrateMbps { out += String(format: " inst=%.2fMbps", inst) } + if let avg = telemetry.averageBitrateMbps { out += String(format: " avg=%.2fMbps", avg) } if let mbps = telemetry.networkThroughputMbps { out += String(format: " net=%.2fMbps", mbps) } if let rx = telemetry.networkTransferredBytes { out += String(format: " rx=%.1fMB", Double(rx) / 1_048_576) } out += String(format: " origin=%.1fMB", Double(telemetry.demuxerBytesFetched) / 1_048_576) @@ -1322,6 +1324,16 @@ private func playSmokeTest(url: URL, seconds: Double, live: Bool, forceSoftware: // one when the question is what a host's picker ends up showing. let finalSubtitleTracks = engine.subtitleTracks let finalActiveSubtitle = engine.activeSubtitleTrackIndex + // The settled stats-panel identity. The SOURCE line at load prints before the remote-HLS bypass has + // read anything back from AVPlayer's item, so it reads empty there by construction. + let settledSource = "SOURCE codec=\(engine.sourceVideoCodecName ?? "nil") " + + "\(engine.sourceVideoWidth)x\(engine.sourceVideoHeight) " + + "fmt=\(engine.sourceVideoFormat) stream=\(engine.sourceVideoStreamFormat.map { String(describing: $0) } ?? "nil")" + let settledAudio = engine.audioTracks.map { + "#\($0.id) \($0.name) codec=\($0.codec) ch=\($0.channels) sr=\($0.sampleRate)" + + "\($0.profile.map { " profile=\($0)" } ?? "")\($0.isAtmos ? " atmos" : "")" + }.joined(separator: ", ") + let settledActiveAudio = engine.activeAudioTrackIndex if record != nil { await engine.stopRecording() print(" RECORD final state: \(engine.recordingState)") @@ -1364,6 +1376,8 @@ private func playSmokeTest(url: URL, seconds: Double, live: Bool, forceSoftware: print("subtitle tracks (* = external): \(listed)") print("active subtitle: \(finalActiveSubtitle.map(String.init) ?? "none")") } + print("settled \(settledSource)") + print("audio tracks: \(settledAudio.isEmpty ? "none" : settledAudio) active=\(settledActiveAudio.map(String.init) ?? "none")") print("final t=\(String(format: "%.2f", finalTime))s state=\(String(describing: endState)) cues=\(cueCount)") let closingWindow = await MainActor.run { lastCues } print("WINDOW \(closingWindow.count) cues in the last published window") diff --git a/Tests/AetherEngineTests/RemoteHLSStreamDescriptionTests.swift b/Tests/AetherEngineTests/RemoteHLSStreamDescriptionTests.swift new file mode 100644 index 00000000..d1444967 --- /dev/null +++ b/Tests/AetherEngineTests/RemoteHLSStreamDescriptionTests.swift @@ -0,0 +1,264 @@ +import Foundation +import Testing +import CoreMedia +import AudioToolbox +@testable import AetherEngine + +/// The `nativeRemoteHLS` bypass runs no libav probe, so a stats panel had nothing but the host's own +/// metadata of the ORIGINAL file to show: a capped 1280x720 H.264 transcode read "3840x2160, Main 10". +/// AVPlayer's parsed item tracks already carry the delivered stream, so these pin the mapping from its +/// format descriptions onto the fields the probe path publishes, in the probe path's vocabulary. +@Suite("RemoteHLSStreamDescription") +struct RemoteHLSStreamDescriptionTests { + + // MARK: - Fixtures + + private static let avc1: FourCharCode = 0x61766331 + private static let hvc1: FourCharCode = 0x68766331 + + /// avcC with no SPS/PPS: version, profile_idc, constraint flags, level, lengthSizeMinusOne, 0 SPS, 0 PPS. + private static func avcC(profileIDC: UInt8, compatibility: UInt8 = 0, highExtension: (chroma: UInt8, lumaDepth: UInt8)? = nil) -> Data { + var bytes: [UInt8] = [1, profileIDC, compatibility, 31, 0xFF, 0xE0, 0x00] + if let ext = highExtension { + bytes += [0xFC | ext.chroma, 0xF8 | (ext.lumaDepth - 8), 0xF8 | (ext.lumaDepth - 8), 0x00] + } + return Data(bytes) + } + + /// hvcC header (23 bytes) with no parameter-set arrays. + private static func hvcC(profileIDC: UInt8, chroma: UInt8 = 1, lumaDepth: UInt8 = 8) -> Data { + var bytes = [UInt8](repeating: 0, count: 23) + bytes[0] = 1 + bytes[1] = profileIDC & 0x1F + bytes[12] = 150 + bytes[16] = 0xFC | chroma + bytes[17] = 0xF8 | (lumaDepth - 8) + bytes[18] = 0xF8 | (lumaDepth - 8) + return Data(bytes) + } + + private static func videoDescription( + subType: FourCharCode, width: Int32, height: Int32, + extensions: [CFString: Any] + ) throws -> CMFormatDescription { + var desc: CMFormatDescription? + let status = CMVideoFormatDescriptionCreate( + allocator: kCFAllocatorDefault, codecType: subType, width: width, height: height, + extensions: extensions as CFDictionary, formatDescriptionOut: &desc) + try #require(status == noErr) + return try #require(desc) + } + + private static func audioDescription(formatID: AudioFormatID, sampleRate: Double, channels: UInt32, + bitsPerChannel: UInt32 = 0, cookie: Data? = nil) throws -> CMFormatDescription { + var asbd = AudioStreamBasicDescription( + mSampleRate: sampleRate, mFormatID: formatID, mFormatFlags: 0, mBytesPerPacket: 0, + mFramesPerPacket: 1536, mBytesPerFrame: 0, mChannelsPerFrame: channels, + mBitsPerChannel: bitsPerChannel, mReserved: 0) + var desc: CMFormatDescription? + let status: OSStatus + if let cookie { + status = cookie.withUnsafeBytes { raw in + CMAudioFormatDescriptionCreate( + allocator: kCFAllocatorDefault, asbd: &asbd, layoutSize: 0, layout: nil, + magicCookieSize: cookie.count, magicCookie: raw.baseAddress, extensions: nil, + formatDescriptionOut: &desc) + } + } else { + status = CMAudioFormatDescriptionCreate( + allocator: kCFAllocatorDefault, asbd: &asbd, layoutSize: 0, layout: nil, + magicCookieSize: 0, magicCookie: nil, extensions: nil, formatDescriptionOut: &desc) + } + try #require(status == noErr) + return try #require(desc) + } + + /// dec3 payload for one independent substream without dependents, JOC flag as given. + private static func dec3(joc: Bool) -> Data { + // data_rate(13)=640 num_ind_sub(3)=0 | fscod bsid reserved asvc bsmod acmod lfeon reserved + // num_dep_sub(4)=0 reserved(1) | reserved(7) flag_ec3_extension_type_a(1) | complexity_index(8) + Data([0x50, 0x00, 0x20, 0x0F, 0x00, joc ? 0x01 : 0x00, joc ? 16 : 0]) + } + + // MARK: - Delivered video + + @Test("the device case: a 720p H.264 High transcode reads as what was delivered") + func deliveredH264Transcode() throws { + let desc = try Self.videoDescription( + subType: Self.avc1, width: 1280, height: 720, + extensions: [ + kCMFormatDescriptionExtension_ColorPrimaries: kCMFormatDescriptionColorPrimaries_ITU_R_709_2, + kCMFormatDescriptionExtension_TransferFunction: kCMFormatDescriptionTransferFunction_ITU_R_709_2, + kCMFormatDescriptionExtension_YCbCrMatrix: kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2, + kCMFormatDescriptionExtension_FullRangeVideo: false, + kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms: ["avcC": Self.avcC(profileIDC: 100)], + ]) + let video = try #require(RemoteHLSStreamDescription.video(from: desc)) + #expect(video.width == 1280) + #expect(video.height == 720) + #expect(video.codecName == "h264") + #expect(video.format == VideoStreamFormat( + pixelFormat: "yuv420p", bitDepth: 8, colorPrimaries: "bt709", transfer: "bt709", + matrix: "bt709", range: "tv", profile: "High")) + } + + @Test("HEVC Main 10 PQ reads its profile, depth and BT.2020 description from hvcC and the extensions") + func hevcMain10PQ() throws { + let desc = try Self.videoDescription( + subType: Self.hvc1, width: 3840, height: 2160, + extensions: [ + kCMFormatDescriptionExtension_ColorPrimaries: kCMFormatDescriptionColorPrimaries_ITU_R_2020, + kCMFormatDescriptionExtension_TransferFunction: kCMFormatDescriptionTransferFunction_SMPTE_ST_2084_PQ, + kCMFormatDescriptionExtension_YCbCrMatrix: kCMFormatDescriptionYCbCrMatrix_ITU_R_2020, + kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms: ["hvcC": Self.hvcC(profileIDC: 2, lumaDepth: 10)], + ]) + let video = try #require(RemoteHLSStreamDescription.video(from: desc)) + #expect(video.codecName == "hevc") + #expect(video.format.pixelFormat == "yuv420p10le") + #expect(video.format.bitDepth == 10) + #expect(video.format.profile == "Main 10") + #expect(video.format.colorPrimaries == "bt2020") + #expect(video.format.transfer == "smpte2084") + #expect(video.format.matrix == "bt2020nc") + #expect(video.format.range == nil) + } + + /// Same rule as AE#658 on the probe path: an untagged stream is not reported as BT.709. + @Test("an untagged stream keeps its gaps") + func untaggedStaysNil() throws { + let desc = try Self.videoDescription(subType: Self.avc1, width: 640, height: 360, extensions: [:]) + let video = try #require(RemoteHLSStreamDescription.video(from: desc)) + #expect(video.format.colorPrimaries == nil) + #expect(video.format.transfer == nil) + #expect(video.format.matrix == nil) + #expect(video.format.range == nil) + #expect(video.format.profile == nil) + #expect(video.format.pixelFormat == nil) + #expect(video.format.bitDepth == nil) + } + + @Test("CoreMedia colour names map onto libav's") + func colourNames() { + #expect(RemoteHLSStreamDescription.primariesName(kCMFormatDescriptionColorPrimaries_P3_D65 as String) == "smpte432") + #expect(RemoteHLSStreamDescription.primariesName(kCMFormatDescriptionColorPrimaries_DCI_P3 as String) == "smpte431") + #expect(RemoteHLSStreamDescription.primariesName(kCMFormatDescriptionColorPrimaries_EBU_3213 as String) == "bt470bg") + #expect(RemoteHLSStreamDescription.primariesName(kCMFormatDescriptionColorPrimaries_SMPTE_C as String) == "smpte170m") + #expect(RemoteHLSStreamDescription.transferName(kCMFormatDescriptionTransferFunction_ITU_R_2100_HLG as String) == "arib-std-b67") + #expect(RemoteHLSStreamDescription.transferName(kCMFormatDescriptionTransferFunction_ITU_R_2020 as String) == "bt2020-10") + #expect(RemoteHLSStreamDescription.transferName(kCMFormatDescriptionTransferFunction_Linear as String) == "linear") + #expect(RemoteHLSStreamDescription.transferName(kCMFormatDescriptionTransferFunction_sRGB as String) == "iec61966-2-1") + #expect(RemoteHLSStreamDescription.matrixName(kCMFormatDescriptionYCbCrMatrix_ITU_R_601_4 as String) == "smpte170m") + #expect(RemoteHLSStreamDescription.matrixName(kCMFormatDescriptionYCbCrMatrix_SMPTE_240M_1995 as String) == "smpte240m") + // A name CoreMedia adds later passes through rather than vanishing. + #expect(RemoteHLSStreamDescription.primariesName("Future_Primaries") == "Future_Primaries") + } + + @Test("constraint flags in avcC reach the profile name") + func constrainedBaseline() { + let config = RemoteHLSStreamDescription.avcConfiguration(Self.avcC(profileIDC: 66, compatibility: 0x40)) + #expect(config?.profileName == "Constrained Baseline") + let high10 = RemoteHLSStreamDescription.avcConfiguration( + Self.avcC(profileIDC: 110, highExtension: (chroma: 1, lumaDepth: 10))) + #expect(high10?.profileName == "High 10") + #expect(high10?.pixelFormat == "yuv420p10le") + #expect(high10?.bitDepth == 10) + } + + @Test("a truncated configuration record yields nothing rather than a guess") + func truncatedRecords() { + #expect(RemoteHLSStreamDescription.avcConfiguration(Data([1, 100])) == nil) + #expect(RemoteHLSStreamDescription.hevcConfiguration(Data([1, 2, 0, 0])) == nil) + } + + @Test("an audio description is not a video one") + func audioIsNotVideo() throws { + let desc = try Self.audioDescription(formatID: kAudioFormatEnhancedAC3, sampleRate: 48_000, channels: 6) + #expect(RemoteHLSStreamDescription.video(from: desc) == nil) + } + + // MARK: - Audio tracks + + @Test("the device case: E-AC-3 5.1 at 48 kHz becomes an eac3 TrackInfo") + func eac3Track() throws { + let desc = try Self.audioDescription(formatID: kAudioFormatEnhancedAC3, sampleRate: 48_000, channels: 6) + let reading = try #require(RemoteHLSStreamDescription.audioReading(from: desc, isEnabled: true, language: "eng")) + let (tracks, active) = RemoteHLSStreamDescription.audioTracks([reading]) + let track = try #require(tracks.first) + #expect(tracks.count == 1) + #expect(track.id == RemoteHLSStreamDescription.audioTrackIDBase) + #expect(active == track.id) + #expect(track.codec == "eac3") + #expect(track.channels == 6) + #expect(track.sampleRate == 48_000) + #expect(track.bitsPerSample == 0) + #expect(track.language == "eng") + #expect(track.name == "ENG (eac3)") + #expect(track.isAtmos == false) + #expect(track.isDefault) + } + + @Test("a dec3 cookie announcing JOC marks the track Atmos, as a box or as its payload") + func eac3JOC() throws { + let payload = Self.dec3(joc: true) + var box = Data([0, 0, 0, UInt8(8 + payload.count)]) + box.append(contentsOf: Array("dec3".utf8)) + box.append(payload) + for cookie in [payload, box] { + let desc = try Self.audioDescription( + formatID: kAudioFormatEnhancedAC3, sampleRate: 48_000, channels: 6, cookie: cookie) + let reading = try #require(RemoteHLSStreamDescription.audioReading(from: desc, isEnabled: true, language: nil)) + let track = try #require(RemoteHLSStreamDescription.audioTracks([reading]).tracks.first) + #expect(track.isAtmos) + #expect(track.profile == "Dolby Digital Plus + Dolby Atmos") + } + #expect(RemoteHLSStreamDescription.eac3CarriesJOC(dec3: Self.dec3(joc: false)) == false) + #expect(RemoteHLSStreamDescription.eac3CarriesJOC(dec3: Data([0x50])) == false) + } + + @Test("audio format IDs map onto libavcodec names, AAC with its profile") + func audioCodecNames() { + #expect(RemoteHLSStreamDescription.audioCodec(formatID: kAudioFormatMPEG4AAC).name == "aac") + #expect(RemoteHLSStreamDescription.audioCodec(formatID: kAudioFormatMPEG4AAC).profile == "LC") + #expect(RemoteHLSStreamDescription.audioCodec(formatID: kAudioFormatMPEG4AAC_HE).profile == "HE-AAC") + #expect(RemoteHLSStreamDescription.audioCodec(formatID: kAudioFormatMPEG4AAC_HE_V2).profile == "HE-AACv2") + #expect(RemoteHLSStreamDescription.audioCodec(formatID: kAudioFormatAC3).name == "ac3") + #expect(RemoteHLSStreamDescription.audioCodec(formatID: kAudioFormatEnhancedAC3).name == "eac3") + #expect(RemoteHLSStreamDescription.audioCodec(formatID: kAudioFormatMPEGLayer3).name == "mp3") + #expect(RemoteHLSStreamDescription.audioCodec(formatID: kAudioFormatFLAC).name == "flac") + #expect(RemoteHLSStreamDescription.audioCodec(formatID: kAudioFormatOpus).name == "opus") + #expect(RemoteHLSStreamDescription.audioCodec(formatID: kAudioFormatAppleLossless).name == "alac") + // An unmapped format still names itself, like the video side. + #expect(RemoteHLSStreamDescription.audioCodec(formatID: 0x7A7A7A7A).name == "zzzz") + } + + @Test("ids are synthetic and ordinal, the enabled track is the active one, und is no language") + func multipleTracks() throws { + let a = try Self.audioDescription(formatID: kAudioFormatMPEG4AAC, sampleRate: 44_100, channels: 2) + let b = try Self.audioDescription(formatID: kAudioFormatAC3, sampleRate: 48_000, channels: 6) + let readings = [ + try #require(RemoteHLSStreamDescription.audioReading(from: a, isEnabled: false, language: "und")), + try #require(RemoteHLSStreamDescription.audioReading(from: b, isEnabled: true, language: "deu")), + ] + let (tracks, active) = RemoteHLSStreamDescription.audioTracks(readings) + #expect(tracks.map(\.id) == [RemoteHLSStreamDescription.audioTrackIDBase, RemoteHLSStreamDescription.audioTrackIDBase + 1]) + #expect(active == RemoteHLSStreamDescription.audioTrackIDBase + 1) + #expect(tracks[0].language == nil) + #expect(tracks[0].name == "Track 1 (aac)") + #expect(tracks[0].isDefault == false) + #expect(tracks[1].name == "DEU (ac3)") + } + + @Test("no enabled track means no active index") + func nothingEnabled() throws { + let a = try Self.audioDescription(formatID: kAudioFormatMPEG4AAC, sampleRate: 48_000, channels: 2) + let reading = try #require(RemoteHLSStreamDescription.audioReading(from: a, isEnabled: false, language: nil)) + #expect(RemoteHLSStreamDescription.audioTracks([reading]).activeID == nil) + #expect(RemoteHLSStreamDescription.audioTracks([]).tracks.isEmpty) + } + + @Test("a video description is not an audio one") + func videoIsNotAudio() throws { + let desc = try Self.videoDescription(subType: Self.avc1, width: 16, height: 16, extensions: [:]) + #expect(RemoteHLSStreamDescription.audioReading(from: desc, isEnabled: true, language: nil) == nil) + } +} diff --git a/Tests/AetherEngineTests/RemoteHLSTelemetryTests.swift b/Tests/AetherEngineTests/RemoteHLSTelemetryTests.swift new file mode 100644 index 00000000..308a80eb --- /dev/null +++ b/Tests/AetherEngineTests/RemoteHLSTelemetryTests.swift @@ -0,0 +1,53 @@ +import Foundation +import Testing +@testable import AetherEngine + +/// On the `nativeRemoteHLS` bypass the sampler never ran, because its bitrate halves read the loopback's +/// demuxer counter, and so `liveTelemetry` stayed nil for the whole session. AVPlayer's own access log is +/// there on this route too; these pin how it fills the same two bitrate fields. +/// +/// The fields name the stream's bitrate, so on the bypass they come from what the variant declares, not +/// from what AVPlayer transferred: a player filling its buffer after a start or a seek pulls at link +/// speed, and a transfer-based figure read 22.6 Mbps for a 3.7 Mbps Jellyfin transcode (device, 2026-09-28). +/// The transfer stays in the network fields, where it belongs. +@Suite("RemoteHLSTelemetry") +struct RemoteHLSTelemetryTests { + + @Test("the bypass reads the variant's declared bitrate, every other route the demuxer") + func counterPerRoute() { + #expect(LiveTelemetrySampler.bitrateCounter(for: .remoteBypass) == .declaredVariant) + #expect(LiveTelemetrySampler.bitrateCounter(for: .loopback) == .demuxer) + #expect(LiveTelemetrySampler.bitrateCounter(for: .software) == .demuxer) + #expect(LiveTelemetrySampler.bitrateCounter(for: .none) == .demuxer) + } + + @Test("the loopback-only readings stay nil on the bypass") + func loopbackOnlyReadings() { + #expect(LiveTelemetrySampler.readsLoopbackPipeline(.remoteBypass) == false) + #expect(LiveTelemetrySampler.readsLoopbackPipeline(.loopback)) + } + + @Test("instant is BANDWIDTH, average is AVERAGE-BANDWIDTH") + func bothDeclared() { + let rates = LiveTelemetrySampler.declaredBitrates(indicated: 4_000_000, indicatedAverage: 3_700_000) + #expect(rates.instant == 4.0) + #expect(rates.average == 3.7) + } + + /// AVERAGE-BANDWIDTH is optional in a master; without it the peak is the only declaration there is. + @Test("a master without AVERAGE-BANDWIDTH reports the peak for both") + func averageFallsBackToPeak() { + let rates = LiveTelemetrySampler.declaredBitrates(indicated: 4_000_000, indicatedAverage: 0) + #expect(rates.instant == 4.0) + #expect(rates.average == 4.0) + } + + /// The access log reports an unknown bitrate as a negative number or zero. + @Test("nothing declared is nil, not a measured zero") + func nothingDeclared() { + let rates = LiveTelemetrySampler.declaredBitrates(indicated: -1, indicatedAverage: -1) + #expect(rates.instant == nil) + #expect(rates.average == nil) + #expect(LiveTelemetrySampler.declaredBitrates(indicated: .nan, indicatedAverage: .infinity).instant == nil) + } +} diff --git a/docs/api.md b/docs/api.md index 5304f034..9760f6f3 100644 --- a/docs/api.md +++ b/docs/api.md @@ -604,7 +604,7 @@ suppressing `AVPlayerItemLegibleOutput` to keep the measurement running. | `$dolbyVisionConversion` | A `DolbyVisionConversion?`: the Dolby Vision profile rewrite applied to the served stream, nil when the session serves the source's own profile or no Dolby Vision. `.profile7ToProfile81` is a Profile 7 source played on a display presenting Dolby Vision, where `$videoFormat` reads `.dolbyVision` and `$sourceDVProfile` keeps 7, so a label can say "P7 -> P8.1". The enhancement layer is dropped by that conversion. Set when the loopback session starts, cleared on stop. | | `$sourceVideoCodecName` | The source video codec in the libavcodec spelling ("hevc", "h264", "av1"), nil when the source carries no video. The probe-free remote-HLS bypass maps it back from the item's video sample type, so the field answers on every route rather than going quiet on one of them. Not the same question as `$activeVideoDecoder`: a codec has several decoders, and which one runs depends on the hardware. | | `$sourceContainerFormat` | The container libavformat opened ("matroska,webm", "mpegts"), nil on the remote-HLS bypass, where AVFoundation opens the source and there is no libav context to ask. This is the container that ARRIVED, which on a remux or transcode session is not the one a host's library metadata describes. | -| `$sourceVideoStreamFormat` | A `VideoStreamFormat?` (AE#658): the source video stream's `pixelFormat` ("yuv420p10le"), `bitDepth`, `colorPrimaries`, `transfer`, `matrix`, `range` and `profile` ("Main 10"), in libav's names as the container and the probe's decoder declared them. A field the stream leaves unspecified is nil, never a guessed BT.709. `colorPrimariesLabel`, `transferLabel`, `matrixLabel` and `rangeLabel` give the names a viewer reads ("BT.2020", "PQ (SMPTE ST 2084)", "BT.2020 NCL", "Limited"). nil before load, without video, and on the probe-free remote-HLS bypass. | +| `$sourceVideoStreamFormat` | A `VideoStreamFormat?` (AE#658): the source video stream's `pixelFormat` ("yuv420p10le"), `bitDepth`, `colorPrimaries`, `transfer`, `matrix`, `range` and `profile` ("Main 10"), in libav's names as the container and the probe's decoder declared them. A field the stream leaves unspecified is nil, never a guessed BT.709. `colorPrimariesLabel`, `transferLabel`, `matrixLabel` and `rangeLabel` give the names a viewer reads ("BT.2020", "PQ (SMPTE ST 2084)", "BT.2020 NCL", "Limited"). nil before load and without video. On the probe-free remote-HLS bypass it is the DELIVERED stream, read back from AVPlayer's item video track once it resolves: colour and range from the format description, `profile`, `bitDepth` and `pixelFormat` from its avcC / hvcC record (nil where the codec has none of those, AV1 and VP9 included). Under a server-side transcode that is the transcode, not the library's file, which is the point of reading it there. | | `$decodedVideoFormat` | A `DecodedVideoFormat?` (AE#658): what the engine's own decoder actually produced. `frame` is a `VideoStreamFormat` for the decoded picture (libavcodec's output pixel format, or the libav name of the VideoToolbox buffer on the software host's hardware decoder), `pixelBufferFormat` the CoreVideo buffer it was displayed from (`420v`, `x420`; `pixelBufferLabel` reads "P010 (x420)"). Republished when either changes mid-stream. **nil on every native route**: AVPlayer decodes there and its frames never pass through the engine, so a panel shows `$sourceVideoStreamFormat` and names AVPlayer as the decoder instead of inferring a pixel format it never saw. | | `$activeVideoDecoder`, `$activeAudioDecoder` | The decoder names actually in use, for a stats overlay. This is the honest "what is decoding this" surface; `playbackBackend` is not. Show `$activeAudioDecoder`, classify on `$audioDelivery`: a label cannot separate a source without audio from a source whose audio was dropped. | | `$metadata` | `MediaMetadata` parsed at load (title / artist / album / cover). | @@ -632,9 +632,9 @@ suppressing `AVPlayerItemLegibleOutput` to keep the measurement running. | Symbol | Notes | | --- | --- | -| `$audioTracks` | `[TrackInfo]`. Republished when `LoadOptions.confirmAtmos` confirms a track. | -| `$activeAudioTrackIndex` | The selected track's id. | -| `selectAudioTrack(index:)` | Session-preserving reload, roughly 0.5 to 1 s of black. `index` is `TrackInfo.id`. A no-op when out of range, already active, or on a forward-only custom source (live ingest included), which cannot rebuild its pipeline: there a track change is a fresh `load` naming the stream. Every refusal is logged, so a picker that does nothing is explainable. | +| `$audioTracks` | `[TrackInfo]`. Republished when `LoadOptions.confirmAtmos` confirms a track. On `.remoteBypass` it lists the audio tracks AVPlayer built for the item (normally just the one playing), read from their format descriptions: `codec` in libavcodec's spelling (`eac3`, `ac3`, `aac`), `channels`, `sampleRate`, `profile` (AAC's object type; "Dolby Digital Plus + Dolby Atmos" with `isAtmos` when the E-AC-3 dec3 record announces JOC) and `language` from the track or the selected audible option. The ids are synthetic (from 400000 up) because there is no stream index on that route; compare them only with `activeAudioTrackIndex`. | +| `$activeAudioTrackIndex` | The selected track's id. On `.remoteBypass`, the enabled item track's. | +| `selectAudioTrack(index:)` | Session-preserving reload, roughly 0.5 to 1 s of black. `index` is `TrackInfo.id`. A no-op when out of range, already active, or on a forward-only custom source (live ingest included), which cannot rebuild its pipeline: there a track change is a fresh `load` naming the stream. Every refusal is logged, so a picker that does nothing is explainable. Also a no-op on `.remoteBypass`, where AVPlayer owns the audio selection and the list above is informational; a different language there is a different URL. | | `installAudioTap()`, `removeAudioTap()`, `audioTapHasDeliverySource`, `AetherEngine.audioTapFormat` | Opt-in decoded PCM, mono Float32 48 kHz with source-PTS stamps, off the render path. See the contract above. | ## Subtitles @@ -860,7 +860,7 @@ say so on the tracker rather than working around it. | `nativePlayerLayer` | The engine's own `AVPlayerLayer`, for a host building `AVPictureInPictureController` around a layer rather than around `currentAVPlayer`. | | `$softwarePiPSource` | `SoftwarePiPSource` for sample-buffer PiP on the software path: the display layer plus transport answers on the enqueued frames' axis. iOS only in practice; tvOS AVKit does not evaluate sample-buffer content sources (FB9751461). | | `$softwareDisplaySize` | The rectangle the software path's picture presents at: coded size under the decoder's pixel aspect. Mirrored, not latched, so a mid-stream resolution change re-shapes it. nil on every other path. | -| `sourceVideoWidth`, `sourceVideoHeight`, `sourceVideoPixelAspectRatio` | The source's CODED size and the multiplier that turns it into the presented one (`width * ratio`), read once from the probe. The ratio is 1 on square pixels and on a declared ratio the engine refuses (#290), never a guess; on the paths that draw, prefer what is on screen (`softwareDisplaySize`, `AVPlayerLayer.videoRect`) over recomputing it here. | +| `sourceVideoWidth`, `sourceVideoHeight`, `sourceVideoPixelAspectRatio` | The source's CODED size and the multiplier that turns it into the presented one (`width * ratio`), read once from the probe (on `.remoteBypass`, from AVPlayer's item video track once it resolves: the delivered size). The ratio is 1 on square pixels and on a declared ratio the engine refuses (#290), never a guess; on the paths that draw, prefer what is on screen (`softwareDisplaySize`, `AVPlayerLayer.videoRect`) over recomputing it here. | | `pictureInPictureActive` | Host-set. Keeps the pipeline and the loopback server alive across a background transition, and keeps the software path decoding video for the window. | | `backgroundPlaybackEnabled`, `backgroundTeardownGraceSeconds` | Background audio policy; the grace window (15 s default) is what lets a paused session survive a quick app switch. | | `presentationAxisMap` | `PresentationAxisMap`: `sourceSeconds(forItemSeconds:)`, `itemSeconds(forSourceSeconds:)`, `shiftSeconds(atItemSeconds:)`, `seams` (each a `PresentationAxisMap.Seam` of `itemSeconds` and `shiftSeconds`), `isEmpty`. Readable off the main actor. Cue times and `sourceTime` live on the source axis; `AVPlayerItem.currentTime()` lives on the item axis, and they differ by the producer shift. Returns nil rather than a guess where no axis is established. | @@ -925,7 +925,7 @@ as well. | Symbol | Notes | | --- | --- | | `AetherEngine.version` | The engine release this source descends from, as the string a published tag carries. SwiftPM resolves a package to a revision rather than to a tag, so an About panel or the header of a diagnostic log has nothing else to name the engine with. Between releases, and under a pin on an unreleased commit, it names the last published version the checkout descends from. | -| `diagnostics.liveTelemetry` | 1 Hz `LiveTelemetry?` snapshot while playing or paused, nil while idle. On a separate `ObservableObject` so its ticks cannot re-render a host observing the engine. | +| `diagnostics.liveTelemetry` | 1 Hz `LiveTelemetry?` snapshot while playing or paused, nil while idle. On a separate `ObservableObject` so its ticks cannot re-render a host observing the engine. On `.remoteBypass` it is fed from AVPlayer's access log alone: `instantBitrateMbps` and `averageBitrateMbps` are what the playing variant declares (BANDWIDTH, and AVERAGE-BANDWIDTH or BANDWIDTH where the master omits it), because the bytes AVPlayer transferred are buffer fill at link speed rather than the stream's rate, `networkThroughputMbps` is the access log's `observedBitrate`, `networkTransferredBytes` and `droppedFrameCount` its session totals, `forwardBufferSeconds` the loaded range ahead of the playhead. `avSyncGapMs` and `observedFps` are nil, and the loopback counters (`producerRestartCount`, `muxedBytesLifetime`, `serverBytesSentLifetime`, `serverRequestCount`, `demuxerBytesFetched`) read 0 because there is no loopback. | | `LiveTelemetry.softwareCacheSeekHits`, `softwareCacheSeekMisses`, `softwareCacheSourceEpoch` | Optional cumulative software-VOD packet-cache counters. A hit repositions the retained consumer cursor without changing the source epoch; a miss repositions the demuxer and advances it. `nil` on other paths. `cachedBytes` includes retained compressed packet records on software VOD, distinct from decoded `displayCushionSeconds` and the underlying byte-reader window. | | `EngineLog.handler` | Mirror every info-level line into a host capture path. Fires from whatever thread emitted it, so it must be thread-safe and non-blocking. | | `EngineLog.subsystem`, `EngineLog.Category` | `de.superuser404.AetherEngine`, one category per subsystem: `engine`, `ffmpeg`, `session`, `muxer`, `demux`, `hls.server`, `audio.bridge`, `sw.playback`, `scrub`. | diff --git a/docs/architecture.md b/docs/architecture.md index 3acb9764..d0fb8ed2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -432,6 +432,7 @@ Sources/AetherEngine/ │ ├── AirPlayPlaylistDecision.swift Pure playlist choice for the wireless-AirPlay loopback rewrite: which of master / media a receiver is handed, kept separate and testable offline like `MasterFallbackDecision` (#86, #227) │ ├── NativeLegibleDeselectPin.swift Keeps AVKit's ready-time legible auto-select from engaging a rendition the host never asked for (Sodalite#38), and publishes the caption requests the system makes on its own as `systemCaptionRequest`, by BCP 47 language rather than by ordinal (Sodalite#65) │ ├── RemoteHLSFormatDetection.swift Dynamic-range classification for the `nativeRemoteHLS` bypass, read back from AVPlayer's own track format instead of a second origin probe, which that path exists to avoid (#168) +│ ├── RemoteHLSStreamDescription.swift The rest of that read-back for stats panels: delivered dimensions, colour description and avcC/hvcC profile as a `VideoStreamFormat`, and the item's audio tracks as `TrackInfo` (codec, channels, rate, dec3 JOC) │ ├── RemoteHLSIngestFallback.swift Reroutes a bypass session onto the live-ingest loopback when the master advertises HEVC / DV / AV1 but delivers MPEG-TS, carriage AVFoundation builds no video track for (#168) │ ├── RemoteHLSMasterRewrite.swift Declares host-supplied sidecar subtitles as legible renditions on the bypass by rewriting the origin master, since media selection on an HLS asset comes from the playlist and nowhere else (#316) │ ├── RemoteHLSSubtitleProxy.swift Stands a loopback origin in front of that remote master: two playlist GETs, a rewrite and a socket, and anything that does not line up keeps the plain origin URL (#316)