From 611c7e38ee0cdcb56a982fc9e61cfac68e57d98d Mon Sep 17 00:00:00 2001 From: Chrizz Gelbach Date: Mon, 16 Nov 2026 20:25:47 +0000 Subject: [PATCH] Detect a non-optimized MP4 from the mdat header A non-optimized MP4 (moov after mdat) was only recognized once the whole mdat had been received, so a remote stream downloaded almost the entire file before fetching the moov and streaming the file a second time. The mdat header already gives the moov offset. --- .../Audio Source/Mp4/Mp4Restructure.swift | 9 +- .../Mp4 Restructure/Mp4RestructureTests.swift | 106 ++++++++++++++++++ .../Streaming/Mp4 Restructure/moov-at-end.m4a | Bin 0 -> 8048 bytes .../Mp4 Restructure/moov-at-start.m4a | Bin 0 -> 8048 bytes Package.swift | 4 +- 5 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 AudioStreamingTests/Streaming/Mp4 Restructure/Mp4RestructureTests.swift create mode 100644 AudioStreamingTests/Streaming/Mp4 Restructure/moov-at-end.m4a create mode 100644 AudioStreamingTests/Streaming/Mp4 Restructure/moov-at-start.m4a diff --git a/AudioStreaming/Streaming/Audio Source/Mp4/Mp4Restructure.swift b/AudioStreaming/Streaming/Audio Source/Mp4/Mp4Restructure.swift index 0b7df80..18593f1 100644 --- a/AudioStreaming/Streaming/Audio Source/Mp4/Mp4Restructure.swift +++ b/AudioStreaming/Streaming/Audio Source/Mp4/Mp4Restructure.swift @@ -211,10 +211,17 @@ final class Mp4Restructure { foundMoov = true case Atoms.mdat: foundMdat = true + // The header gives the mdat size, so the moov offset is known without + // receiving the whole mdat (usually almost the whole file) + if ftyp != nil, !foundMoov { + atoms.append(MP4Atom(type: atomType, size: atomSize, offset: atomOffset)) + Logger.debug("🕵️ detected a non-optimized mp4 (header-only observation)", category: .generic) + return .needsRestructure(moovOffset: atomOffset + atomSize) + } default: break } - + if ftyp != nil { if foundMoov && !foundMdat { Logger.debug("🕵️ detected an optimized mp4 (header-only observation)", category: .generic) diff --git a/AudioStreamingTests/Streaming/Mp4 Restructure/Mp4RestructureTests.swift b/AudioStreamingTests/Streaming/Mp4 Restructure/Mp4RestructureTests.swift new file mode 100644 index 0000000..da9258b --- /dev/null +++ b/AudioStreamingTests/Streaming/Mp4 Restructure/Mp4RestructureTests.swift @@ -0,0 +1,106 @@ +// +// Mp4RestructureTests.swift +// AudioStreamingTests +// + +import AVFoundation +import XCTest +@testable import AudioStreaming + +class Mp4RestructureTests: XCTestCase { + // moov-at-start.m4a: 1 s AAC from afconvert, ftyp (0, 28), moov (28, 974), free (1002, 3086), mdat (4088, 3960) + // moov-at-end.m4a: the same file with the moov moved behind the mdat and the chunk offsets adjusted, + // ftyp (0, 28), free (28, 3086), mdat (3114, 3960), moov (7074, 974) + private let mdatOffset = 3114 + private let moovOffset = 7074 + + private func fixtureData(_ name: String) throws -> Data { + let url = try XCTUnwrap(Bundle.module.url(forResource: name, withExtension: "m4a")) + return try Data(contentsOf: url) + } + + func test_Non_Optimized_Mp4_Is_Detected_From_The_Mdat_Header() throws { + let data = try fixtureData("moov-at-end") + let restructure = Mp4Restructure() + + // Only the start of the mdat atom has been received, as with a stream that just started + let result = try restructure.checkIsOptimized(data: data[0 ..< mdatOffset + 100]) + + XCTAssertEqual(result, .needsRestructure(moovOffset: moovOffset)) + } + + func test_Non_Optimized_Mp4_Waits_For_The_Mdat_Header() throws { + let data = try fixtureData("moov-at-end") + let restructure = Mp4Restructure() + + XCTAssertEqual(try restructure.checkIsOptimized(data: data[0 ..< mdatOffset + 4]), .undetermined) + XCTAssertEqual( + try restructure.checkIsOptimized(data: data[0 ..< mdatOffset + 8]), + .needsRestructure(moovOffset: moovOffset) + ) + } + + func test_Optimized_Mp4_Is_Detected_From_The_Moov_Header() throws { + let data = try fixtureData("moov-at-start") + let restructure = Mp4Restructure() + + XCTAssertEqual(try restructure.checkIsOptimized(data: data[0 ..< 100]), .optimized) + } + + func test_Restructured_Mp4_Detected_Early_Is_Parsed_Without_Error() throws { + let data = try fixtureData("moov-at-end") + let restructure = Mp4Restructure() + XCTAssertEqual( + try restructure.checkIsOptimized(data: data[0 ..< mdatOffset + 100]), + .needsRestructure(moovOffset: moovOffset) + ) + + // RemoteMp4Restructure fetches the moov from its offset and streams again from mdatOffset + let (initialData, streamOffset) = try restructure.restructureMoov(data: Data(data[moovOffset...])) + let restructured = initialData + data[streamOffset ..< moovOffset] + + let outputFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 44100, channels: 2, interleaved: true)! + let playerContext = AudioPlayerContext() + let rendererContext = AudioRendererContext(configuration: .default, outputAudioFormat: outputFormat) + defer { rendererContext.clean() } + let processor = AudioFileStreamProcessor(playerContext: playerContext, + rendererContext: rendererContext, + outputAudioFormat: outputFormat.basicStreamDescription) + var raisedErrors: [AudioPlayerError] = [] + processor.fileStreamCallback = { effect in + if case let .raiseError(error) = effect { raisedErrors.append(error) } + } + let entry = AudioEntry(source: RestructureTestSource(length: restructured.count), + entryId: AudioEntryId(id: "entry"), + outputAudioFormat: outputFormat) + playerContext.audioReadingEntry = entry + + XCTAssertEqual(processor.openFileStream(with: kAudioFileM4AType), noErr) + defer { processor.closeFileStreamIfNeeded() } + var offset = 0 + while offset < restructured.count { + let chunk = restructured[offset ..< min(offset + 1024, restructured.count)] + XCTAssertEqual(processor.parseFileStreamBytes(data: chunk), noErr, "chunk at \(offset)") + offset += chunk.count + } + XCTAssertEqual(processor.flushRemainingPackets(), noErr) + + XCTAssertTrue(raisedErrors.isEmpty) + XCTAssertEqual(entry.audioStreamState.dataByteCount, 3952) + } +} + +private final class RestructureTestSource: CoreAudioStreamSource { + let position = 0 + let length: Int + weak var delegate: AudioStreamSourceDelegate? + let audioFileHint: AudioFileTypeID = kAudioFileM4AType + let underlyingQueue = DispatchQueue(label: "mp4-restructure-tests") + + init(length: Int) { self.length = length } + + func close() {} + func suspend() {} + func resume() {} + func seek(at _: Int) {} +} diff --git a/AudioStreamingTests/Streaming/Mp4 Restructure/moov-at-end.m4a b/AudioStreamingTests/Streaming/Mp4 Restructure/moov-at-end.m4a new file mode 100644 index 0000000000000000000000000000000000000000..9baf5656fa5bf8b11f63856959a708e813377774 GIT binary patch literal 8048 zcmeHMdpwle*WYt7?!}BmQNti6NrRja(=hI8esYS67{(ao(io&pj7t|vM^Y}iCLPtO zTsqOkF$Y~0l9J??QmIgh>QtC_@0oOddOz>y^M2lcUtiC9_S$Rj=UIENwf5Tk82|vS zU_o4@hxrOJfIwwfq&bZj6%hu2(u`m}hlAwPn6|*Q1*R=9ZGmYEOj}^u0@D_lw!pLn z{(o6u=GL&F0D%q}M-S1`@pvl4qUqy>J3=ezi`uRm^^$hgXUFjqCVmmuclt(Ob6PGQ zU!3syOZ?)s?w3wU!Oqtwg1g(UB<5>q(B7F^42VRX)da%N1FggU5qtgW zwq}wyb>{Q`IPddfV!&xkc3fQFi^d4Re$K1DTZ#1ia+O}4O|9>cNWNoH7J-3P#fw%L z$~ly%f}B5N>)wc8B-~DOxNqq9@q(u5)>iT_+lCvz;AzkAS06{PjN z$3-gmhh{(l$k4&H7ZrqG$D}%EK%HrDS`Ot*GPIThHqdrQ{6_JIvfMJZA;3d5$rr|e zv$M0y?NL31UdThNOeGQ_@7fUFbACbIY|>IjFfTgo^5^!73g>-SgxED;F2^5x&KsN{ zdwYW!wr$X3bL=F+0l-gCX_D%7C40jwQl)D~`mgmDi3i)wm>K}1W9F{BF@#9+461e~ zMnct%iHjIn&u)08HfSE#ettMObPYq_H2MC9O2ejY<53imJ4t^oAReMCbO1vZh6QZi z6$6E-cHN?SU72wNIGR!{>x<@xi^b`sAP00Z)o`gp26?>W=-O=Xsv?h41RA0#xVS5KDjkz8li9+s$}<*DjGR>-N-#c$IoM5K0g+Z?FcN; zpuW=Ht)!$RTl?05ZaOMI>y*JfzG$ApVIZ5!b`~18JOV+*fU5_ZfK1$d7#qs-|CTxh z#Tt+3tVk&rC9zae8}Ly_zHT3B6_4@QIV$sw`tui$?N@-|70-hqcF$(4dqLs0;KI0; z6D_W*|LqMx<#JEPG3C>AnUi3aA-D+$Vk_a4Q76xERbZyk9a$o|x zc0u8hve^?(2mSQ7YG`Db_C66w(U=HvA)|NG)wh1dZnRb17iz1u{m7fIS)Gkn&zL0? z$+WQ)vnoLAsz=~l7BrM5jBOF9{V~iGIEG9zMMLG**KMEr#S`5taRh1yFom$KiI0&& zpe`*9V6n}&pe%qZK|lc4iTh&k_ZNdNj18S%yl~;z+}vA+&AvDod-0QE7BK9l%V-(0 zYBgN|O(;8TFvXZj(3lQ6LWVyyrY=UasAzt+_|t3RVhvZ-2LuS&g|8f+XKBn`jgvr^ zKwujEG2@I>B&^zh!roF_lBy@Em`z9aHpP$WoH>(##<==lm$-EWdNwB34t-D zR4nx#P6*wH;=JRFz0zBRV8 zFgp6<^GD7ASm|>_7|jj?lD@zE&s@0lsc?O{1KXAK9|C*(oF;jP6F}k%{Q9BR){|nf zuU>Xk^8=%DIg1v+Da+#Y;z?E48j z@rN5G4oePXP%0!ULXDVjh0<`7+@1b|=dyQ~bv?-K>gsBPx9-wqBztQj$(l%1P$a5^ zaph!}@d|*0#%QKUSBf_y5_u+WI?eOg4L`nz%ZX{ViF&`7?|S z9&CLncjeED_vw=kt!sXTt0lcu?PU3w8DGq+p-Mq_s;L_ZS_M*{LnF zWU;_Q-i>XPcbM7>&j@k1PXZgcI|LCXwHy=(ajImv z?)qPQ{*{IP^(`hk(=hmA9TMEgqU~(!9TUDZx8E`l?H3(iUPI>Vn&XObsy9_Q zo~bB~M_LE$1|Gryk6vXnQW)3Q8+GQWQCx}%i1#wr2b}m`f)$VvV(Gvs=SLsHAePgi zIaA>}M8})FI`O>AcH?*VKE-(NWPu@H>$>i`rl8>AziQm!V#!bMYB?Wca3=3nkH@)A z4?&%U#DhnwtMk*>Ta%!tN!C`F%-KX7&C{0GpLgQ5!`Yji)I98Loh{pizv)Edb53-C z-wy*lmfT!A=o@0eaZQQry>L9iX>@pOP#GzTXzmoBew3|Wa$X-R%ME|?=i5u|j=Mb5 zZ}xqUDa7EW)JP<2#buYEh-AI2UY5Ot7Qkk3xfn~XzyDzJxxe-_?{k=Yla5V{^gWy- zR9*@cG65IhTJD28VxhzX7}4YklNTlFqZJnM{H=Jn*)4s>6Vtf>Z6tT}%M0xH)%>%n zc$#oMeaAz`m>6~V7Zr1$K)a}8XuS;zc|5yC1j1W8b=M^YiLkvGdaY$8`yqz;(o4A* zY=I{V>eS4;4K6Qc1w74=QsnM$f7|(+_*ANS65q%OD8G+5=w@7Iv2JjqaaFT%d1Cre z2d!{c5MA_8Lka~l&m(#5;SG1@z9e5=QFgU7u>WNx)}3MpFi|j|ocBV_+0-iOX7@e$?#`KP?nptgtvq@i5f&r`&)i7q%u$WyCQ1cXdhq-z z%E&CSbWIs3F=8+rQl9DQm))n1^smh~CQ_F8gF`|>mp3@A4t=1m4sgGo`_bJNC8_ot z9eK1lSC4*;4=d8%zNLL4+nC+7Sb5)uy{Grk@eU65%XjKhpS^h^-xVOUPA9F=x7y{w z^}gZZk2)zv{TIB_cQ}^DIM>ZL`nNIh*vF4PR{~?l>)wjw7}Tt|xY6tc&lp$V5pJKK z&&GD&agVO!Hs`o~yCfBt{IbW)+&@ehzbu?L znE4^B{F77C)T&)_2g$j zMudBlFT&8j?@Qk%!bEAL$2LBNsIX_v zbF(9YdkcN2N;fD=`5b3LTI-=YZBlM&+r1q>+;dO9%)hO5TRYTja92OH2gc4c5j(R8}0CvFI9sn?s3pGgr>r?PUoj#@Uzg6IeJ=R~q57-1- zHV8IJbjtpt6KO+$djA6w;SYKydro$vPco7cGZq%a3xF&+Ea-ojm$bibNYG~fvtsp)JwIiT0Dn;`*lchW+CQ88G40%{aNvtcC zgJAf7P&7J(f98b+DoB9r4q?WL3W`DuPAC-W!V(WlHM$zmfIQz2fGO0dKX0xU4- z7n7M$Y=K0L%aWzAsu3j+NZtT>qy@eN4+~%jM-zw`2=o|gLz+>EgET42hyAv}`Xu1< zfB@+tqBYN z>`%cDefpHa|JFeRfc6&(0ye>s4Z=+_or>ReBH9o@z5fB3@CQAUcTV0$pJZe$YAig6 z9{^cmc+meaFZuljp+TDklaEG6#zai=lP#(`oF5Skdm7Q<@;7}OF$|K+^@0REWE=rv zqgsq$Gf~c?VxxuJ&5(y}w9LD5IS7UkK&(UXeO_pwfd~-4Ls;;lgQ5`zBNB-WV2g#V z7F`SIKwdxr;0sWI5;Q&&2OAW0ESMS170T4OE?Ek@I?+O*>~fZGh)6r z%;R|k$b5+j5(dEbeK=3nLKaMkGt3P^KvWbGneS-kZ;nQ0a(NL1Z1uu<0kUiej0`tJ z?&g{Cy<;MHh{akF0pUF4QAD!Zlg)r^YEXbo8{$PkH2(%U^OL2yoS`{>W4;YrJDGnM z{O}OG<^LpssW&57z?0d9*f5Q04@`St+5^)bnD)T52c|tR?SW|zOncz}j|XOM4ToPz zNJI=hRFCBORD^u316a|HuqyhZ_Uk4R!mfs#ctPUCFVcoCPRuoz<h|M>O@W(y9(Luvd-%qtZ(VtU;cp5|t$)cr9I@3Ns_8fQ zs;bWK`Rcb$LPQOAy*?4#(|#qXKu3r6&fIE1Ebgko;eH-y8}^Uf>sP-ui@2$)K=8+T z-xm`DE@O)0(gq1)BLMq(uljB!(F-axB&1Dk?+}r4=b~&J1FeY_ub?P7m1=_AKjZ4( zNM9u0PItOb@%wl|*L>?M02Gg^sQCTi);RRqH`J_jmrlrpJ-T#ve&R^Kw|PZ>X^dF$ z99S6Ix$J5JCk{A<0xHl3zB}&z9Tf`DVlyBe6(P#ZoIU2*&;jZVNwrOa2$YT)X3j~a ziBiQRe5z9Pz|cU$eCa*?CJiOCf#t!b3rD0_3^~@emeor>)N}NM?pm@?G0z)pWgmNr zP8TUJ1==ORI}=?3;)6gLCb1KFC(t9QHpgpTS0yWT8W1(Iu{@M@-scY^4${|ebS_pL zNxJ^UZbbZbn@iD1{F}I<`2M1PNbzrKi$;R$6D)7h!93<{fbz$CZ4jx)KQaVWbOjOS z?<}Ay3YCgw8h`|Uab&jFdF@Pwa@iCLQPFs+fHw#k;RYw_@z>Ria2n3`Rt% z486)S;Mcrlv5wR_-Ch5l?{2fQ|JPY2d-(${c_(wNn7@0{36+HP{Kv%_*oPKC1t`$L zwHKARU&rJ+XF#3la9U0kOd|A_6FSg-N5V$whw{8~E(Ktrnv@G;fW>0D-X1kXAb~u@ z$}~J4@@|b0z2_I?&n7Ho1oLCkFMsZ+tYq!GB0{eLb9w&gb3Wh%(Z>hOuy2PEo8ur$ z4gh|FN|QCOt2t7x$d#@c>A%)rEFJ8yVCn!A$-+Z@VIN3HFR>Z22p!N&Y#D=e10q)+Ywl(Lw%*cTTM+(vF@!C z-F#Ge)+yt80`WYR!$2{Q%MwvqAAz6}5Ml_LfkOO!m>cTz|CR>@$u&NnrAn?4C$lxu z8nMwwzHT3BlaBGZxf=6L`U@71?N@>6mB5E7cF%6CXF<`n;G+1}6RmEm{|$e*sao#E zIHrD@u5c2}qJWzKC$$kxu{vdjn+h|X?#$NsFfi029T#3wjSHX)^b3oQl+T`UIp}A! zRYxb&T=GOLM^h5Sg-pq&t8e{E+-a+NF4WcN`w=%^vpJifojFS+R%mCdW>7d_hXu=@F+|oSx4j6*KMErr4v0XF*s@`Kz?4auT6-RL#QopjbXEmQ&=7l zB11qxh)c+e!QWpDzA&Y*Uc7MS+1=b*j?TF_7^fanKocnr8&7d& zGIXX>u883eooRs5EiRs)BmMLmzgWjj^8pS*ap5cH=h-@QS7T%_z~PuCf6O?e7OAm$ zMMesh6S?B$a!Hux+|osl19uvlS$CDw6?{a>PubuET>*dwnh2Pp%B1q>VMH)|NM1Nn zJm@QQG@PEGm)#!w`)izZ{7m~5tx9#%>XeN=?q;+WlM6g+lMDZHU%cA%kC*Q0&PLW^X$iAXUt@DAEQ{npZ0QM{CKP2|{InByWCx9##*!4qgZ6~Etj$sbYTtjWNYh@pq zNOs`)oAqAABh{vYwTh%n#X}H;NPj_}3B&D>8y>%gkeB~;WXry%r(@zT-{gbsr93en z4zI;PD>7Ck}G8@ z!c3TNMe=l$U0wcz58>`E?|zWi-QC>|FFj?;2#&URf-N4eqKelD4^dKFlHWo2Wmc32 z`e&faW25f_^Jo)+V&sVo3$cnlwDh!+{Cy$)#%qmF zt6a^ii0G;-aK9`8Q2>T}_}G+iN5gPCnbWDVs>R+T@^rBKdsObcZ&Vto3@Su^~7V7HV$-%x&$!nK1?J+r~uv1@Q$zq|WvOCu#*`xXk z&@+Su2kQz@&nthd_uf$D3UgL48!43RB>lB#+gct=$Ism@s1~7a&+B;X^XRW6@hbMx zow}>%<@2+M;=wtTvZLu%cbNJM&xkO$PXaroJ2){atpXH^Fq%ZT?)qQ*U|c5T@l}NJ z<&@xAG!(XkM1UJvjDvlHbK;klj$6j!{o>=xYl#8_OH2ty^QH#RD-Frxk+uPcfrl`` zV^-OX6vg*RqR$*PiBB~H3Eq}QfR`Y_*#HF*nhsoYe~ckaVkHudMTP4SIo{;gOXpp7 z7{9ysDavOj8w_!5>wD^(gMx?us`Y@2r68lb^?a=HnfzD1p69weh4ogl2p*}aDacrF zOMsCk*xI17X5%q5FMED}{)yL4XK!{<^UKcFA^=&f#2<`cj~h1%L9#wB84I zq#{`aP~yoIrYuf2LRMJh^S9FB7Wa%9Pt4~6WFxs_R8i=-ulD<@;_Jfo^ba07$Hr>I zzv$Qlh5E&vL+kC3l*e*g#UP@s%V1q{kQgmN(d(?MxDQdxm)`0nU<*7^P^VVjZE$%p zE8%H|kg9Zl``fPHq^Hs>lLaOwK>dB>L3h(~t964LO{-f>E0QvfI_X8QgXrRiI&!EG z`JO3j4{x|L_a*V_it?*oe`cpal*GW+VEmTPl%@X?34z-JPW; z1lbp7$Cm>f^Ns~SDy}dJ#TViBz<VXYeOOAPTq=YiCF@6)$kh9YFmt3Nn z(F$hF-mAA&{5>7n#r^J0dTKs8`aMUlq$C}!na_Hy5j^A(;iEX@t_HuWE3(mQdRujp==vMLTIM5n%PLja$4% z+GuCGwXNV;1o>RPIXwSr>LIG}P)BV<@1@vR+iMx?E_%CHWjiE?J=*ZUI|joVsT7=# zoHgeUgcuBGCKaE(batfcGmS>OfXmgoexvXk(h3VXlRVNEed5aUjBCyC z=SGs3o3%!?Ml_2X#m)O$f1yyBr~EPA>zX@JwkWIfg`bCkXPG&G$9|xeuxaX+i)7n8-2ZHZNA+w_)$;J#?&-ljHK82GnP7o+x(*D6Au;H*soQA6)Mn9{xy5HR-?L zow38YES6P2-{jw>_+uYG`d$f)8?S#WR$@@Ia9Z!qgac*Q4|E04r>C#WsQ~}$%mKV0)pzAU({cUwQqVsf*XZNQ5H^A|_`7l8lD;{X5v literal 0 HcmV?d00001 diff --git a/Package.swift b/Package.swift index 5249157..6fddc13 100644 --- a/Package.swift +++ b/Package.swift @@ -60,7 +60,9 @@ let package = Package( exclude: ["Info.plist", "Streaming/output"], resources: [ // Test resources for metadata stream processor tests - .copy("Streaming/Metadata Stream Processor/raw-audio-streams") + .copy("Streaming/Metadata Stream Processor/raw-audio-streams"), + .copy("Streaming/Mp4 Restructure/moov-at-end.m4a"), + .copy("Streaming/Mp4 Restructure/moov-at-start.m4a") ] ) ]