From 76c40ab0a32149aef3390e51a46ddaa38ac085e7 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Thu, 12 Nov 2026 15:20:16 -0500 Subject: [PATCH 1/4] Keep a new iOS request when a power loss undoes its entry save The entry save wrote entry.json.tmp, synced the file, and renamed it onto entry.json, but never synced the directory. A power loss right after mutate() resolved could leave only entry.json.tmp, which nothing reads, so the request was lost without a trace. - Every entry save now syncs the entry directory after the rename. The first save also syncs the queue root, which holds the new directory's name. A failed sync is logged and does not fail the save. - At startup, before anything loads the entries, a lone entry.json.tmp that decodes to the entry of its directory becomes entry.json. A tmp that does not decode is left on disk. A tmp next to a readable entry.json is a later save that did not finish; entry.json wins and the tmp is deleted. - A write that throws now deletes its tmp, so a rejected enqueue cannot come back at startup. - Removing an entry deletes a stray entry.json.tmp first and syncs that, so a removal cut short cannot leave a lone tmp to bring back. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 10 + README.md | 12 +- ios/FileIO.swift | 27 ++- ios/QueueCoordinator.swift | 3 + ios/QueueStore.swift | 89 ++++++++- ios/Tests/EntrySaveDurabilityTests.swift | 224 +++++++++++++++++++++++ 6 files changed, 352 insertions(+), 13 deletions(-) create mode 100644 ios/Tests/EntrySaveDurabilityTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 55718671..69be57b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,16 @@ caller's file into its queue, so this deleted the only copy. The file now stays, `enqueue` still rejects `E_STORAGE`, and a retry with the same id uses it, as Android already did. +iOS now keeps a new request through a power loss right after `mutate()` +resolves. Before, the entry file's rename was not flushed to disk, so the +request directory could come back holding only `entry.json.tmp`, which +nothing read: the request was silently lost. Each entry save now syncs its +directory, and the first save also syncs the queue root, as Android does. At +startup, iOS turns a lone `entry.json.tmp` that reads as a valid entry into +the entry, logs it, and runs the request as usual. A missing body then +settles `error` with `errorKind: 'file'`. A tmp that does not read is left +on disk. A failed directory sync is logged and does not fail the save. + ## 11.1.0 Added `takeDiagnostics()`. On Android it returns the module's recent diff --git a/README.md b/README.md index 27702e4e..b1640611 100644 --- a/README.md +++ b/README.md @@ -268,12 +268,12 @@ while the app was dead from being sent twice. 1. **Write-ahead.** Entry, descriptor, and staged body persist before any attempt. `mutate()` resolves after the row and every staged body copy are - durably on disk (temp file plus rename), so the caller may delete its - source file then. A native failure rejects with a code: `E_INVALID`, - `E_RUNNING`, `E_FILE_MISSING`, or `E_STORAGE`. `E_INVALID` is malformed - input native cannot send: a non-http(s) URL, header names or values the - platform HTTP client rejects, a GET with a body, or parts that do not - tile the moved file. + durably on disk (temp file, rename, then a sync of the directory), so the + caller may delete its source file then. A native failure rejects with a + code: `E_INVALID`, `E_RUNNING`, `E_FILE_MISSING`, or `E_STORAGE`. + `E_INVALID` is malformed input native cannot send: a non-http(s) URL, + header names or values the platform HTTP client rejects, a GET with a + body, or parts that do not tile the moved file. 2. **Journal before emit, ack after the handler.** Every terminal outcome is journaled natively, then delivered. The library acknowledges after the handler's promise resolves. A rejection, or app death before the ack, diff --git a/ios/FileIO.swift b/ios/FileIO.swift index e41690ec..cf4d04db 100644 --- a/ios/FileIO.swift +++ b/ios/FileIO.swift @@ -3,7 +3,9 @@ import Foundation // Small file helpers shared by the store, the journal, the task map and body // staging. Every durable write is tmp + fsync + rename, so a reader never sees // a half-written file: after a crash there is either the old file or the new -// one, plus at most a stray `.tmp` that the next write replaces. +// one, plus at most a stray `.tmp` that the next write replaces. The rename +// itself survives a power loss only after its directory is synced +// (`syncDirectory`). QueueStore does that for every entry.json. enum FileIO { struct IOError: LocalizedError { let message: String @@ -15,10 +17,29 @@ enum FileIO { } /// Writes `data` to `url.tmp`, flushes it to disk, then renames it onto `url`. + /// A write that throws deletes its tmp, so only a crash leaves one. static func writeAtomically(_ data: Data, to url: URL) throws { let tmp = tmpURL(for: url) - try writeSynced(data, to: tmp) - try rename(tmp, onto: url) + do { + try writeSynced(data, to: tmp) + try rename(tmp, onto: url) + } catch { + try? FileManager.default.removeItem(at: tmp) + throw error + } + } + + /// Flushes a directory, so the names that renames and new files put in it + /// survive a power loss. Plain fsync, as Android does. + static func syncDirectory(_ url: URL) throws { + let fd = open(url.path, O_RDONLY) + guard fd >= 0 else { + throw IOError(message: "cannot open \(url.lastPathComponent) to sync it: errno \(errno)") + } + defer { close(fd) } + guard fsync(fd) == 0 else { + throw IOError(message: "fsync \(url.lastPathComponent) failed: errno \(errno)") + } } /// Writes and fsyncs a file in place. Callers rename it afterwards. diff --git a/ios/QueueCoordinator.swift b/ios/QueueCoordinator.swift index dcc46db8..d0bfd0d0 100644 --- a/ios/QueueCoordinator.swift +++ b/ios/QueueCoordinator.swift @@ -119,6 +119,9 @@ final class QueueCoordinator { queue.asyncAfter(deadline: .now() + .milliseconds(ms), execute: block) } settings = store.loadSettings() + // First: a promoted entry must load below, and never count as a + // directory with no entry. + store.promoteInterruptedSaves() finishSetAsideForgets() // Synchronous, before any session exists, so getRequests() is warm by // the time JS can call it, legacy rows included. diff --git a/ios/QueueStore.swift b/ios/QueueStore.swift index 3d3382eb..4f651f8d 100644 --- a/ios/QueueStore.swift +++ b/ios/QueueStore.swift @@ -13,6 +13,9 @@ import Foundation /// Next to the directories: `settings.json` and the `v10-imported` marker. /// Writes are tmp + fsync + rename. A corrupt or half-written file reads as /// absent. Synchronous on a private serial queue, like the v9 store. +/// +/// `entry.json.tmp` alone in a directory is an entry save that a crash or a +/// power loss cut short. Boot finishes it (`promoteInterruptedSaves`). final class QueueStore { static let shared = QueueStore( root: FileIO.applicationSupport().appendingPathComponent("RNFileUploaderChunked", isDirectory: true)) @@ -27,6 +30,9 @@ final class QueueStore { /// Id directory names are base64url, which never starts with ".". private static let asidePrefix = ".forget-" + /// Flushes a directory. A test seam; the default is FileIO.syncDirectory. + var syncDirectory: (URL) throws -> Void = FileIO.syncDirectory + init(root: URL) { self.root = root FileIO.makeLocalDirectory(root) @@ -58,10 +64,22 @@ final class QueueStore { /// Throws on a write failure. An entry that did not persist must fail the /// enqueue. + /// + /// After the rename it syncs the entry directory, so a power loss cannot + /// undo the rename: the first save would leave only entry.json.tmp, and a + /// later save would bring back an older entry. The first save also syncs + /// the root, which holds the new directory's name. A failed sync is logged + /// and the save still counts, because the rename itself happened. If a + /// power loss then undoes a first save, boot promotes the lone tmp. func save(_ entry: QueueEntry) throws { try queue.sync { - try FileManager.default.createDirectory(at: dir(entry.id), withIntermediateDirectories: true) - try FileIO.writeAtomically(Self.encode(entry), to: fileURL(entry.id, Self.entryName)) + let d = dir(entry.id) + let url = d.appendingPathComponent(Self.entryName) + try FileManager.default.createDirectory(at: d, withIntermediateDirectories: true) + let first = !FileIO.exists(url) + try FileIO.writeAtomically(Self.encode(entry), to: url) + syncLocked(d) + if first { syncLocked(root) } } } @@ -77,9 +95,63 @@ final class QueueStore { } /// Deletes the id directory: entry, body, blob, part files, and a v9 - /// manifest. + /// manifest. A stray entry.json.tmp goes first, and that delete is synced: + /// a removal cut short must never leave the tmp alone, because boot would + /// promote it and bring the request back. func remove(_ id: String) { - queue.sync { _ = try? FileManager.default.removeItem(at: dir(id)) } + queue.sync { + let d = dir(id) + let tmp = FileIO.tmpURL(for: d.appendingPathComponent(Self.entryName)) + if (try? FileManager.default.removeItem(at: tmp)) != nil { syncLocked(d) } + _ = try? FileManager.default.removeItem(at: d) + } + } + + // MARK: - Boot repair + + /// Runs at boot, before anything reads, counts or cleans the entry + /// directories. Finishes entry saves that a crash or a power loss cut + /// short. For each id directory that holds `entry.json.tmp`: + /// - No `entry.json`, and the tmp decodes to the entry of this directory: + /// the tmp is renamed onto `entry.json` and loads like any entry. The + /// app asked for the request, even when the process died before its + /// enqueue resolved. + /// - No `entry.json`, and the tmp does not decode: both stay as they are. + /// - A readable `entry.json`: the tmp is a later save that did not finish. + /// `entry.json` wins and the tmp is deleted. + /// - An unreadable `entry.json`: both stay as they are. + /// It reads nothing but `entry.json.tmp`, so v9 files stay as they are. It + /// skips set-aside directories, which are hidden. Returns the promoted ids. + @discardableResult + func promoteInterruptedSaves() -> [String] { + queue.sync { + var promoted: [String] = [] + for d in subdirectories() { + let url = d.appendingPathComponent(Self.entryName) + let tmp = FileIO.tmpURL(for: url) + guard FileIO.exists(tmp) else { continue } + if FileIO.exists(url) { + if Self.read(url) != nil { _ = try? FileManager.default.removeItem(at: tmp) } + continue + } + guard let e = Self.read(tmp), dir(e.id).lastPathComponent == d.lastPathComponent else { + NSLog("[RNFileUploader] \(d.lastPathComponent)/\(tmp.lastPathComponent) is not an entry of this " + + "directory; left as it is") + continue + } + do { + try FileIO.rename(tmp, onto: url) + } catch { + NSLog("[RNFileUploader] cannot recover entry \(e.id): \(error.localizedDescription)") + continue + } + syncLocked(d) + syncLocked(root) + promoted.append(e.id) + NSLog("[RNFileUploader] recovered entry \(e.id) from an entry save that did not finish") + } + return promoted + } } // MARK: - Forget in steps (cancel on a settled entry) @@ -250,6 +322,15 @@ final class QueueStore { // MARK: - Private (callers hold `queue`) + /// A failed sync is logged, never thrown: see `save`. + private func syncLocked(_ d: URL) { + do { + try syncDirectory(d) + } catch { + NSLog("[RNFileUploader] cannot sync \(d.lastPathComponent): \(error.localizedDescription)") + } + } + // The "part-." prefix cannot collide across indexes ("part-1." is not a // prefix of "part-12."). private func removePartFilesLocked(_ id: String, _ index: Int, keeping: URL?) { diff --git a/ios/Tests/EntrySaveDurabilityTests.swift b/ios/Tests/EntrySaveDurabilityTests.swift new file mode 100644 index 00000000..e93bd08b --- /dev/null +++ b/ios/Tests/EntrySaveDurabilityTests.swift @@ -0,0 +1,224 @@ +import XCTest +@testable import RNBGUCore + +/// The entry save survives a power loss: the directory sync after the +/// rename, and the boot promotion of a lone entry.json.tmp. +final class EntrySaveDurabilityTests: XCTestCase { + private var h: Harness! + + override func setUp() { + h = Harness() + } + + override func tearDown() { + try? FileManager.default.removeItem(at: h.root) + } + + private func entryURL(_ id: String) -> URL { h.store.fileURL(id, QueueStore.entryName) } + private func tmpURL(_ id: String) -> URL { FileIO.tmpURL(for: entryURL(id)) } + + /// What a power loss before the directory sync can leave: the complete + /// tmp, and no entry.json. + private func undoRename(_ id: String) throws { + try FileManager.default.moveItem(at: entryURL(id), to: tmpURL(id)) + } + + private func errorKind(_ settled: [String: Any]?) -> String? { + (settled?["error"] as? [String: Any])?["errorKind"] as? String + } + + // MARK: - Directory sync on save + + func testEverySaveSyncsTheEntryDirectoryAndTheFirstAlsoSyncsTheRoot() throws { + h.boot() + var synced: [URL] = [] + h.store.syncDirectory = { synced.append($0.standardizedFileURL) } + let dir = h.store.dir("a").standardizedFileURL + let root = h.store.root.standardizedFileURL + _ = try h.enqueue(h.dataRaw(id: "a")).get() + XCTAssertEqual(h.entry("a")?.state, .running) + // The first save syncs the new entry.json and the new directory. The + // issue then saves again and syncs only the entry directory. + XCTAssertEqual(synced, [dir, root, dir]) + } + + func testAFailedDirectorySyncDoesNotFailTheSave() throws { + var failures = 0 + h.store.syncDirectory = { _ in + failures += 1 + throw FileIO.IOError(message: "fsync failed") + } + XCTAssertEqual(try h.enqueue(h.dataRaw(id: "a")).get(), "a", "mutate() still resolves") + XCTAssertGreaterThan(failures, 0) + XCTAssertNotNil(QueueStore(root: h.store.root).load("a"), "the entry is on disk") + h.boot() + XCTAssertEqual(h.transport.live.count, 1) + } + + func testTheRealDirectorySyncWorksAndAMissingDirectoryThrows() throws { + XCTAssertNoThrow(try FileIO.syncDirectory(h.store.root)) + XCTAssertThrowsError(try FileIO.syncDirectory(h.root.appendingPathComponent("missing"))) + } + + func testAFailedWriteDeletesItsTmp() throws { + let target = h.root.appendingPathComponent("target") + // A non-empty directory at the target makes the rename fail. + writeFile(target.appendingPathComponent("x"), "x") + XCTAssertThrowsError(try FileIO.writeAtomically(Data("{}".utf8), to: target)) + XCTAssertFalse(FileIO.exists(FileIO.tmpURL(for: target))) + } + + func testARejectedFirstSaveLeavesNoTmpForBootToPromote() throws { + writeFile(entryURL("a").appendingPathComponent("x"), "x") // the rename onto entry.json fails + guard case .failure(let error) = h.enqueue(h.dataRaw(id: "a")) else { return XCTFail("enqueue resolved") } + XCTAssertEqual(error.code, "E_STORAGE") + XCTAssertFalse(FileIO.exists(tmpURL("a"))) + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertNil(fresh.entry("a"), "a rejected request does not come back") + XCTAssertTrue(fresh.transport.created.isEmpty) + } + + // MARK: - Boot promotion + + func testALoneTmpIsPromotedAtBootAndTheRequestRuns() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try undoRename("a") + let fresh = Harness(root: h.root) + XCTAssertTrue(FileIO.exists(entryURL("a")), "promoted before the index loads") + XCTAssertFalse(FileIO.exists(tmpURL("a"))) + XCTAssertEqual(fresh.row("a")?["state"] as? String, "queued") + fresh.boot() + let task = try XCTUnwrap(fresh.transport.live.first) + XCTAssertEqual(fresh.entry("a")?.attempts, 1) + fresh.complete(task) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.sink.settled.last?["kind"] as? String, "completed", "the normal outcome path") + } + + func testPromotionSyncsTheEntryDirectoryAndTheRoot() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try undoRename("a") + let store = QueueStore(root: h.store.root) + var synced: [URL] = [] + store.syncDirectory = { synced.append($0.standardizedFileURL) } + XCTAssertEqual(store.promoteInterruptedSaves(), ["a"]) + XCTAssertEqual(synced, [store.dir("a").standardizedFileURL, store.root.standardizedFileURL]) + XCTAssertEqual(store.promoteInterruptedSaves(), [], "a second boot finds nothing to do") + } + + func testAPromotedEntryTakesARepeatEnqueueAsAnExistingEntry() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try undoRename("a") + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertEqual(fresh.transport.live.count, 1) + // The app never saw the first enqueue resolve, so it calls again. + XCTAssertEqual(try fresh.enqueue(fresh.dataRaw(id: "a")).get(), "a") + XCTAssertEqual(fresh.transport.live.count, 1, "the running attempt keeps going") + XCTAssertEqual(fresh.entry("a")?.attempts, 1) + XCTAssertEqual(fresh.entry("a")?.generation, 1) + } + + func testAPromotedEntryWithAMissingBodySettlesAsAFileError() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let body = try XCTUnwrap(h.store.bodyURL(try XCTUnwrap(h.entry("a")))) + try FileManager.default.removeItem(at: body) + try undoRename("a") + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertEqual(fresh.entry("a")?.state, .error) + XCTAssertEqual(errorKind(fresh.sink.settled.last), "file") + XCTAssertTrue(fresh.transport.created.isEmpty) + fresh.advance(60_000) + XCTAssertEqual(fresh.sink.settled.count, 1, "settled once, no loop") + } + + func testAPromotedChunkedEntryWithAMissingBlobSettlesAsAFileError() throws { + _ = try h.enqueue(h.chunkedRaw(id: "c")).get() + let blob = try XCTUnwrap(h.store.bodyURL(try XCTUnwrap(h.entry("c")))) + try FileManager.default.removeItem(at: blob) + try undoRename("c") + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertEqual(fresh.entry("c")?.state, .error) + XCTAssertEqual(errorKind(fresh.sink.settled.last), "file") + XCTAssertTrue(fresh.transport.created.isEmpty) + } + + func testATmpThatDoesNotDecodeIsLeftAlone() throws { + writeFile(tmpURL("a"), #"{"id":"a","state":"#) // cut short + writeFile(h.store.fileURL("a", "body-1"), "{}") + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertNil(fresh.entry("a")) + XCTAssertFalse(FileIO.exists(entryURL("a"))) + XCTAssertEqual(try String(contentsOf: tmpURL("a")), #"{"id":"a","state":"#) + XCTAssertTrue(FileIO.exists(h.store.fileURL("a", "body-1")), "nothing is deleted") + } + + func testATmpOfAnotherIdIsLeftAlone() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try undoRename("a") + // The same file under the directory of another id. + let other = h.store.dir("b").appendingPathComponent(QueueStore.entryName + ".tmp") + try FileManager.default.createDirectory(at: h.store.dir("b"), withIntermediateDirectories: true) + try FileManager.default.moveItem(at: tmpURL("a"), to: other) + let fresh = Harness(root: h.root) + XCTAssertNil(fresh.entry("a")) + XCTAssertNil(fresh.entry("b")) + XCTAssertTrue(FileIO.exists(other)) + } + + func testATmpNextToAReadableEntryIsDeletedAndTheEntryWins() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + var later = try XCTUnwrap(h.entry("a")) + later.state = .cancelled // a later save that did not finish + try QueueStore.encode(later).write(to: tmpURL("a")) + let fresh = Harness(root: h.root) + XCTAssertEqual(fresh.entry("a")?.state, .queued, "entry.json wins") + XCTAssertFalse(FileIO.exists(tmpURL("a")), "so a later removal cannot leave it alone") + fresh.boot() + XCTAssertEqual(fresh.transport.live.count, 1) + } + + func testATmpNextToAnUnreadableEntryIsLeftAlone() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try FileManager.default.copyItem(at: entryURL("a"), to: tmpURL("a")) + try Data("{".utf8).write(to: entryURL("a")) + let fresh = Harness(root: h.root) + XCTAssertNil(fresh.entry("a")) + XCTAssertTrue(FileIO.exists(tmpURL("a"))) + XCTAssertEqual(try String(contentsOf: entryURL("a")), "{") + } + + func testRemoveDeletesAStrayTmpFirstAndSyncsThatDelete() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try FileManager.default.copyItem(at: entryURL("a"), to: tmpURL("a")) + var remainingAtSync: [String] = [] + h.store.syncDirectory = { [unowned self] dir in + remainingAtSync = (try? FileManager.default.contentsOfDirectory(atPath: dir.path)) ?? [] + XCTAssertEqual(dir.standardizedFileURL, self.h.store.dir("a").standardizedFileURL) + } + h.store.remove("a") + XCTAssertTrue(remainingAtSync.contains(QueueStore.entryName), "the tmp went before entry.json") + XCTAssertFalse(remainingAtSync.contains(QueueStore.entryName + ".tmp")) + XCTAssertFalse(FileIO.exists(h.store.dir("a"))) + } + + func testSetAsideAndV9DirectoriesAreNotTouched() throws { + // A v9 directory: a manifest and its blob, no entry file. + let v9 = h.store.dir("v9") + writeFile(v9.appendingPathComponent(QueueStore.manifestName), "{}") + writeFile(v9.appendingPathComponent(ChunkedManifestV9.blobName), bytes: 10) + // A set-aside directory holding a lone tmp. + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try undoRename("a") + let aside = h.store.root.appendingPathComponent(".forget-x", isDirectory: true) + try FileManager.default.moveItem(at: h.store.dir("a"), to: aside) + let before = try FileManager.default.contentsOfDirectory(atPath: v9.path).sorted() + XCTAssertEqual(QueueStore(root: h.store.root).promoteInterruptedSaves(), []) + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: v9.path).sorted(), before) + XCTAssertTrue(FileIO.exists(aside.appendingPathComponent(QueueStore.entryName + ".tmp"))) + } +} From eb16ac78a56c13ab26eda6b34fe8b12f6ee1ee78 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Thu, 12 Nov 2026 17:34:13 -0500 Subject: [PATCH 2/4] Sync the directory after every iOS atomic write Review follow-up. The outcome journal renamed its file into place without a directory sync, while the completed entry save after it was synced. A power loss could then keep the completed row and lose the outcome file, and boot would forget the row before JS saw the outcome. - FileIO.writeAtomically now syncs the parent directory after the rename, as Android does. Entries, staged bodies, the journal, the task map, settings and the import marker all get it. QueueStore keeps the extra root sync on an entry's first save. - A failed sync is still logged and never thrown. The test seam moves to FileIO.syncDirectory. - Tests pin the sync after the rename and the journal sync before the completed entry save. - The tmp-delete lines in writeAtomically match the open storage-cause branch, so both merge cleanly. - CHANGELOG states what the code does. README says a failed sync does not fail the call, and that iOS fsync does not empty the drive cache. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 12 ++-- README.md | 13 ++-- ios/FileIO.swift | 50 +++++++++----- ios/QueueStore.swift | 33 +++------- ios/Tests/EntrySaveDurabilityTests.swift | 84 +++++++++++++++++++----- 5 files changed, 127 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69be57b8..919e8f1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,14 +22,16 @@ stays, `enqueue` still rejects `E_STORAGE`, and a retry with the same id uses it, as Android already did. iOS now keeps a new request through a power loss right after `mutate()` -resolves. Before, the entry file's rename was not flushed to disk, so the -request directory could come back holding only `entry.json.tmp`, which -nothing read: the request was silently lost. Each entry save now syncs its -directory, and the first save also syncs the queue root, as Android does. At +resolves. Before, no rename into place was flushed to disk, so the request +directory could come back holding only `entry.json.tmp`, which nothing read: +the request was silently lost. Every atomic write on iOS (entries, staged +bodies, outcome journal, task map, settings) now syncs its directory after +the rename, as Android does, and an entry's first save also syncs the queue +root. A failed directory sync is logged and does not fail the write. At startup, iOS turns a lone `entry.json.tmp` that reads as a valid entry into the entry, logs it, and runs the request as usual. A missing body then settles `error` with `errorKind: 'file'`. A tmp that does not read is left -on disk. A failed directory sync is logged and does not fail the save. +on disk. A tmp next to a readable `entry.json` is deleted. ## 11.1.0 diff --git a/README.md b/README.md index b1640611..e35ec8b5 100644 --- a/README.md +++ b/README.md @@ -269,11 +269,14 @@ while the app was dead from being sent twice. 1. **Write-ahead.** Entry, descriptor, and staged body persist before any attempt. `mutate()` resolves after the row and every staged body copy are durably on disk (temp file, rename, then a sync of the directory), so the - caller may delete its source file then. A native failure rejects with a - code: `E_INVALID`, `E_RUNNING`, `E_FILE_MISSING`, or `E_STORAGE`. - `E_INVALID` is malformed input native cannot send: a non-http(s) URL, - header names or values the platform HTTP client rejects, a GET with a - body, or parts that do not tile the moved file. + caller may delete its source file then. A failed directory sync is logged + and does not fail the call. The library uses `fsync`, which on iOS does not + force the drive to empty its cache. A native failure rejects with a code: + `E_INVALID`, + `E_RUNNING`, `E_FILE_MISSING`, or `E_STORAGE`. `E_INVALID` is malformed + input native cannot send: a non-http(s) URL, header names or values the + platform HTTP client rejects, a GET with a body, or parts that do not + tile the moved file. 2. **Journal before emit, ack after the handler.** Every terminal outcome is journaled natively, then delivered. The library acknowledges after the handler's promise resolves. A rejection, or app death before the ack, diff --git a/ios/FileIO.swift b/ios/FileIO.swift index cf4d04db..bbe9e5e5 100644 --- a/ios/FileIO.swift +++ b/ios/FileIO.swift @@ -4,8 +4,8 @@ import Foundation // staging. Every durable write is tmp + fsync + rename, so a reader never sees // a half-written file: after a crash there is either the old file or the new // one, plus at most a stray `.tmp` that the next write replaces. The rename -// itself survives a power loss only after its directory is synced -// (`syncDirectory`). QueueStore does that for every entry.json. +// itself survives a power loss only after its directory is synced, so +// `writeAtomically` syncs the directory after every rename, as Android does. enum FileIO { struct IOError: LocalizedError { let message: String @@ -17,8 +17,12 @@ enum FileIO { } /// Writes `data` to `url.tmp`, flushes it to disk, then renames it onto `url`. - /// A write that throws deletes its tmp, so only a crash leaves one. + /// On a failure the tmp file is deleted, so a partly written file does not + /// hold space that a retry needs. static func writeAtomically(_ data: Data, to url: URL) throws { + // Runs last: after the rename, so the new name is what the sync makes + // durable. After a failure, it makes the tmp delete durable. + defer { syncDirectoryOrLog(url.deletingLastPathComponent()) } let tmp = tmpURL(for: url) do { try writeSynced(data, to: tmp) @@ -29,19 +33,6 @@ enum FileIO { } } - /// Flushes a directory, so the names that renames and new files put in it - /// survive a power loss. Plain fsync, as Android does. - static func syncDirectory(_ url: URL) throws { - let fd = open(url.path, O_RDONLY) - guard fd >= 0 else { - throw IOError(message: "cannot open \(url.lastPathComponent) to sync it: errno \(errno)") - } - defer { close(fd) } - guard fsync(fd) == 0 else { - throw IOError(message: "fsync \(url.lastPathComponent) failed: errno \(errno)") - } - } - /// Writes and fsyncs a file in place. Callers rename it afterwards. static func writeSynced(_ data: Data, to url: URL) throws { let fm = FileManager.default @@ -70,6 +61,33 @@ enum FileIO { FileManager.default.fileExists(atPath: url.path) } + /// Flushes a directory. Tests replace it to watch the syncs or fail them. + static var syncDirectory: (URL) throws -> Void = fsyncDirectory + + /// Flushes a directory, so the names that renames and new files put in it + /// survive a power loss. Plain fsync, as Android does. On Apple platforms + /// fsync does not make the drive empty its cache; F_FULLFSYNC would. + static func fsyncDirectory(_ url: URL) throws { + let fd = open(url.path, O_RDONLY) + guard fd >= 0 else { + throw IOError(message: "cannot open \(url.lastPathComponent) to sync it: errno \(errno)") + } + defer { close(fd) } + guard fsync(fd) == 0 else { + throw IOError(message: "fsync \(url.lastPathComponent) failed: errno \(errno)") + } + } + + /// A failed sync is logged, never thrown: the rename it follows already + /// happened, so the write still counts. + static func syncDirectoryOrLog(_ url: URL) { + do { + try syncDirectory(url) + } catch { + NSLog("[RNFileUploader] cannot sync \(url.lastPathComponent): \(error.localizedDescription)") + } + } + /// Creates a directory that holds device-local upload state and keeps it /// out of iCloud and iTunes backups. static func makeLocalDirectory(_ url: URL) { diff --git a/ios/QueueStore.swift b/ios/QueueStore.swift index 4f651f8d..e7a13d34 100644 --- a/ios/QueueStore.swift +++ b/ios/QueueStore.swift @@ -30,9 +30,6 @@ final class QueueStore { /// Id directory names are base64url, which never starts with ".". private static let asidePrefix = ".forget-" - /// Flushes a directory. A test seam; the default is FileIO.syncDirectory. - var syncDirectory: (URL) throws -> Void = FileIO.syncDirectory - init(root: URL) { self.root = root FileIO.makeLocalDirectory(root) @@ -65,12 +62,12 @@ final class QueueStore { /// Throws on a write failure. An entry that did not persist must fail the /// enqueue. /// - /// After the rename it syncs the entry directory, so a power loss cannot - /// undo the rename: the first save would leave only entry.json.tmp, and a - /// later save would bring back an older entry. The first save also syncs - /// the root, which holds the new directory's name. A failed sync is logged - /// and the save still counts, because the rename itself happened. If a - /// power loss then undoes a first save, boot promotes the lone tmp. + /// `writeAtomically` syncs the entry directory after the rename, so a + /// power loss cannot undo it: the first save would leave only + /// entry.json.tmp, and a later save would bring back an older entry. The + /// first save also syncs the root, which holds the new directory's name. + /// A failed sync is logged and the save still counts. If a power loss then + /// undoes a first save, boot promotes the lone tmp. func save(_ entry: QueueEntry) throws { try queue.sync { let d = dir(entry.id) @@ -78,8 +75,7 @@ final class QueueStore { try FileManager.default.createDirectory(at: d, withIntermediateDirectories: true) let first = !FileIO.exists(url) try FileIO.writeAtomically(Self.encode(entry), to: url) - syncLocked(d) - if first { syncLocked(root) } + if first { FileIO.syncDirectoryOrLog(root) } } } @@ -102,7 +98,7 @@ final class QueueStore { queue.sync { let d = dir(id) let tmp = FileIO.tmpURL(for: d.appendingPathComponent(Self.entryName)) - if (try? FileManager.default.removeItem(at: tmp)) != nil { syncLocked(d) } + if (try? FileManager.default.removeItem(at: tmp)) != nil { FileIO.syncDirectoryOrLog(d) } _ = try? FileManager.default.removeItem(at: d) } } @@ -145,8 +141,8 @@ final class QueueStore { NSLog("[RNFileUploader] cannot recover entry \(e.id): \(error.localizedDescription)") continue } - syncLocked(d) - syncLocked(root) + FileIO.syncDirectoryOrLog(d) + FileIO.syncDirectoryOrLog(root) promoted.append(e.id) NSLog("[RNFileUploader] recovered entry \(e.id) from an entry save that did not finish") } @@ -322,15 +318,6 @@ final class QueueStore { // MARK: - Private (callers hold `queue`) - /// A failed sync is logged, never thrown: see `save`. - private func syncLocked(_ d: URL) { - do { - try syncDirectory(d) - } catch { - NSLog("[RNFileUploader] cannot sync \(d.lastPathComponent): \(error.localizedDescription)") - } - } - // The "part-." prefix cannot collide across indexes ("part-1." is not a // prefix of "part-12."). private func removePartFilesLocked(_ id: String, _ index: Int, keeping: URL?) { diff --git a/ios/Tests/EntrySaveDurabilityTests.swift b/ios/Tests/EntrySaveDurabilityTests.swift index e93bd08b..e10fe896 100644 --- a/ios/Tests/EntrySaveDurabilityTests.swift +++ b/ios/Tests/EntrySaveDurabilityTests.swift @@ -1,8 +1,8 @@ import XCTest @testable import RNBGUCore -/// The entry save survives a power loss: the directory sync after the -/// rename, and the boot promotion of a lone entry.json.tmp. +/// Writes survive a power loss: the directory sync after each rename, and +/// the boot promotion of a lone entry.json.tmp. final class EntrySaveDurabilityTests: XCTestCase { private var h: Harness! @@ -11,6 +11,7 @@ final class EntrySaveDurabilityTests: XCTestCase { } override func tearDown() { + FileIO.syncDirectory = FileIO.fsyncDirectory try? FileManager.default.removeItem(at: h.root) } @@ -27,24 +28,72 @@ final class EntrySaveDurabilityTests: XCTestCase { (settled?["error"] as? [String: Any])?["errorKind"] as? String } - // MARK: - Directory sync on save + // MARK: - Directory sync after every atomic write - func testEverySaveSyncsTheEntryDirectoryAndTheFirstAlsoSyncsTheRoot() throws { - h.boot() + func testAnAtomicWriteSyncsItsDirectoryAfterTheRename() throws { + let target = h.root.appendingPathComponent("file.json") var synced: [URL] = [] - h.store.syncDirectory = { synced.append($0.standardizedFileURL) } + FileIO.syncDirectory = { dir in + synced.append(dir.standardizedFileURL) + XCTAssertTrue(FileIO.exists(target), "the rename happened before the sync") + XCTAssertFalse(FileIO.exists(FileIO.tmpURL(for: target))) + } + try FileIO.writeAtomically(Data("{}".utf8), to: target) + XCTAssertEqual(synced, [h.root.standardizedFileURL]) + } + + func testEveryEntrySaveSyncsTheEntryDirectoryAfterTheRenameAndTheFirstAlsoSyncsTheRoot() throws { + h.boot() let dir = h.store.dir("a").standardizedFileURL let root = h.store.root.standardizedFileURL + // For each sync of the entry directory or the root: which one, and + // whether entry.json and entry.json.tmp were there. + var synced: [(url: URL, entry: Bool, tmp: Bool)] = [] + FileIO.syncDirectory = { [unowned self] url in + let u = url.standardizedFileURL + guard u == dir || u == root else { return } + synced.append((u, FileIO.exists(self.entryURL("a")), FileIO.exists(self.tmpURL("a")))) + } _ = try h.enqueue(h.dataRaw(id: "a")).get() XCTAssertEqual(h.entry("a")?.state, .running) - // The first save syncs the new entry.json and the new directory. The - // issue then saves again and syncs only the entry directory. - XCTAssertEqual(synced, [dir, root, dir]) + // The staged body (no entry yet), the first save (then the root, which + // holds the new directory's name), and the issue's save. + XCTAssertEqual(synced.map(\.url), [dir, dir, root, dir]) + XCTAssertEqual(synced.map(\.entry), [false, true, true, true], "each entry sync follows its rename") + XCTAssertEqual(synced.map(\.tmp), [false, false, false, false]) + } + + func testAJournalWriteIsSyncedBeforeTheCompletedEntrySave() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = try XCTUnwrap(h.transport.live.first) + let journalDir = h.root.appendingPathComponent("events").standardizedFileURL + let dir = h.store.dir("a").standardizedFileURL + // In order: each journal sync with the outcome files there, and each + // entry sync with the entry's state. + var synced: [String] = [] + FileIO.syncDirectory = { [unowned self] url in + let u = url.standardizedFileURL + if u == journalDir { + let files = (try? FileManager.default.contentsOfDirectory(atPath: u.path)) ?? [] + synced.append("journal:" + files.filter { $0.hasSuffix(".json") }.count.description) + } else if u == dir { + // Read the file directly: the sync runs inside the store's queue. + let saved = (try? Data(contentsOf: self.entryURL("a"))) + .flatMap { try? JSONDecoder().decode(QueueEntry.self, from: $0) } + synced.append("entry:" + (saved?.state.rawValue ?? "none")) + } + } + h.complete(task) + XCTAssertEqual(h.entry("a")?.state, .completed) + let journal = try XCTUnwrap(synced.firstIndex(of: "journal:1"), "the journal file is synced after its rename") + let completed = try XCTUnwrap(synced.firstIndex(of: "entry:completed")) + XCTAssertLessThan(journal, completed, "the outcome is durable before the completed row") } - func testAFailedDirectorySyncDoesNotFailTheSave() throws { + func testAFailedDirectorySyncFailsNeitherTheSaveNorTheSettle() throws { var failures = 0 - h.store.syncDirectory = { _ in + FileIO.syncDirectory = { _ in failures += 1 throw FileIO.IOError(message: "fsync failed") } @@ -52,12 +101,15 @@ final class EntrySaveDurabilityTests: XCTestCase { XCTAssertGreaterThan(failures, 0) XCTAssertNotNil(QueueStore(root: h.store.root).load("a"), "the entry is on disk") h.boot() - XCTAssertEqual(h.transport.live.count, 1) + h.complete(try XCTUnwrap(h.transport.live.first)) + XCTAssertEqual(h.entry("a")?.state, .completed) + XCTAssertEqual(h.sink.settled.last?["kind"] as? String, "completed") + XCTAssertEqual(h.unacknowledged().count, 1, "the outcome is journaled") } func testTheRealDirectorySyncWorksAndAMissingDirectoryThrows() throws { - XCTAssertNoThrow(try FileIO.syncDirectory(h.store.root)) - XCTAssertThrowsError(try FileIO.syncDirectory(h.root.appendingPathComponent("missing"))) + XCTAssertNoThrow(try FileIO.fsyncDirectory(h.store.root)) + XCTAssertThrowsError(try FileIO.fsyncDirectory(h.root.appendingPathComponent("missing"))) } func testAFailedWriteDeletesItsTmp() throws { @@ -101,7 +153,7 @@ final class EntrySaveDurabilityTests: XCTestCase { try undoRename("a") let store = QueueStore(root: h.store.root) var synced: [URL] = [] - store.syncDirectory = { synced.append($0.standardizedFileURL) } + FileIO.syncDirectory = { synced.append($0.standardizedFileURL) } XCTAssertEqual(store.promoteInterruptedSaves(), ["a"]) XCTAssertEqual(synced, [store.dir("a").standardizedFileURL, store.root.standardizedFileURL]) XCTAssertEqual(store.promoteInterruptedSaves(), [], "a second boot finds nothing to do") @@ -196,7 +248,7 @@ final class EntrySaveDurabilityTests: XCTestCase { _ = try h.enqueue(h.dataRaw(id: "a")).get() try FileManager.default.copyItem(at: entryURL("a"), to: tmpURL("a")) var remainingAtSync: [String] = [] - h.store.syncDirectory = { [unowned self] dir in + FileIO.syncDirectory = { [unowned self] dir in remainingAtSync = (try? FileManager.default.contentsOfDirectory(atPath: dir.path)) ?? [] XCTAssertEqual(dir.standardizedFileURL, self.h.store.dir("a").standardizedFileURL) } From 7ea8084644dc60997d6bf88de28d7843372f8e4c Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Thu, 12 Nov 2026 17:47:31 -0500 Subject: [PATCH 3/4] Test the directory sync after a failed write, and list synced bodies exactly - New test: a write whose rename fails throws, deletes its tmp, and syncs the parent directory once, after the tmp is gone. - CHANGELOG: only data bodies go through the atomic write. Form, file and chunked bodies are placed by a rename or move, and the entry save that follows syncs the same directory before mutate() resolves. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 7 +++++-- ios/Tests/EntrySaveDurabilityTests.swift | 13 +++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 919e8f1c..869714a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,10 +24,13 @@ uses it, as Android already did. iOS now keeps a new request through a power loss right after `mutate()` resolves. Before, no rename into place was flushed to disk, so the request directory could come back holding only `entry.json.tmp`, which nothing read: -the request was silently lost. Every atomic write on iOS (entries, staged +the request was silently lost. Every atomic write on iOS (entries, data bodies, outcome journal, task map, settings) now syncs its directory after the rename, as Android does, and an entry's first save also syncs the queue -root. A failed directory sync is logged and does not fail the write. At +root. Form, file and chunked bodies are placed by a rename or move into the +request directory, and the entry save that follows syncs that directory +before `mutate()` resolves. A failed directory sync is logged and does not +fail the write. At startup, iOS turns a lone `entry.json.tmp` that reads as a valid entry into the entry, logs it, and runs the request as usual. A missing body then settles `error` with `errorKind: 'file'`. A tmp that does not read is left diff --git a/ios/Tests/EntrySaveDurabilityTests.swift b/ios/Tests/EntrySaveDurabilityTests.swift index e10fe896..2242ce71 100644 --- a/ios/Tests/EntrySaveDurabilityTests.swift +++ b/ios/Tests/EntrySaveDurabilityTests.swift @@ -120,6 +120,19 @@ final class EntrySaveDurabilityTests: XCTestCase { XCTAssertFalse(FileIO.exists(FileIO.tmpURL(for: target))) } + func testAFailedWriteSyncsItsDirectoryOnceAfterTheTmpIsGone() throws { + let target = h.root.appendingPathComponent("target") + let tmp = FileIO.tmpURL(for: target) + // A non-empty directory at the target makes the rename fail. + writeFile(target.appendingPathComponent("x"), "x") + var synced: [(url: URL, tmp: Bool)] = [] + FileIO.syncDirectory = { synced.append(($0.standardizedFileURL, FileIO.exists(tmp))) } + XCTAssertThrowsError(try FileIO.writeAtomically(Data("{}".utf8), to: target)) + XCTAssertFalse(FileIO.exists(tmp)) + XCTAssertEqual(synced.map(\.url), [h.root.standardizedFileURL], "the parent, once") + XCTAssertEqual(synced.map(\.tmp), [false], "the tmp delete is what the sync makes durable") + } + func testARejectedFirstSaveLeavesNoTmpForBootToPromote() throws { writeFile(entryURL("a").appendingPathComponent("x"), "x") // the rename onto entry.json fails guard case .failure(let error) = h.enqueue(h.dataRaw(id: "a")) else { return XCTFail("enqueue resolved") } From 7f06e7478a9e198cd381db91745fba14144aa1f5 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Fri, 13 Nov 2026 14:56:58 -0500 Subject: [PATCH 4/4] Give the current reason the orphan cleanup keeps entry.json.tmp The isCopiedBodyFile comment from #78 said FileIO.writeAtomically does not sync the directory. This branch makes it sync, and boot promotes a tmp that reads as the directory's entry before the cleanup runs. The rule is unchanged: a directory with entry.json.tmp is never deleted. The comment now says a tmp still there is one boot promotion could not use. Co-Authored-By: Claude Fable 5.2 --- ios/QueueStore.swift | 226 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 225 insertions(+), 1 deletion(-) diff --git a/ios/QueueStore.swift b/ios/QueueStore.swift index e7a13d34..2f9229b7 100644 --- a/ios/QueueStore.swift +++ b/ios/QueueStore.swift @@ -8,7 +8,8 @@ import Foundation /// - `entry.json`: the QueueEntry (v10). /// - `body-` or `blob-` / `blob`: the staged body. /// - `part-..-`: a chunked part while in flight. -/// - `manifest.json`: a v9 manifest, until a same-id enqueue adopts it. +/// - `manifest.json`: a v9 manifest, until a same-id enqueue adopts it or +/// cancel deletes it. /// /// Next to the directories: `settings.json` and the `v10-imported` marker. /// Writes are tmp + fsync + rename. A corrupt or half-written file reads as @@ -150,6 +151,229 @@ final class QueueStore { } } + /// Deletes the id directory when it holds no `entry.json`: v9 files that + /// no row owns, or a chunked file kept after an enqueue whose save failed + /// (the only copy of the caller's bytes). A directory with an + /// `entry.json`, readable or not, is kept. The empty id names the store + /// root, so it deletes nothing. + /// Returns the error of a delete that failed. + @discardableResult + func removeUnowned(_ id: String) -> Error? { + guard !id.isEmpty else { return nil } + return queue.sync { + let d = dir(id) + guard FileIO.exists(d), Self.isUnowned(d) else { return nil } + do { + try FileManager.default.removeItem(at: d) + return nil + } catch { + return error + } + } + } + + /// The most unowned v9 ids one count lists. The count itself stays exact. + static let maxUnownedIds = 100 + + /// The id whose `dir(_:)` is named `name`, or nil when no id encodes to it. + func id(forDirectoryName name: String) -> String? { + var b64 = name.replacingOccurrences(of: "-", with: "+").replacingOccurrences(of: "_", with: "/") + while b64.count % 4 != 0 { b64 += "=" } + guard let data = Data(base64Encoded: b64), let id = String(data: data, encoding: .utf8), + dir(id).lastPathComponent == name else { return nil } + return id + } + + /// A directory that `removeUnowned` deletes: it holds no `entry.json`. + private static func isUnowned(_ d: URL) -> Bool { + !FileIO.exists(d.appendingPathComponent(entryName)) + } + + /// Test seam: runs before the count lists an unowned directory. + var beforeCountListing: ((URL) -> Void)? + + /// Counts what no v10 entry owns, for the boot snapshot. Only directories + /// that `removeUnowned` would delete (no `entry.json`, not empty) count: + /// - v9: the directory holds `manifest.json` or a file named exactly + /// `blob` (v10 names its blobs `blob-`). Its id, decoded from the + /// directory name, goes in unownedIds, sorted, at most `maxUnownedIds`. + /// A name that decodes to no id is a warning; the directory still counts; + /// - orphan: any other, such as `body-`, `blob-` or a `.tmp`. + /// Each legacy row in `legacyIds` counts once, and the v9 manifest and + /// blob it keeps count as legacy bytes. A legacy directory has its + /// `entry.json`, so it is never an unowned upload too. Hidden directories + /// (a `.forget-` set aside by a cancel) are skipped. + /// + /// Lists directories and reads file sizes only. It opens no file and + /// changes nothing. It does not run on `queue`, so store calls do not + /// wait; a directory that changes during the count, such as one an + /// enqueue is staging, can be counted either way. An error goes in + /// `warnings`, and the count goes on. + func countLeftovers(legacyIds: [String]) -> (Leftovers, warnings: [(message: String, error: Error?)]) { + var found = Leftovers() + found.v9.legacyRows = legacyIds.count + var warnings: [(message: String, error: Error?)] = [] + var ids: [String] = [] + let fm = FileManager.default + let dirs: [URL] + do { + dirs = try fm.contentsOfDirectory(at: root, includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles]) + } catch { + warnings.append(("leftover count: could not list the queue directory", error)) + dirs = [] + } + for d in dirs where (try? d.resourceValues(forKeys: [.isDirectoryKey]).isDirectory) == true && Self.isUnowned(d) { + beforeCountListing?(d) + let files: [URL] + do { + files = try fm.contentsOfDirectory(at: d, includingPropertiesForKeys: [.isRegularFileKey, .fileSizeKey]) + } catch { + // Gone since the root listing (a cancel or a forget) is not an error. + if FileIO.exists(d) { + warnings.append(("leftover count: could not list \(d.lastPathComponent)", error)) + } + continue + } + guard !files.isEmpty else { continue } + let size = files.reduce(0) { $0 + (Self.regularFileSize($1) ?? 0) } + let names = Set(files.map(\.lastPathComponent)) + if names.contains(Self.manifestName) || names.contains(ChunkedManifestV9.blobName) { + found.v9.unownedUploads += 1 + found.v9.unownedBytes += size + if let id = id(forDirectoryName: d.lastPathComponent) { + ids.append(id) + } else { + warnings.append(("leftover count: no id encodes to the directory \(d.lastPathComponent)", nil)) + } + } else { + found.orphans.dirs += 1 + found.orphans.bytes += size + found.orphanDirs.append(d) + } + } + // A legacy id with no entry file was counted above, as the scan found it. + for id in legacyIds where !Self.isUnowned(dir(id)) { + found.v9.legacyBytes += [fileURL(id, Self.manifestName), fileURL(id, ChunkedManifestV9.blobName)] + .compactMap(Self.regularFileSize).reduce(0, +) + } + found.v9.unownedIds = Array(ids.sorted().prefix(Self.maxUnownedIds)) + return (found, warnings) + } + + // MARK: - Boot cleanup of orphans + + /// The boot cleanup deletes an orphan directory only when its newest file, + /// and the directory itself, changed longer ago than this. A guard against + /// a staging path the name rules below do not know. + static let orphanMinAgeMs: Double = 24 * 60 * 60 * 1000 + + /// `body-`, as `BodyStaging.uniqueName(bodyPrefix)` makes it, with + /// an optional `.tmp`. + private static let copiedBody = try! NSRegularExpression( + pattern: "^body-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(\\.tmp)?$") + + /// The files of an orphan that the boot cleanup may delete: copies whose + /// source the caller still has, and the tmp files of their atomic writes. + /// - `body-`: a bodiless, data, form or copied file body + /// (`BodyStaging.stage`); + /// - `body-.tmp`: an atomic write that did not finish + /// (`FileIO.tmpURL`). + /// Never a chunked blob (`blob`, `blob-`): that file was moved in, + /// so it can be the only copy, and a same-id enqueue adopts it + /// (`adoptableBlob`). Never a part file: its entry is gone, so its use is + /// not known. Never `entry.json.tmp`: boot promotion + /// (`promoteInterruptedSaves`) runs before this cleanup, so a tmp still + /// here is one it could not use: it does not read as this directory's + /// entry, or its rename failed. It can still be the only trace of a + /// request, so the directory stays. + static func isCopiedBodyFile(_ name: String) -> Bool { + let range = NSRange(name.startIndex..., in: name) + return copiedBody.firstMatch(in: name, range: range) != nil + } + + /// Test seam: runs before the cleanup checks an orphan directory. + var beforeOrphanRemoval: ((URL) -> Void)? + + /// The boot cleanup. Deletes each directory of `dirs` (the orphans that + /// `countLeftovers` listed) when all of these hold, checked on `queue` + /// just before the delete: + /// - it has no `entry.json`; + /// - every item in it is a regular file that `isCopiedBodyFile` accepts. + /// So a v9 manifest or bare `blob`, a chunked blob, `entry.json.tmp`, a + /// part file, an unknown file, a symbolic link or a subdirectory keeps + /// the whole directory; + /// - its newest file, and the directory itself, changed more than + /// `orphanMinAgeMs` before `now` (epoch ms). A time at or before the + /// epoch counts as unknown and keeps the directory. + /// The caller runs it where no enqueue can be staging (the coordinator + /// queue). A failed delete goes in `warnings`, and the other directories + /// go on. Returns what it deleted: directories fully gone, and the bytes + /// of every file deleted. + func removeOrphans(_ dirs: [URL], now: Double) + -> (removedDirs: Int, removedBytes: Int64, warnings: [(message: String, error: Error?)]) { + var removedDirs = 0 + var removedBytes: Int64 = 0 + var warnings: [(message: String, error: Error?)] = [] + for d in dirs { + beforeOrphanRemoval?(d) + let result = queue.sync { Self.removeOrphan(d, now: now) } + removedBytes += result.bytes + if result.gone { removedDirs += 1 } + if let w = result.warning { warnings.append(w) } + } + return (removedDirs, removedBytes, warnings) + } + + /// One directory of `removeOrphans`. Callers hold `queue`. + private static func removeOrphan(_ d: URL, now: Double) + -> (bytes: Int64, gone: Bool, warning: (message: String, error: Error?)?) { + let keep: (bytes: Int64, gone: Bool, warning: (message: String, error: Error?)?) = (0, false, nil) + let fm = FileManager.default + let keys: [URLResourceKey] = [.isRegularFileKey, .fileSizeKey, .contentModificationDateKey] + guard isUnowned(d), + let dirDate = try? d.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate, + let files = try? fm.contentsOfDirectory(at: d, includingPropertiesForKeys: keys), + !files.isEmpty else { return keep } + var newest = dirDate + var sizes: [Int64] = [] + for f in files { + // URL resource values do not follow a symbolic link, so a link is not + // a regular file and keeps the directory. + guard isCopiedBodyFile(f.lastPathComponent), + let v = try? f.resourceValues(forKeys: Set(keys)), v.isRegularFile == true, + let date = v.contentModificationDate else { return keep } + // A time at or before the epoch is not a real write time: keep. + guard date.timeIntervalSince1970 > 0 else { return keep } + newest = max(newest, date) + sizes.append(Int64(v.fileSize ?? 0)) + } + guard dirDate.timeIntervalSince1970 > 0, + now - newest.timeIntervalSince1970 * 1000 > orphanMinAgeMs else { return keep } + var freed: Int64 = 0 + for (f, size) in zip(files, sizes) { + do { + try fm.removeItem(at: f) + } catch { + return (freed, false, ("orphan cleanup: could not delete \(d.lastPathComponent)/\(f.lastPathComponent)", error)) + } + freed += size + } + // rmdir deletes an empty directory only. + guard Foundation.rmdir(d.path) == 0 else { + let error = NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) + return (freed, false, ("orphan cleanup: could not delete \(d.lastPathComponent)", error)) + } + return (freed, true, nil) + } + + /// The size of a regular file, or nil for anything else or a failed read. + private static func regularFileSize(_ url: URL) -> Int64? { + guard let values = try? url.resourceValues(forKeys: [.isRegularFileKey, .fileSizeKey]), + values.isRegularFile == true else { return nil } + return Int64(values.fileSize ?? 0) + } + // MARK: - Forget in steps (cancel on a settled entry) /// Step one of a forget that must not half-happen: renames the id