diff --git a/CHANGELOG.md b/CHANGELOG.md index 4811236e..f1aaf8a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,21 @@ pending until the returned function is called, so a test can model a native that is slow to answer. Later enqueues wait behind it, as they do on a device. +iOS now keeps a new request through a power loss right after `mutate()` +resolves. Before, no rename into place was flushed to disk, so the request +directory could come back holding only `entry.json.tmp`, which nothing read: +the request was silently lost. Every atomic write on iOS (entries, data +bodies, outcome journal, task map, settings) now syncs its directory after +the rename, as Android does, and an entry's first save also syncs the queue +root. Form, file and chunked bodies are placed by a rename or move into the +request directory, and the entry save that follows syncs that directory +before `mutate()` resolves. A failed directory sync is logged and does not +fail the write. At +startup, iOS turns a lone `entry.json.tmp` that reads as a valid entry into +the entry, logs it, and runs the request as usual. A missing body then +settles `error` with `errorKind: 'file'`. A tmp that does not read is left +on disk. A tmp next to a readable `entry.json` is deleted. + ## 11.1.0 Added `takeDiagnostics()`. On Android it returns the module's recent diff --git a/README.md b/README.md index 6e84bf0d..97ce5524 100644 --- a/README.md +++ b/README.md @@ -268,8 +268,11 @@ while the app was dead from being sent twice. 1. **Write-ahead.** Entry, descriptor, and staged body persist before any attempt. `mutate()` resolves after the row and every staged body copy are - durably on disk (temp file plus rename), so the caller may delete its - source file then. A native failure rejects with a code: `E_INVALID`, + durably on disk (temp file, rename, then a sync of the directory), so the + caller may delete its source file then. A failed directory sync is logged + and does not fail the call. The library uses `fsync`, which on iOS does not + force the drive to empty its cache. A native failure rejects with a code: + `E_INVALID`, `E_RUNNING`, `E_FILE_MISSING`, or `E_STORAGE`. `E_INVALID` is malformed input native cannot send: a non-http(s) URL, header names or values the platform HTTP client rejects, a GET with a body, or parts that do not diff --git a/ios/FileIO.swift b/ios/FileIO.swift index e41690ec..bbe9e5e5 100644 --- a/ios/FileIO.swift +++ b/ios/FileIO.swift @@ -3,7 +3,9 @@ import Foundation // Small file helpers shared by the store, the journal, the task map and body // staging. Every durable write is tmp + fsync + rename, so a reader never sees // a half-written file: after a crash there is either the old file or the new -// one, plus at most a stray `.tmp` that the next write replaces. +// one, plus at most a stray `.tmp` that the next write replaces. The rename +// itself survives a power loss only after its directory is synced, so +// `writeAtomically` syncs the directory after every rename, as Android does. enum FileIO { struct IOError: LocalizedError { let message: String @@ -15,10 +17,20 @@ enum FileIO { } /// Writes `data` to `url.tmp`, flushes it to disk, then renames it onto `url`. + /// On a failure the tmp file is deleted, so a partly written file does not + /// hold space that a retry needs. static func writeAtomically(_ data: Data, to url: URL) throws { + // Runs last: after the rename, so the new name is what the sync makes + // durable. After a failure, it makes the tmp delete durable. + defer { syncDirectoryOrLog(url.deletingLastPathComponent()) } let tmp = tmpURL(for: url) - try writeSynced(data, to: tmp) - try rename(tmp, onto: url) + do { + try writeSynced(data, to: tmp) + try rename(tmp, onto: url) + } catch { + try? FileManager.default.removeItem(at: tmp) + throw error + } } /// Writes and fsyncs a file in place. Callers rename it afterwards. @@ -49,6 +61,33 @@ enum FileIO { FileManager.default.fileExists(atPath: url.path) } + /// Flushes a directory. Tests replace it to watch the syncs or fail them. + static var syncDirectory: (URL) throws -> Void = fsyncDirectory + + /// Flushes a directory, so the names that renames and new files put in it + /// survive a power loss. Plain fsync, as Android does. On Apple platforms + /// fsync does not make the drive empty its cache; F_FULLFSYNC would. + static func fsyncDirectory(_ url: URL) throws { + let fd = open(url.path, O_RDONLY) + guard fd >= 0 else { + throw IOError(message: "cannot open \(url.lastPathComponent) to sync it: errno \(errno)") + } + defer { close(fd) } + guard fsync(fd) == 0 else { + throw IOError(message: "fsync \(url.lastPathComponent) failed: errno \(errno)") + } + } + + /// A failed sync is logged, never thrown: the rename it follows already + /// happened, so the write still counts. + static func syncDirectoryOrLog(_ url: URL) { + do { + try syncDirectory(url) + } catch { + NSLog("[RNFileUploader] cannot sync \(url.lastPathComponent): \(error.localizedDescription)") + } + } + /// Creates a directory that holds device-local upload state and keeps it /// out of iCloud and iTunes backups. static func makeLocalDirectory(_ url: URL) { diff --git a/ios/QueueCoordinator.swift b/ios/QueueCoordinator.swift index b1bd7491..fc47f732 100644 --- a/ios/QueueCoordinator.swift +++ b/ios/QueueCoordinator.swift @@ -125,6 +125,9 @@ final class QueueCoordinator { queue.asyncAfter(deadline: .now() + .milliseconds(ms), execute: block) } settings = store.loadSettings() + // First: a promoted entry must load below, and never count as a + // directory with no entry. + store.promoteInterruptedSaves() finishSetAsideForgets() // Synchronous, before any session exists, so getRequests() is warm by // the time JS can call it, legacy rows included. diff --git a/ios/QueueStore.swift b/ios/QueueStore.swift index 048911e6..2f9229b7 100644 --- a/ios/QueueStore.swift +++ b/ios/QueueStore.swift @@ -14,6 +14,9 @@ import Foundation /// Next to the directories: `settings.json` and the `v10-imported` marker. /// Writes are tmp + fsync + rename. A corrupt or half-written file reads as /// absent. Synchronous on a private serial queue, like the v9 store. +/// +/// `entry.json.tmp` alone in a directory is an entry save that a crash or a +/// power loss cut short. Boot finishes it (`promoteInterruptedSaves`). final class QueueStore { static let shared = QueueStore( root: FileIO.applicationSupport().appendingPathComponent("RNFileUploaderChunked", isDirectory: true)) @@ -59,10 +62,21 @@ final class QueueStore { /// Throws on a write failure. An entry that did not persist must fail the /// enqueue. + /// + /// `writeAtomically` syncs the entry directory after the rename, so a + /// power loss cannot undo it: the first save would leave only + /// entry.json.tmp, and a later save would bring back an older entry. The + /// first save also syncs the root, which holds the new directory's name. + /// A failed sync is logged and the save still counts. If a power loss then + /// undoes a first save, boot promotes the lone tmp. func save(_ entry: QueueEntry) throws { try queue.sync { - try FileManager.default.createDirectory(at: dir(entry.id), withIntermediateDirectories: true) - try FileIO.writeAtomically(Self.encode(entry), to: fileURL(entry.id, Self.entryName)) + let d = dir(entry.id) + let url = d.appendingPathComponent(Self.entryName) + try FileManager.default.createDirectory(at: d, withIntermediateDirectories: true) + let first = !FileIO.exists(url) + try FileIO.writeAtomically(Self.encode(entry), to: url) + if first { FileIO.syncDirectoryOrLog(root) } } } @@ -78,9 +92,63 @@ final class QueueStore { } /// Deletes the id directory: entry, body, blob, part files, and a v9 - /// manifest. + /// manifest. A stray entry.json.tmp goes first, and that delete is synced: + /// a removal cut short must never leave the tmp alone, because boot would + /// promote it and bring the request back. func remove(_ id: String) { - queue.sync { _ = try? FileManager.default.removeItem(at: dir(id)) } + queue.sync { + let d = dir(id) + let tmp = FileIO.tmpURL(for: d.appendingPathComponent(Self.entryName)) + if (try? FileManager.default.removeItem(at: tmp)) != nil { FileIO.syncDirectoryOrLog(d) } + _ = try? FileManager.default.removeItem(at: d) + } + } + + // MARK: - Boot repair + + /// Runs at boot, before anything reads, counts or cleans the entry + /// directories. Finishes entry saves that a crash or a power loss cut + /// short. For each id directory that holds `entry.json.tmp`: + /// - No `entry.json`, and the tmp decodes to the entry of this directory: + /// the tmp is renamed onto `entry.json` and loads like any entry. The + /// app asked for the request, even when the process died before its + /// enqueue resolved. + /// - No `entry.json`, and the tmp does not decode: both stay as they are. + /// - A readable `entry.json`: the tmp is a later save that did not finish. + /// `entry.json` wins and the tmp is deleted. + /// - An unreadable `entry.json`: both stay as they are. + /// It reads nothing but `entry.json.tmp`, so v9 files stay as they are. It + /// skips set-aside directories, which are hidden. Returns the promoted ids. + @discardableResult + func promoteInterruptedSaves() -> [String] { + queue.sync { + var promoted: [String] = [] + for d in subdirectories() { + let url = d.appendingPathComponent(Self.entryName) + let tmp = FileIO.tmpURL(for: url) + guard FileIO.exists(tmp) else { continue } + if FileIO.exists(url) { + if Self.read(url) != nil { _ = try? FileManager.default.removeItem(at: tmp) } + continue + } + guard let e = Self.read(tmp), dir(e.id).lastPathComponent == d.lastPathComponent else { + NSLog("[RNFileUploader] \(d.lastPathComponent)/\(tmp.lastPathComponent) is not an entry of this " + + "directory; left as it is") + continue + } + do { + try FileIO.rename(tmp, onto: url) + } catch { + NSLog("[RNFileUploader] cannot recover entry \(e.id): \(error.localizedDescription)") + continue + } + FileIO.syncDirectoryOrLog(d) + FileIO.syncDirectoryOrLog(root) + promoted.append(e.id) + NSLog("[RNFileUploader] recovered entry \(e.id) from an entry save that did not finish") + } + return promoted + } } /// Deletes the id directory when it holds no `entry.json`: v9 files that @@ -214,9 +282,11 @@ final class QueueStore { /// Never a chunked blob (`blob`, `blob-`): that file was moved in, /// so it can be the only copy, and a same-id enqueue adopts it /// (`adoptableBlob`). Never a part file: its entry is gone, so its use is - /// not known. Never `entry.json.tmp`: `FileIO.writeAtomically` does not - /// sync the directory, so after a crash it can be the only trace of an - /// entry whose enqueue already resolved. + /// not known. Never `entry.json.tmp`: boot promotion + /// (`promoteInterruptedSaves`) runs before this cleanup, so a tmp still + /// here is one it could not use: it does not read as this directory's + /// entry, or its rename failed. It can still be the only trace of a + /// request, so the directory stays. static func isCopiedBodyFile(_ name: String) -> Bool { let range = NSRange(name.startIndex..., in: name) return copiedBody.firstMatch(in: name, range: range) != nil diff --git a/ios/Tests/EntrySaveDurabilityTests.swift b/ios/Tests/EntrySaveDurabilityTests.swift new file mode 100644 index 00000000..2242ce71 --- /dev/null +++ b/ios/Tests/EntrySaveDurabilityTests.swift @@ -0,0 +1,289 @@ +import XCTest +@testable import RNBGUCore + +/// Writes survive a power loss: the directory sync after each rename, and +/// the boot promotion of a lone entry.json.tmp. +final class EntrySaveDurabilityTests: XCTestCase { + private var h: Harness! + + override func setUp() { + h = Harness() + } + + override func tearDown() { + FileIO.syncDirectory = FileIO.fsyncDirectory + try? FileManager.default.removeItem(at: h.root) + } + + private func entryURL(_ id: String) -> URL { h.store.fileURL(id, QueueStore.entryName) } + private func tmpURL(_ id: String) -> URL { FileIO.tmpURL(for: entryURL(id)) } + + /// What a power loss before the directory sync can leave: the complete + /// tmp, and no entry.json. + private func undoRename(_ id: String) throws { + try FileManager.default.moveItem(at: entryURL(id), to: tmpURL(id)) + } + + private func errorKind(_ settled: [String: Any]?) -> String? { + (settled?["error"] as? [String: Any])?["errorKind"] as? String + } + + // MARK: - Directory sync after every atomic write + + func testAnAtomicWriteSyncsItsDirectoryAfterTheRename() throws { + let target = h.root.appendingPathComponent("file.json") + var synced: [URL] = [] + FileIO.syncDirectory = { dir in + synced.append(dir.standardizedFileURL) + XCTAssertTrue(FileIO.exists(target), "the rename happened before the sync") + XCTAssertFalse(FileIO.exists(FileIO.tmpURL(for: target))) + } + try FileIO.writeAtomically(Data("{}".utf8), to: target) + XCTAssertEqual(synced, [h.root.standardizedFileURL]) + } + + func testEveryEntrySaveSyncsTheEntryDirectoryAfterTheRenameAndTheFirstAlsoSyncsTheRoot() throws { + h.boot() + let dir = h.store.dir("a").standardizedFileURL + let root = h.store.root.standardizedFileURL + // For each sync of the entry directory or the root: which one, and + // whether entry.json and entry.json.tmp were there. + var synced: [(url: URL, entry: Bool, tmp: Bool)] = [] + FileIO.syncDirectory = { [unowned self] url in + let u = url.standardizedFileURL + guard u == dir || u == root else { return } + synced.append((u, FileIO.exists(self.entryURL("a")), FileIO.exists(self.tmpURL("a")))) + } + _ = try h.enqueue(h.dataRaw(id: "a")).get() + XCTAssertEqual(h.entry("a")?.state, .running) + // The staged body (no entry yet), the first save (then the root, which + // holds the new directory's name), and the issue's save. + XCTAssertEqual(synced.map(\.url), [dir, dir, root, dir]) + XCTAssertEqual(synced.map(\.entry), [false, true, true, true], "each entry sync follows its rename") + XCTAssertEqual(synced.map(\.tmp), [false, false, false, false]) + } + + func testAJournalWriteIsSyncedBeforeTheCompletedEntrySave() throws { + h.boot() + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let task = try XCTUnwrap(h.transport.live.first) + let journalDir = h.root.appendingPathComponent("events").standardizedFileURL + let dir = h.store.dir("a").standardizedFileURL + // In order: each journal sync with the outcome files there, and each + // entry sync with the entry's state. + var synced: [String] = [] + FileIO.syncDirectory = { [unowned self] url in + let u = url.standardizedFileURL + if u == journalDir { + let files = (try? FileManager.default.contentsOfDirectory(atPath: u.path)) ?? [] + synced.append("journal:" + files.filter { $0.hasSuffix(".json") }.count.description) + } else if u == dir { + // Read the file directly: the sync runs inside the store's queue. + let saved = (try? Data(contentsOf: self.entryURL("a"))) + .flatMap { try? JSONDecoder().decode(QueueEntry.self, from: $0) } + synced.append("entry:" + (saved?.state.rawValue ?? "none")) + } + } + h.complete(task) + XCTAssertEqual(h.entry("a")?.state, .completed) + let journal = try XCTUnwrap(synced.firstIndex(of: "journal:1"), "the journal file is synced after its rename") + let completed = try XCTUnwrap(synced.firstIndex(of: "entry:completed")) + XCTAssertLessThan(journal, completed, "the outcome is durable before the completed row") + } + + func testAFailedDirectorySyncFailsNeitherTheSaveNorTheSettle() throws { + var failures = 0 + FileIO.syncDirectory = { _ in + failures += 1 + throw FileIO.IOError(message: "fsync failed") + } + XCTAssertEqual(try h.enqueue(h.dataRaw(id: "a")).get(), "a", "mutate() still resolves") + XCTAssertGreaterThan(failures, 0) + XCTAssertNotNil(QueueStore(root: h.store.root).load("a"), "the entry is on disk") + h.boot() + h.complete(try XCTUnwrap(h.transport.live.first)) + XCTAssertEqual(h.entry("a")?.state, .completed) + XCTAssertEqual(h.sink.settled.last?["kind"] as? String, "completed") + XCTAssertEqual(h.unacknowledged().count, 1, "the outcome is journaled") + } + + func testTheRealDirectorySyncWorksAndAMissingDirectoryThrows() throws { + XCTAssertNoThrow(try FileIO.fsyncDirectory(h.store.root)) + XCTAssertThrowsError(try FileIO.fsyncDirectory(h.root.appendingPathComponent("missing"))) + } + + func testAFailedWriteDeletesItsTmp() throws { + let target = h.root.appendingPathComponent("target") + // A non-empty directory at the target makes the rename fail. + writeFile(target.appendingPathComponent("x"), "x") + XCTAssertThrowsError(try FileIO.writeAtomically(Data("{}".utf8), to: target)) + XCTAssertFalse(FileIO.exists(FileIO.tmpURL(for: target))) + } + + func testAFailedWriteSyncsItsDirectoryOnceAfterTheTmpIsGone() throws { + let target = h.root.appendingPathComponent("target") + let tmp = FileIO.tmpURL(for: target) + // A non-empty directory at the target makes the rename fail. + writeFile(target.appendingPathComponent("x"), "x") + var synced: [(url: URL, tmp: Bool)] = [] + FileIO.syncDirectory = { synced.append(($0.standardizedFileURL, FileIO.exists(tmp))) } + XCTAssertThrowsError(try FileIO.writeAtomically(Data("{}".utf8), to: target)) + XCTAssertFalse(FileIO.exists(tmp)) + XCTAssertEqual(synced.map(\.url), [h.root.standardizedFileURL], "the parent, once") + XCTAssertEqual(synced.map(\.tmp), [false], "the tmp delete is what the sync makes durable") + } + + func testARejectedFirstSaveLeavesNoTmpForBootToPromote() throws { + writeFile(entryURL("a").appendingPathComponent("x"), "x") // the rename onto entry.json fails + guard case .failure(let error) = h.enqueue(h.dataRaw(id: "a")) else { return XCTFail("enqueue resolved") } + XCTAssertEqual(error.code, "E_STORAGE") + XCTAssertFalse(FileIO.exists(tmpURL("a"))) + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertNil(fresh.entry("a"), "a rejected request does not come back") + XCTAssertTrue(fresh.transport.created.isEmpty) + } + + // MARK: - Boot promotion + + func testALoneTmpIsPromotedAtBootAndTheRequestRuns() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try undoRename("a") + let fresh = Harness(root: h.root) + XCTAssertTrue(FileIO.exists(entryURL("a")), "promoted before the index loads") + XCTAssertFalse(FileIO.exists(tmpURL("a"))) + XCTAssertEqual(fresh.row("a")?["state"] as? String, "queued") + fresh.boot() + let task = try XCTUnwrap(fresh.transport.live.first) + XCTAssertEqual(fresh.entry("a")?.attempts, 1) + fresh.complete(task) + XCTAssertEqual(fresh.entry("a")?.state, .completed) + XCTAssertEqual(fresh.sink.settled.last?["kind"] as? String, "completed", "the normal outcome path") + } + + func testPromotionSyncsTheEntryDirectoryAndTheRoot() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try undoRename("a") + let store = QueueStore(root: h.store.root) + var synced: [URL] = [] + FileIO.syncDirectory = { synced.append($0.standardizedFileURL) } + XCTAssertEqual(store.promoteInterruptedSaves(), ["a"]) + XCTAssertEqual(synced, [store.dir("a").standardizedFileURL, store.root.standardizedFileURL]) + XCTAssertEqual(store.promoteInterruptedSaves(), [], "a second boot finds nothing to do") + } + + func testAPromotedEntryTakesARepeatEnqueueAsAnExistingEntry() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try undoRename("a") + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertEqual(fresh.transport.live.count, 1) + // The app never saw the first enqueue resolve, so it calls again. + XCTAssertEqual(try fresh.enqueue(fresh.dataRaw(id: "a")).get(), "a") + XCTAssertEqual(fresh.transport.live.count, 1, "the running attempt keeps going") + XCTAssertEqual(fresh.entry("a")?.attempts, 1) + XCTAssertEqual(fresh.entry("a")?.generation, 1) + } + + func testAPromotedEntryWithAMissingBodySettlesAsAFileError() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + let body = try XCTUnwrap(h.store.bodyURL(try XCTUnwrap(h.entry("a")))) + try FileManager.default.removeItem(at: body) + try undoRename("a") + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertEqual(fresh.entry("a")?.state, .error) + XCTAssertEqual(errorKind(fresh.sink.settled.last), "file") + XCTAssertTrue(fresh.transport.created.isEmpty) + fresh.advance(60_000) + XCTAssertEqual(fresh.sink.settled.count, 1, "settled once, no loop") + } + + func testAPromotedChunkedEntryWithAMissingBlobSettlesAsAFileError() throws { + _ = try h.enqueue(h.chunkedRaw(id: "c")).get() + let blob = try XCTUnwrap(h.store.bodyURL(try XCTUnwrap(h.entry("c")))) + try FileManager.default.removeItem(at: blob) + try undoRename("c") + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertEqual(fresh.entry("c")?.state, .error) + XCTAssertEqual(errorKind(fresh.sink.settled.last), "file") + XCTAssertTrue(fresh.transport.created.isEmpty) + } + + func testATmpThatDoesNotDecodeIsLeftAlone() throws { + writeFile(tmpURL("a"), #"{"id":"a","state":"#) // cut short + writeFile(h.store.fileURL("a", "body-1"), "{}") + let fresh = Harness(root: h.root) + fresh.boot() + XCTAssertNil(fresh.entry("a")) + XCTAssertFalse(FileIO.exists(entryURL("a"))) + XCTAssertEqual(try String(contentsOf: tmpURL("a")), #"{"id":"a","state":"#) + XCTAssertTrue(FileIO.exists(h.store.fileURL("a", "body-1")), "nothing is deleted") + } + + func testATmpOfAnotherIdIsLeftAlone() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try undoRename("a") + // The same file under the directory of another id. + let other = h.store.dir("b").appendingPathComponent(QueueStore.entryName + ".tmp") + try FileManager.default.createDirectory(at: h.store.dir("b"), withIntermediateDirectories: true) + try FileManager.default.moveItem(at: tmpURL("a"), to: other) + let fresh = Harness(root: h.root) + XCTAssertNil(fresh.entry("a")) + XCTAssertNil(fresh.entry("b")) + XCTAssertTrue(FileIO.exists(other)) + } + + func testATmpNextToAReadableEntryIsDeletedAndTheEntryWins() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + var later = try XCTUnwrap(h.entry("a")) + later.state = .cancelled // a later save that did not finish + try QueueStore.encode(later).write(to: tmpURL("a")) + let fresh = Harness(root: h.root) + XCTAssertEqual(fresh.entry("a")?.state, .queued, "entry.json wins") + XCTAssertFalse(FileIO.exists(tmpURL("a")), "so a later removal cannot leave it alone") + fresh.boot() + XCTAssertEqual(fresh.transport.live.count, 1) + } + + func testATmpNextToAnUnreadableEntryIsLeftAlone() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try FileManager.default.copyItem(at: entryURL("a"), to: tmpURL("a")) + try Data("{".utf8).write(to: entryURL("a")) + let fresh = Harness(root: h.root) + XCTAssertNil(fresh.entry("a")) + XCTAssertTrue(FileIO.exists(tmpURL("a"))) + XCTAssertEqual(try String(contentsOf: entryURL("a")), "{") + } + + func testRemoveDeletesAStrayTmpFirstAndSyncsThatDelete() throws { + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try FileManager.default.copyItem(at: entryURL("a"), to: tmpURL("a")) + var remainingAtSync: [String] = [] + FileIO.syncDirectory = { [unowned self] dir in + remainingAtSync = (try? FileManager.default.contentsOfDirectory(atPath: dir.path)) ?? [] + XCTAssertEqual(dir.standardizedFileURL, self.h.store.dir("a").standardizedFileURL) + } + h.store.remove("a") + XCTAssertTrue(remainingAtSync.contains(QueueStore.entryName), "the tmp went before entry.json") + XCTAssertFalse(remainingAtSync.contains(QueueStore.entryName + ".tmp")) + XCTAssertFalse(FileIO.exists(h.store.dir("a"))) + } + + func testSetAsideAndV9DirectoriesAreNotTouched() throws { + // A v9 directory: a manifest and its blob, no entry file. + let v9 = h.store.dir("v9") + writeFile(v9.appendingPathComponent(QueueStore.manifestName), "{}") + writeFile(v9.appendingPathComponent(ChunkedManifestV9.blobName), bytes: 10) + // A set-aside directory holding a lone tmp. + _ = try h.enqueue(h.dataRaw(id: "a")).get() + try undoRename("a") + let aside = h.store.root.appendingPathComponent(".forget-x", isDirectory: true) + try FileManager.default.moveItem(at: h.store.dir("a"), to: aside) + let before = try FileManager.default.contentsOfDirectory(atPath: v9.path).sorted() + XCTAssertEqual(QueueStore(root: h.store.root).promoteInterruptedSaves(), []) + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: v9.path).sorted(), before) + XCTAssertTrue(FileIO.exists(aside.appendingPathComponent(QueueStore.entryName + ".tmp"))) + } +}