From bfe54daf07b80ce6e17d903f9cb60d5aa575cac8 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 16 Nov 2026 11:24:47 -0800 Subject: [PATCH 1/4] refactor(client-runtime): share the Effect attachment upload registry with web The keyed upload registry moves from mobile into client-runtime with per-environment concurrency, explicit holds, joining, and interruption that waits for a transfer to settle. runAttachmentUploadCycle and verifyPersistedAttachmentUpload are Effects: interrupting the cycle aborts the transfer and deletes the fresh mint, replacing onMinted cancel and the abort handoff. Mobile configures the registry for its composer. Web keeps its upload policy (previous uploads across environment switches, release deletes, reconnect retries, expiry checks) but runs each upload as a registry hold, replacing its job map, pump, settled promises and cancel flags. Co-Authored-By: Claude Opus 5.5 --- apps/mobile/src/lib/attachmentUpload.test.ts | 31 +- apps/mobile/src/lib/attachmentUpload.ts | 81 ++-- .../lib/composerAttachmentUploadQueue.test.ts | 246 ----------- .../src/lib/composerAttachmentUploadQueue.ts | 212 ++-------- apps/mobile/src/state/attachments.test.ts | 50 ++- .../src/state/composer-attachment-uploads.ts | 25 +- apps/mobile/src/state/thread-outbox-model.ts | 2 +- .../threadOutboxDelivery.permissions.test.ts | 6 +- apps/web/src/lib/attachmentUploadQueue.ts | 391 ++++++++---------- packages/client-runtime/package.json | 4 + .../src/state/attachmentUploads.test.ts | 289 +++++++++++++ .../src/state/attachmentUploads.ts | 250 +++++++++++ .../src/state/attachments.test.ts | 135 +++--- .../client-runtime/src/state/attachments.ts | 162 ++++---- 14 files changed, 1003 insertions(+), 881 deletions(-) create mode 100644 packages/client-runtime/src/state/attachmentUploads.test.ts create mode 100644 packages/client-runtime/src/state/attachmentUploads.ts diff --git a/apps/mobile/src/lib/attachmentUpload.test.ts b/apps/mobile/src/lib/attachmentUpload.test.ts index fb7fa933fbca9..9c58c9a3b05f6 100644 --- a/apps/mobile/src/lib/attachmentUpload.test.ts +++ b/apps/mobile/src/lib/attachmentUpload.test.ts @@ -93,7 +93,6 @@ vi.mock("expo-file-system", () => ({ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; -import { uploadError } from "./composerAttachmentUploadQueue"; import { prepareTurnAttachments as prepareWithUploads, releasePendingAttachmentUploads, @@ -109,30 +108,14 @@ function prepare( ) { return prepareWithUploads({ ...input, - // An interrupted send waits for its transfer to settle, as the shared registry does. uploadAttachment: (attachment) => - Effect.suspend(() => { - const controller = new AbortController(); - const upload = uploadComposerAttachment( - input.environmentId, - attachment, - controller.signal, - input.onUploadProgress - ? (progress) => input.onUploadProgress?.(attachment.id, progress) - : undefined, - ); - return Effect.tryPromise({ try: () => upload, catch: uploadError }).pipe( - Effect.onInterrupt(() => - Effect.promise(() => { - controller.abort(); - return upload.then( - () => undefined, - () => undefined, - ); - }), - ), - ); - }), + uploadComposerAttachment( + input.environmentId, + attachment, + input.onUploadProgress + ? (progress) => input.onUploadProgress?.(attachment.id, progress) + : undefined, + ), }); } import type { DraftComposerAttachment } from "./composerImages"; diff --git a/apps/mobile/src/lib/attachmentUpload.ts b/apps/mobile/src/lib/attachmentUpload.ts index b4bf8b0aeb472..e51005bd1e474 100644 --- a/apps/mobile/src/lib/attachmentUpload.ts +++ b/apps/mobile/src/lib/attachmentUpload.ts @@ -32,7 +32,11 @@ import { } from "./composerImages"; import { imageMimeType } from "@t3tools/shared/image"; import { uuidv4 } from "./uuid"; -import { type ComposerAttachmentUploadError, uploadError } from "./composerAttachmentUploadQueue"; +import { + abortableTransfer, + attachmentUploadError, + type AttachmentUploadError, +} from "@t3tools/client-runtime/state/attachment-uploads"; export type UploadedMobileAttachment = | UploadChatImageAttachment @@ -257,40 +261,46 @@ function requireUploadAccess(environmentId: EnvironmentId): void { } /** - * Uploads one attachment and resolves to its uploaded id, reusing an earlier upload the server - * still has. An abort deletes the pending upload it created. + * Uploads one attachment and returns its uploaded id, reusing an earlier upload the server + * still has. Interrupting it aborts the native transfer and deletes the pending upload it minted; + * a failed transfer deletes it too. */ -export async function uploadComposerAttachment( +export const uploadComposerAttachment = Effect.fn("uploadComposerAttachment")(function* ( environmentId: EnvironmentId, attachment: DraftComposerAttachment, - signal: AbortSignal, onProgress?: (progress: number) => void, -): Promise { - requireUploadAccess(environmentId); +) { + yield* Effect.try({ + try: () => requireUploadAccess(environmentId), + catch: attachmentUploadError, + }); if ( attachment.uploadEnvironmentId === environmentId && attachment.uploadedAttachmentId !== undefined ) { - const verification = await verifyPersistedAttachmentUpload({ + const verification = yield* verifyPersistedAttachmentUpload({ registry: appAtomRegistry, createAssetUrl: assetEnvironment.createUrl, environmentId, attachmentId: attachment.uploadedAttachmentId, }); - if (verification.status === "failed") throw verification.error; + if (verification.status === "failed") { + return yield* attachmentUploadError(verification.error); + } if (verification.status === "verified") return attachment.uploadedAttachmentId; } - if (signal.aborted) throw new Error("Upload cancelled."); - requireUploadAccess(environmentId); + yield* Effect.try({ + try: () => requireUploadAccess(environmentId), + catch: attachmentUploadError, + }); if ( Option.isNone( appAtomRegistry.get(environmentSession.preparedConnectionValueAtom(environmentId)), ) ) { - throw new Error("The environment is not connected."); + return yield* attachmentUploadError(new Error("The environment is not connected.")); } - let minted: string | null = null; - const result = await runAttachmentUploadCycle({ + const result = yield* runAttachmentUploadCycle({ registry: appAtomRegistry, createUploadUrl: attachmentEnvironment.createUploadUrl, remove: attachmentEnvironment.remove, @@ -298,7 +308,7 @@ export async function uploadComposerAttachment( upload: attachmentUploadInput(attachment), // Resolve at transfer time; reconnects may change the base URL. resolveUploadUrl: (relativeUrl) => { - requireUploadAccess(environmentId); + if (!readEnvironmentScope(environmentId, AuthOrchestrationOperateScope)) return null; const connection = appAtomRegistry.get( environmentSession.preparedConnectionValueAtom(environmentId), ); @@ -306,22 +316,26 @@ export async function uploadComposerAttachment( ? null : resolveAssetUrl(connection.value.httpBaseUrl, relativeUrl); }, - transport: (url) => ({ - done: uploadFileBytes(environmentId, attachment, url, signal, onProgress), - abort: () => {}, - }), - onMinted: (attachmentId) => { - if (signal.aborted) return "cancel"; - minted = attachmentId; - return "continue"; - }, + transport: (url) => + abortableTransfer((signal) => + uploadFileBytes(environmentId, attachment, url, signal, onProgress), + ), }); if (result.status === "uploaded") return result.attachmentId; - if (minted !== null) await releaseCreatedUploadsQuietly(environmentId, [minted]); - throw result.status === "failed" && result.error !== undefined - ? result.error - : new Error(`Upload failed for '${attachment.name}'.`); -} + if (result.attachmentId !== null) { + yield* Effect.promise(() => + releaseCreatedUploadsQuietly(environmentId, [result.attachmentId!]), + ); + } + return yield* attachmentUploadError( + result.step === "resolve-url" && + !readEnvironmentScope(environmentId, AuthOrchestrationOperateScope) + ? new Error("This connection cannot upload attachments.") + : result.error instanceof Error + ? result.error + : new Error(`Upload failed for '${attachment.name}'.`), + ); +}); /** * Turns a message's attachments into wire references. `uploadAttachment` is the shared upload @@ -335,7 +349,7 @@ export const prepareTurnAttachments = Effect.fn("prepareTurnAttachments")(functi readonly supportsImageUploads?: boolean; readonly uploadAttachment: ( attachment: DraftComposerAttachment, - ) => Effect.Effect; + ) => Effect.Effect; readonly persistUploadedReferences?: ( draftAttachments: ReadonlyArray, ) => Effect.Effect<"persisted" | "abandon">; @@ -349,7 +363,7 @@ export const prepareTurnAttachments = Effect.fn("prepareTurnAttachments")(functi (attachment): attachment is DraftComposerImageAttachment => attachment.type === "image", ), ), - catch: uploadError, + catch: attachmentUploadError, }); const files = input.attachments.filter((attachment) => attachment.type === "file"); if (input.attachments.length === 0 || (files.length === 0 && !input.supportsImageUploads)) { @@ -362,7 +376,10 @@ export const prepareTurnAttachments = Effect.fn("prepareTurnAttachments")(functi } satisfies PreparedTurnAttachments as PrepareTurnAttachmentsResult; } - yield* Effect.try({ try: () => requireUploadAccess(environmentId), catch: uploadError }); + yield* Effect.try({ + try: () => requireUploadAccess(environmentId), + catch: attachmentUploadError, + }); const uploadedAttachments: UploadedMobileAttachment[] = []; const pendingAttachmentIds: string[] = []; for (const attachment of input.attachments) { diff --git a/apps/mobile/src/lib/composerAttachmentUploadQueue.test.ts b/apps/mobile/src/lib/composerAttachmentUploadQueue.test.ts index 9a65e348a28d0..cfff340939a65 100644 --- a/apps/mobile/src/lib/composerAttachmentUploadQueue.test.ts +++ b/apps/mobile/src/lib/composerAttachmentUploadQueue.test.ts @@ -1,6 +1,4 @@ import { ApprovalRequestId, EnvironmentId, ThreadId } from "@t3tools/contracts"; -import * as Effect from "effect/Effect"; -import * as Fiber from "effect/Fiber"; import { describe, expect, it } from "@effect/vitest"; import { vi } from "vite-plus/test"; import { @@ -14,9 +12,7 @@ import { composerAttachmentsStillUploading, composerAttachmentUploadKey, composerDraftEnvironmentId, - createComposerAttachmentUploadQueue, type ComposerAttachmentUploadRequest, - type ComposerAttachmentUploadState, } from "./composerAttachmentUploadQueue"; const environmentId = EnvironmentId.make("environment-1"); @@ -34,248 +30,6 @@ function request(id: string, environment = environmentId): ComposerAttachmentUpl }; } -describe("composer attachment upload queue", () => { - it("bounds concurrency, deduplicates updates, and drains all attachments", async () => { - const gates = new Map>>(); - const fourthStarted = Promise.withResolvers(); - const firstThreeStarted = Promise.withResolvers(); - let active = 0; - let maximum = 0; - const upload = vi.fn(async (input: ComposerAttachmentUploadRequest) => { - active += 1; - maximum = Math.max(maximum, active); - const gate = Promise.withResolvers(); - gates.set(input.attachment.id, gate); - if (gates.size === 3) firstThreeStarted.resolve(); - if (gates.size === 4) fourthStarted.resolve(); - try { - return await gate.promise; - } finally { - active -= 1; - } - }); - const queue = createComposerAttachmentUploadQueue({ upload, onChange: () => {} }); - const requests = [request("one"), request("two"), request("three"), request("four")]; - queue.sync(requests); - queue.sync(requests); - await firstThreeStarted.promise; - expect(upload).toHaveBeenCalledTimes(3); - gates.get("one")!.resolve("uploaded"); - await fourthStarted.promise; - for (const gate of gates.values()) gate.resolve("uploaded"); - await queue.settled(); - queue.sync(requests); - await queue.settled(); - expect(maximum).toBe(3); - expect(upload).toHaveBeenCalledTimes(4); - queue.dispose(); - }); - - it("cancels on disconnect and resumes from the same local draft on reconnect", async () => { - const started = Promise.withResolvers(); - let states: Readonly> = {}; - let signal: AbortSignal | undefined; - const upload = vi.fn( - async (_request: ComposerAttachmentUploadRequest, currentSignal: AbortSignal) => { - signal = currentSignal; - started.resolve(); - return new Promise((resolve) => - currentSignal.addEventListener("abort", () => resolve("aborted"), { once: true }), - ); - }, - ); - const queue = createComposerAttachmentUploadQueue({ - upload, - onChange: (next) => { - states = next; - }, - }); - const local = request("offline-draft"); - queue.sync([local]); - await started.promise; - queue.sync([]); - await queue.settled(); - expect(signal?.aborted).toBe(true); - expect(states).toEqual({}); - upload.mockResolvedValueOnce("uploaded"); - queue.sync([local]); - await queue.settled(); - expect(upload.mock.calls[1]?.[0]).toBe(local); - expect(states[composerAttachmentUploadKey(environmentId, local.attachment.id)]).toEqual({ - status: "ready", - }); - expect(local.attachment).toMatchObject({ fileUri: "file:///documents/offline-draft.pdf" }); - queue.dispose(); - }); - - it("retries a failed attachment when the worker restores requests after reconnect", async () => { - let states: Readonly> = {}; - const upload = vi - .fn() - .mockRejectedValueOnce(new Error("Disconnected")) - .mockResolvedValue("uploaded"); - const queue = createComposerAttachmentUploadQueue({ - upload, - onChange: (next) => { - states = next; - }, - }); - const local = request("failed-upload"); - const key = composerAttachmentUploadKey(environmentId, local.attachment.id); - queue.sync([local]); - await queue.settled(); - expect(states[key]?.status).toBe("failed"); - queue.sync([local]); - await queue.settled(); - expect(upload).toHaveBeenCalledTimes(1); - queue.sync([]); - queue.sync([local]); - await queue.settled(); - expect(upload).toHaveBeenCalledTimes(2); - expect(states[key]?.status).toBe("ready"); - queue.dispose(); - }); - - it("ignores a late completion after removal or environment switch", async () => { - const gate = Promise.withResolvers(); - const started = Promise.withResolvers(); - let states: Readonly> = {}; - const upload = vi.fn(async () => { - started.resolve(); - return gate.promise; - }); - const queue = createComposerAttachmentUploadQueue({ - upload, - onChange: (next) => { - states = next; - }, - }); - queue.sync([request("photo")]); - await started.promise; - upload.mockResolvedValueOnce("uploaded"); - const other = EnvironmentId.make("environment-2"); - queue.sync([request("photo", other)]); - gate.resolve("uploaded"); - await queue.settled(); - expect(states).toEqual({ [composerAttachmentUploadKey(other, "photo")]: { status: "ready" } }); - queue.sync([]); - expect(states).toEqual({}); - queue.dispose(); - }); - - it("restarts a re-added attachment after its aborted transfer finishes settling", async () => { - const firstStarted = Promise.withResolvers(); - const firstSettled = Promise.withResolvers(); - const secondStarted = Promise.withResolvers(); - const secondSettled = Promise.withResolvers(); - let states: Readonly> = {}; - let firstSignal: AbortSignal | undefined; - const upload = vi.fn(async (_request: ComposerAttachmentUploadRequest, signal: AbortSignal) => { - if (!firstSignal) { - firstSignal = signal; - firstStarted.resolve(); - return firstSettled.promise; - } - secondStarted.resolve(); - return secondSettled.promise; - }); - const queue = createComposerAttachmentUploadQueue({ - upload, - onChange: (next) => { - states = next; - }, - }); - const local = request("re-added"); - queue.sync([local]); - await firstStarted.promise; - queue.sync([]); - queue.sync([local]); - expect(firstSignal?.aborted).toBe(true); - expect(upload).toHaveBeenCalledOnce(); - firstSettled.resolve("aborted"); - await secondStarted.promise; - expect(upload).toHaveBeenCalledTimes(2); - secondSettled.resolve("uploaded"); - await queue.settled(); - expect(states[composerAttachmentUploadKey(environmentId, local.attachment.id)]).toEqual({ - status: "ready", - }); - queue.dispose(); - }); - - it("keeps failures stable until retry and reports bounded progress", async () => { - let states: Readonly> = {}; - const progress: number[] = []; - const upload = vi.fn( - async ( - _request: ComposerAttachmentUploadRequest, - _signal: AbortSignal, - report: (value: number) => void, - ): Promise => { - report(0.12); - report(0.13); - report(1.1); - throw new Error("Server unavailable"); - }, - ); - const queue = createComposerAttachmentUploadQueue({ - upload, - onChange: (next) => { - states = next; - const state = next[composerAttachmentUploadKey(environmentId, "file")]; - if (state?.status === "uploading") progress.push(state.progress); - }, - }); - queue.sync([request("file")]); - await queue.settled(); - queue.sync([request("file")]); - expect(upload).toHaveBeenCalledOnce(); - expect(states[composerAttachmentUploadKey(environmentId, "file")]).toEqual({ - status: "failed", - reason: "Server unavailable", - }); - expect(progress).toEqual([0, 0.1, 1]); - upload.mockImplementationOnce(async () => "uploaded"); - queue.retry(environmentId, "file"); - await queue.settled(); - expect(states[composerAttachmentUploadKey(environmentId, "file")]).toEqual({ status: "ready" }); - queue.dispose(); - }); - - it.effect("joins a send to the running upload instead of uploading again", () => - Effect.gen(function* () { - const gate = Promise.withResolvers(); - const started = Promise.withResolvers(); - const uploaded = vi.fn(); - const upload = vi.fn(async () => { - started.resolve(); - return gate.promise; - }); - let states: Readonly> = {}; - const queue = createComposerAttachmentUploadQueue({ - upload, - onUploaded: uploaded, - onChange: (next) => { - states = next; - }, - }); - const local = request("video"); - queue.sync([local]); - yield* Effect.promise(() => started.promise); - // Sending hands the attachment from the draft to the outbox, which uploads it too. - const sent = yield* Effect.forkChild(queue.upload(local), { startImmediately: true }); - queue.sync([]); - gate.resolve("uploaded-1"); - expect(yield* Fiber.join(sent)).toBe("uploaded-1"); - expect(upload).toHaveBeenCalledOnce(); - expect(uploaded).toHaveBeenCalledExactlyOnceWith(local, "uploaded-1"); - yield* Effect.promise(() => queue.settled()); - expect(states).toEqual({}); - queue.dispose(); - }), - ); -}); - describe("draft upload scope and offline submission", () => { it("uploads question drafts in their environment without matching other thread prefixes", () => { const requestId = ApprovalRequestId.make("request:1"); diff --git a/apps/mobile/src/lib/composerAttachmentUploadQueue.ts b/apps/mobile/src/lib/composerAttachmentUploadQueue.ts index 92434c56b391d..5e6318a31c62a 100644 --- a/apps/mobile/src/lib/composerAttachmentUploadQueue.ts +++ b/apps/mobile/src/lib/composerAttachmentUploadQueue.ts @@ -1,14 +1,11 @@ import { EnvironmentId, type ServerConfig } from "@t3tools/contracts"; -import * as Effect from "effect/Effect"; -import * as Equal from "effect/Equal"; -import * as Exit from "effect/Exit"; -import * as Fiber from "effect/Fiber"; -import * as Hash from "effect/Hash"; -import * as RcMap from "effect/RcMap"; -import * as Schema from "effect/Schema"; -import * as Scope from "effect/Scope"; -import * as Semaphore from "effect/Semaphore"; +import type * as Effect from "effect/Effect"; import { clampFileAttachmentUploadBytes } from "@t3tools/client-runtime/state/attachments"; +import { + createAttachmentUploadRegistry, + type AttachmentUploadError, + type AttachmentUploadStatus, +} from "@t3tools/client-runtime/state/attachment-uploads"; import { parseLegacyNewTaskDraftKey } from "../state/new-task-draft-key"; import type { DraftComposerAttachment } from "./composerImages"; @@ -18,10 +15,7 @@ export interface ComposerAttachmentUploadRequest { readonly attachment: DraftComposerAttachment; } -export type ComposerAttachmentUploadState = - | { readonly status: "uploading"; readonly progress: number } - | { readonly status: "ready" } - | { readonly status: "failed"; readonly reason: string }; +export type ComposerAttachmentUploadState = AttachmentUploadStatus; export function composerAttachmentUploadKey( environmentId: EnvironmentId, @@ -114,189 +108,37 @@ export function composerAttachmentsStillUploading(input: { }); } -/** Equal by environment and attachment id; the request rides along for the upload. */ -class UploadKey implements Equal.Equal { - constructor( - readonly id: string, - readonly request: ComposerAttachmentUploadRequest | null, - ) {} - static of(request: ComposerAttachmentUploadRequest): UploadKey { - return new UploadKey( - composerAttachmentUploadKey(request.environmentId, request.attachment.id), - request, - ); - } - [Equal.symbol](that: unknown): boolean { - return that instanceof UploadKey && that.id === this.id; - } - [Hash.symbol](): number { - return Hash.string(this.id); - } -} - /** - * Every upload of a composer attachment, shared by key: the composer holds the attachments it - * shows or has queued, and a send joins the upload already running instead of starting another. - * An upload is interrupted once nothing holds it. + * The composer's uploads, on the shared registry: keyed by environment and attachment, three at + * a time per environment. */ export function createComposerAttachmentUploadQueue(options: { + /** Uploads one attachment; interrupting it must abort the transfer and wait for it to settle. */ readonly upload: ( request: ComposerAttachmentUploadRequest, - signal: AbortSignal, - onProgress: (progress: number) => void, - ) => Promise; + reportProgress: (progress: number) => void, + ) => Effect.Effect; /** Saves a finished upload's id; the upload is ready once this succeeds. */ readonly onUploaded?: ( request: ComposerAttachmentUploadRequest, uploadedId: string, - ) => Effect.Effect; + ) => Effect.Effect; readonly onChange: (states: Readonly>) => void; }) { - let states: Readonly> = {}; - function setState(key: string, state: ComposerAttachmentUploadState | undefined) { - const previous = states[key]; - if ( - previous === state || - (previous?.status === "uploading" && - state?.status === "uploading" && - previous.progress === state.progress) - ) - return; - const next = { ...states }; - if (state) next[key] = state; - else delete next[key]; - states = next; - options.onChange(states); - } - - const permits = Semaphore.makeUnsafe(3); - const running = new Set>(); - // The latest transfer per attachment: a re-added attachment waits for an aborted one to settle. - const transfers = new Map>(); - - /** The native transfer; interrupting it aborts the request and waits for it to settle. */ - const nativeTransfer = (key: UploadKey) => - Effect.suspend(() => { - const controller = new AbortController(); - if (states[key.id]?.status !== "uploading") { - setState(key.id, { status: "uploading", progress: 0 }); - } - const done = options.upload(key.request!, controller.signal, (progress) => { - if (controller.signal.aborted) return; - setState(key.id, { - status: "uploading", - progress: Math.floor(Math.max(0, Math.min(1, progress)) * 20) / 20, - }); - }); - return Effect.tryPromise({ try: () => done, catch: uploadError }).pipe( - Effect.onInterrupt(() => - Effect.sync(() => controller.abort()).pipe( - Effect.andThen(Effect.tryPromise({ try: () => done, catch: uploadError })), - Effect.ignore, - ), - ), - ); - }); - - const transfer = (key: UploadKey, previous: Fiber.Fiber | undefined) => - (previous === undefined ? Effect.void : Fiber.await(previous)).pipe( - Effect.andThen(nativeTransfer(key).pipe(Semaphore.withPermits(permits, 1))), - Effect.tap((uploadedId) => options.onUploaded?.(key.request!, uploadedId) ?? Effect.void), - Effect.tap(() => Effect.sync(() => setState(key.id, { status: "ready" }))), - Effect.tapError((error) => - Effect.sync(() => setState(key.id, { status: "failed", reason: error.message })), - ), - ); - - const scope = Scope.makeUnsafe(); - const uploads = Effect.runSync( - Scope.provide(scope)( - RcMap.make({ - lookup: Effect.fn("ComposerAttachmentUploads.lookup")(function* (key: UploadKey) { - setState(key.id, { status: "uploading", progress: 0 }); - yield* Effect.addFinalizer(() => Effect.sync(() => setState(key.id, undefined))); - const fiber = yield* Effect.forkScoped(transfer(key, transfers.get(key.id))); - transfers.set(key.id, fiber); - running.add(fiber); - fiber.addObserver(() => { - running.delete(fiber); - if (transfers.get(key.id) === fiber) transfers.delete(key.id); - }); - return fiber; - }), - }), - ), - ); - - const holders = new Map(); - let requested: ReadonlyArray = []; - let disposed = false; - const release = (id: string) => { - const holder = holders.get(id); - if (!holder) return; - holders.delete(id); - Effect.runFork(Scope.close(holder, Exit.void)); - }; - - /** Holds exactly these uploads; anything no longer requested and held nowhere else stops. */ - function sync(requests: ReadonlyArray) { - if (disposed) return; - requested = requests; - const keys = new Map(requests.map((request) => [UploadKey.of(request).id, request])); - for (const id of holders.keys()) { - if (!keys.has(id)) release(id); - } - for (const [id, request] of keys) { - if (holders.has(id)) continue; - const holder = Scope.makeUnsafe(); - holders.set(id, holder); - Effect.runSync(Scope.provide(holder)(RcMap.get(uploads, UploadKey.of(request)))); - } - } - + const registry = createAttachmentUploadRegistry({ + key: (request) => composerAttachmentUploadKey(request.environmentId, request.attachment.id), + environmentId: (request) => request.environmentId, + concurrencyPerEnvironment: 3, + upload: options.upload, + ...(options.onUploaded ? { onUploaded: options.onUploaded } : {}), + onChange: options.onChange, + }); return { - sync, - /** Uploads the attachment, joining its running upload when there is one. */ - upload( - request: ComposerAttachmentUploadRequest, - ): Effect.Effect { - return disposed - ? Effect.fail(uploadError(new Error("Upload cancelled."))) - : RcMap.get(uploads, UploadKey.of(request)).pipe(Effect.flatMap(Fiber.join), Effect.scoped); - }, - retry(environmentId: EnvironmentId, attachmentId: string) { - const id = composerAttachmentUploadKey(environmentId, attachmentId); - if (states[id]?.status !== "failed") return; - Effect.runSync(RcMap.invalidate(uploads, new UploadKey(id, null))); - release(id); - sync(requested); - }, - async settled() { - while (running.size > 0) { - await Effect.runPromise(Fiber.awaitAll([...running])); - } - }, - dispose() { - disposed = true; - holders.clear(); - Effect.runFork(Scope.close(scope, Exit.void)); - states = {}; - options.onChange(states); - }, + sync: registry.sync, + upload: registry.upload, + retry: (environmentId: EnvironmentId, attachmentId: string) => + registry.retry(composerAttachmentUploadKey(environmentId, attachmentId)), + settled: registry.settled, + dispose: registry.dispose, }; } - -export function uploadError(cause: unknown): ComposerAttachmentUploadError { - if (isComposerAttachmentUploadError(cause)) return cause; - return new ComposerAttachmentUploadError({ - message: cause instanceof Error ? cause.message : "Upload failed. Tap to retry.", - cause, - }); -} - -export class ComposerAttachmentUploadError extends Schema.TaggedError()( - "ComposerAttachmentUploadError", - { message: Schema.String, cause: Schema.Defect() }, -) {} - -const isComposerAttachmentUploadError = Schema.is(ComposerAttachmentUploadError); diff --git a/apps/mobile/src/state/attachments.test.ts b/apps/mobile/src/state/attachments.test.ts index 5f727e3304f94..052c5ef045cb1 100644 --- a/apps/mobile/src/state/attachments.test.ts +++ b/apps/mobile/src/state/attachments.test.ts @@ -6,7 +6,10 @@ import { import { runAttachmentUploadCycle } from "@t3tools/client-runtime/state/attachments"; import * as Cause from "effect/Cause"; import { AsyncResult, AtomRegistry } from "effect/reactivity"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import { afterEach, beforeEach, vi } from "vite-plus/test"; const state = vi.hoisted(() => ({ grantedEnvironments: new Set(), @@ -76,24 +79,29 @@ describe("attachment mutation grants", () => { expect(state.remove).toHaveBeenCalledOnce(); }); - it("blocks the upload cycle's asynchronous cleanup after grant revocation", async () => { - state.grantedEnvironments.add("secondary"); - const transport = vi.fn(); - const result = await runAttachmentUploadCycle({ - registry, - ...attachmentEnvironment, - environmentId, - upload: { name: "image.png", mimeType: "image/png", sizeBytes: 1 }, - resolveUploadUrl: () => "https://example.test/upload", - transport, - onMinted: () => { - state.grantedEnvironments.delete("secondary"); - return "cancel"; - }, - }); - expect(result.status).toBe("cancelled"); - expect(state.create).toHaveBeenCalledOnce(); - expect(state.remove).not.toHaveBeenCalled(); - expect(transport).not.toHaveBeenCalled(); - }); + it.effect("blocks the upload cycle's cleanup after grant revocation", () => + Effect.gen(function* () { + state.grantedEnvironments.add("secondary"); + const transferStarted = Promise.withResolvers(); + const cycle = yield* Effect.forkChild( + runAttachmentUploadCycle({ + registry, + ...attachmentEnvironment, + environmentId, + upload: { name: "image.png", mimeType: "image/png", sizeBytes: 1 }, + resolveUploadUrl: () => "https://example.test/upload", + transport: () => + Effect.callback(() => { + state.grantedEnvironments.delete("secondary"); + transferStarted.resolve(); + }), + }), + ); + yield* Effect.promise(() => transferStarted.promise); + // Interrupting deletes the fresh mint, which the revoked grant must refuse. + yield* Fiber.interrupt(cycle); + expect(state.create).toHaveBeenCalledOnce(); + expect(state.remove).not.toHaveBeenCalled(); + }), + ); }); diff --git a/apps/mobile/src/state/composer-attachment-uploads.ts b/apps/mobile/src/state/composer-attachment-uploads.ts index acc7576a8d1d8..c80a9436636ea 100644 --- a/apps/mobile/src/state/composer-attachment-uploads.ts +++ b/apps/mobile/src/state/composer-attachment-uploads.ts @@ -2,6 +2,7 @@ import { useAtomValue } from "@effect/atom-react"; import { AuthOrchestrationOperateScope, type EnvironmentId } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; +import { attachmentUploadError } from "@t3tools/client-runtime/state/attachment-uploads"; import { Atom } from "effect/reactivity"; import { uploadComposerAttachment } from "../lib/attachmentUpload"; @@ -11,7 +12,6 @@ import { type DraftComposerAttachment, } from "../lib/composerImages"; import { - uploadError, composerAttachmentUploadKey, composerDraftEnvironmentId, canUploadComposerAttachment, @@ -162,7 +162,7 @@ const recordQueuedAttachmentUpload = Effect.fn("ComposerAttachments.recordQueued ? null : withUploadedAttachment(current, uploaded), ), - catch: uploadError, + catch: attachmentUploadError, }), { concurrency: "unbounded", discard: true }, ); @@ -226,7 +226,7 @@ const recordComposerAttachmentUpload = Effect.fn("ComposerAttachments.recordUplo yield* recordQueuedAttachmentUpload(environmentId, uploaded); if (!draftsChanged) return; // A failed draft save only costs a re-upload after restart; the outbox saves its own references. - yield* Effect.tryPromise({ try: () => flushComposerDrafts(), catch: uploadError }).pipe( + yield* Effect.tryPromise({ try: () => flushComposerDrafts(), catch: attachmentUploadError }).pipe( Effect.catchIf( (error) => isComposerDraftPersistenceError(error.cause), (error) => @@ -241,14 +241,17 @@ const recordComposerAttachmentUpload = Effect.fn("ComposerAttachments.recordUplo function composerAttachmentUploads() { uploadQueue ??= createComposerAttachmentUploadQueue({ onChange: (states) => appAtomRegistry.set(composerAttachmentUploadsAtom, states), - upload: ({ environmentId, attachment }, signal, onProgress) => { - const release = isFileBackedComposerAttachment(attachment) - ? retainComposerAttachmentFileForPreview(attachment) - : undefined; - return uploadComposerAttachment(environmentId, attachment, signal, onProgress).finally(() => - release?.(), - ); - }, + // The file stays on disk while its upload reads it, even if its draft lets go meanwhile. + upload: ({ environmentId, attachment }, reportProgress) => + Effect.acquireUseRelease( + Effect.sync(() => + isFileBackedComposerAttachment(attachment) + ? retainComposerAttachmentFileForPreview(attachment) + : undefined, + ), + () => uploadComposerAttachment(environmentId, attachment, reportProgress), + (release) => Effect.sync(() => release?.()), + ), onUploaded: recordComposerAttachmentUpload, }); return uploadQueue; diff --git a/apps/mobile/src/state/thread-outbox-model.ts b/apps/mobile/src/state/thread-outbox-model.ts index 59427907c4ebf..2ec0d29eca21a 100644 --- a/apps/mobile/src/state/thread-outbox-model.ts +++ b/apps/mobile/src/state/thread-outbox-model.ts @@ -289,7 +289,7 @@ export function shouldRetryThreadOutboxDelivery(error: unknown): boolean { case "EnvironmentNotRegisteredError": return true; // A shared upload's failure decides by what made the upload fail. - case "ComposerAttachmentUploadError": + case "AttachmentUploadError": return "cause" in error && shouldRetryThreadOutboxDelivery(error.cause); default: break; diff --git a/apps/mobile/src/state/threadOutboxDelivery.permissions.test.ts b/apps/mobile/src/state/threadOutboxDelivery.permissions.test.ts index 700b58499d691..b8588e0d40082 100644 --- a/apps/mobile/src/state/threadOutboxDelivery.permissions.test.ts +++ b/apps/mobile/src/state/threadOutboxDelivery.permissions.test.ts @@ -7,7 +7,7 @@ import { ProviderInstanceId, ThreadId, } from "@t3tools/contracts"; -import { uploadError } from "../lib/composerAttachmentUploadQueue"; +import { attachmentUploadError } from "@t3tools/client-runtime/state/attachment-uploads"; import * as Effect from "effect/Effect"; import * as Cause from "effect/Cause"; import { AsyncResult, type Atom } from "effect/reactivity"; @@ -497,7 +497,9 @@ describe("queued task operation access", () => { Effect.suspend(() => { setOperateGrant("secondary", false); return uploadFails - ? Effect.fail(uploadError(new Error("This connection cannot upload attachments."))) + ? Effect.fail( + attachmentUploadError(new Error("This connection cannot upload attachments.")), + ) : Effect.succeed({ status: "ready", attachments: [], diff --git a/apps/web/src/lib/attachmentUploadQueue.ts b/apps/web/src/lib/attachmentUploadQueue.ts index cf9bbedeb853a..bf89566499e32 100644 --- a/apps/web/src/lib/attachmentUploadQueue.ts +++ b/apps/web/src/lib/attachmentUploadQueue.ts @@ -7,6 +7,11 @@ import { } from "@t3tools/contracts"; import { parseScopedThreadKey } from "@t3tools/client-runtime/environment"; import { resolveAssetUrl } from "@t3tools/client-runtime/state/assets"; +import { + abortableTransfer, + attachmentUploadError, + createAttachmentUploadRegistry, +} from "@t3tools/client-runtime/state/attachment-uploads"; import { deletePendingAttachmentUpload, runAttachmentUploadCycle, @@ -14,6 +19,8 @@ import { type PersistedAttachmentVerification, } from "@t3tools/client-runtime/state/attachments"; import { create } from "zustand"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as Option from "effect/Option"; import { AsyncResult } from "effect/reactivity"; @@ -50,19 +57,19 @@ interface UploadJob { readonly previous?: ReadyAttachmentUpload; /** Draft-owned; delete only through releasePersistedAttachmentUpload, never queue cancel/retry. */ readonly persistedAttachmentId?: string; - readonly settled: Promise; - resolveSettled: () => void; - attachmentId: string | null; - cancelled: boolean; - abort: (() => void) | null; - stopWatchingConnection: () => void; } type DraftFileOwner = Pick; -const jobsByImageId = new Map(); -const queue: UploadJob[] = []; -const activeUploadsByEnvironment = new Map(); +/** The upload each image holds; a failed one stays held for its retry. */ +interface HeldUpload { + readonly job: UploadJob; + readonly release: () => void; + readonly stopWatchingConnection: () => void; +} + +const heldUploads = new Map(); +const jobKey = (job: UploadJob) => `${job.environmentId}:${job.image.id}`; function setUploadState(imageId: string, upload: AttachmentUploadState): void { useAttachmentUploadStore.setState((state) => ({ @@ -137,10 +144,11 @@ function uploadBytes(input: { readonly url: string; readonly file: File; readonly mimeType: string; + readonly signal: AbortSignal; readonly onProgress: (progress: number) => void; -}): { readonly done: Promise; readonly abort: () => void } { +}): Promise { const xhr = new XMLHttpRequest(); - const done = new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { xhr.open("POST", input.url, true); xhr.timeout = UPLOAD_TIMEOUT_MS; xhr.setRequestHeader("Content-Type", input.mimeType); @@ -159,32 +167,38 @@ function uploadBytes(input: { xhr.addEventListener("error", () => reject(new Error("Upload failed"))); xhr.addEventListener("timeout", () => reject(new Error("Upload timed out"))); xhr.addEventListener("abort", () => reject(new Error("Upload cancelled"))); + input.signal.addEventListener("abort", () => xhr.abort(), { once: true }); xhr.send(input.file); }); - - return { done, abort: () => xhr.abort() }; } -async function runUpload(job: UploadJob): Promise { - if (!readEnvironmentScope(job.environmentId, AuthOrchestrationOperateScope)) { +const canOperate = (environmentId: EnvironmentId) => + readEnvironmentScope(environmentId, AuthOrchestrationOperateScope); + +/** Records the failure the composer shows, then fails the upload with it. */ +const failUpload = (job: UploadJob, reason: string, attachmentId?: string) => + Effect.sync(() => setUploadState(job.image.id, { status: "failed", environmentId: job.environmentId, - reason: "This connection cannot upload attachments.", + reason, + ...(attachmentId ? { attachmentId } : {}), ...(job.previous ? { previous: job.previous } : {}), - }); - return; + }), + ).pipe(Effect.andThen(Effect.fail(attachmentUploadError(new Error(reason))))); + +/** Uploads one composer attachment, writing the state its composer shows as it goes. */ +const uploadJob = Effect.fn("WebAttachmentUploads.upload")(function* (job: UploadJob) { + if (!canOperate(job.environmentId)) { + return yield* failUpload(job, "This connection cannot upload attachments."); } if (job.persistedAttachmentId) { - const verification = await verifyPersistedAttachmentUpload({ + const verification = yield* verifyPersistedAttachmentUpload({ registry: appAtomRegistry, createAssetUrl: assetEnvironment.createUrl, environmentId: job.environmentId, attachmentId: job.persistedAttachmentId, }); - if (job.cancelled) { - return; - } if (verification.status === "verified") { setUploadState(job.image.id, { status: "ready", @@ -192,7 +206,7 @@ async function runUpload(job: UploadJob): Promise { attachmentId: job.persistedAttachmentId, }); stampDraftFileUpload(job, job.persistedAttachmentId); - return; + return job.persistedAttachmentId; } if (verification.status === "missing" && !job.image.file && job.image.type === "file") { const draftTarget = resolveCurrentFileDraftTarget(job); @@ -207,22 +221,19 @@ async function runUpload(job: UploadJob): Promise { job.persistedAttachmentId, ) ) { + // The draft now shows the file as missing, so the composer has no upload to show. clearUploadState(job.image.id); - return; + return yield* attachmentUploadError(new Error("Uploaded file expired.")); } } if (verification.status === "failed" || !job.image.file) { // Failed-state ids get deleted on retry/release; exclude persisted ids after transient verification failure. - setUploadState(job.image.id, { - status: "failed", - environmentId: job.environmentId, - reason: - verification.status === "missing" - ? "Uploaded file expired. Remove it and attach it again." - : "Uploaded file could not be verified. Retry when the server reconnects.", - ...(job.previous ? { previous: job.previous } : {}), - }); - return; + return yield* failUpload( + job, + verification.status === "missing" + ? "Uploaded file expired. Remove it and attach it again." + : "Uploaded file could not be verified. Retry when the server reconnects.", + ); } } @@ -232,37 +243,15 @@ async function runUpload(job: UploadJob): Promise { : PROVIDER_SEND_TURN_SUPPORTED_IMAGE_MIME_TYPES.find( (supportedMimeType) => supportedMimeType === job.image.mimeType.toLowerCase(), ); - if (!mimeType) { - setUploadState(job.image.id, { - status: "failed", - environmentId: job.environmentId, - reason: "Unsupported image type", - ...(job.previous ? { previous: job.previous } : {}), - }); - return; - } + if (!mimeType) return yield* failUpload(job, "Unsupported image type"); const file = job.image.file; - if (!file) { - setUploadState(job.image.id, { - status: "failed", - environmentId: job.environmentId, - reason: "Original file is no longer available", - ...(job.previous ? { previous: job.previous } : {}), - }); - return; + if (!file) return yield* failUpload(job, "Original file is no longer available"); + if (!canOperate(job.environmentId)) { + return yield* failUpload(job, "This connection cannot upload attachments."); } let lastStep = -1; - if (!readEnvironmentScope(job.environmentId, AuthOrchestrationOperateScope)) { - setUploadState(job.image.id, { - status: "failed", - environmentId: job.environmentId, - reason: "This connection cannot upload attachments.", - ...(job.previous ? { previous: job.previous } : {}), - }); - return; - } - const result = await runAttachmentUploadCycle({ + const result = yield* runAttachmentUploadCycle({ registry: appAtomRegistry, createUploadUrl: attachmentEnvironment.createUploadUrl, remove: attachmentEnvironment.remove, @@ -274,49 +263,31 @@ async function runUpload(job: UploadJob): Promise { sizeBytes: file.size, }, resolveUploadUrl: (relativeUrl) => { - if ( - job.cancelled || - !readEnvironmentScope(job.environmentId, AuthOrchestrationOperateScope) - ) { - return null; - } + if (!canOperate(job.environmentId)) return null; const connection = readPreparedConnection(job.environmentId); return connection ? resolveAssetUrl(connection.httpBaseUrl, relativeUrl) : null; }, transport: (url) => - uploadBytes({ - url, - file, - mimeType, - onProgress: (progress) => { - const step = Math.floor(progress * 20); - if (step === lastStep || job.cancelled) { - return; - } - lastStep = step; - setUploadState(job.image.id, { - status: "uploading", - environmentId: job.environmentId, - progress, - ...(job.previous ? { previous: job.previous } : {}), - }); - }, - }), - onMinted: (attachmentId) => { - if (job.cancelled && readEnvironmentScope(job.environmentId, AuthOrchestrationOperateScope)) { - return "cancel"; - } - job.attachmentId = attachmentId; - return "continue"; - }, - onTransferStart: (abort) => { - job.abort = abort; - }, + abortableTransfer((signal) => + uploadBytes({ + url, + file, + mimeType, + signal, + onProgress: (progress) => { + const step = Math.floor(progress * 20); + if (step === lastStep || signal.aborted) return; + lastStep = step; + setUploadState(job.image.id, { + status: "uploading", + environmentId: job.environmentId, + progress, + ...(job.previous ? { previous: job.previous } : {}), + }); + }, + }), + ), }); - job.abort = null; - if (result.status === "cancelled" || job.cancelled) { - return; - } if (result.status === "uploaded") { setUploadState(job.image.id, { status: "ready", @@ -327,69 +298,42 @@ async function runUpload(job: UploadJob): Promise { if (job.previous) { deletePendingUpload(job.previous.environmentId, job.previous.attachmentId); } - return; - } - setUploadState(job.image.id, { - status: "failed", - environmentId: job.environmentId, - reason: - result.step === "mint" - ? "Upload could not start" - : result.step === "resolve-url" - ? readEnvironmentScope(job.environmentId, AuthOrchestrationOperateScope) - ? "Not connected" - : "This connection cannot upload attachments." - : result.error instanceof Error - ? result.error.message - : "Upload failed", - ...(result.attachmentId ? { attachmentId: result.attachmentId } : {}), - ...(job.previous ? { previous: job.previous } : {}), - }); -} - -function pumpUploads(): void { - for (let index = 0; index < queue.length;) { - const job = queue[index]!; - const active = activeUploadsByEnvironment.get(job.environmentId) ?? 0; - if (active >= MAX_UPLOADS_PER_ENVIRONMENT) { - index += 1; - continue; - } - - queue.splice(index, 1); - if (job.cancelled) { - continue; - } - activeUploadsByEnvironment.set(job.environmentId, active + 1); - void runUpload(job) - .catch(() => { - if (!job.cancelled) { - setUploadState(job.image.id, { - status: "failed", - environmentId: job.environmentId, - reason: "Upload failed", - ...(job.previous ? { previous: job.previous } : {}), - }); - } - }) - .finally(() => { - if ( - jobsByImageId.get(job.image.id) === job && - readAttachmentUpload(job.image.id)?.status !== "failed" - ) { - job.stopWatchingConnection(); - jobsByImageId.delete(job.image.id); - } - const remaining = (activeUploadsByEnvironment.get(job.environmentId) ?? 1) - 1; - if (remaining > 0) { - activeUploadsByEnvironment.set(job.environmentId, remaining); - } else { - activeUploadsByEnvironment.delete(job.environmentId); - } - job.resolveSettled(); - pumpUploads(); - }); + return result.attachmentId; } + return yield* failUpload( + job, + result.step === "mint" + ? "Upload could not start" + : result.step === "resolve-url" + ? canOperate(job.environmentId) + ? "Not connected" + : "This connection cannot upload attachments." + : result.error instanceof Error + ? result.error.message + : "Upload failed", + result.attachmentId ?? undefined, + ); +}); + +const uploads = createAttachmentUploadRegistry({ + key: jobKey, + environmentId: (job) => job.environmentId, + concurrencyPerEnvironment: MAX_UPLOADS_PER_ENVIRONMENT, + upload: (job) => + uploadJob(job).pipe( + Effect.catchDefect((defect) => + failUpload(job, "Upload failed").pipe( + Effect.ensuring(Effect.logWarning("Composer attachment upload threw", defect)), + ), + ), + ), +}); + +/** Waits for an image's held upload to finish, without keeping it running. */ +function awaitHeldUpload(imageId: string): Effect.Effect { + const held = heldUploads.get(imageId); + const fiber = held === undefined ? undefined : uploads.current(jobKey(held.job)); + return fiber === undefined ? Effect.void : Effect.asVoid(Fiber.await(fiber)); } export function startAttachmentUpload(input: { @@ -398,9 +342,9 @@ export function startAttachmentUpload(input: { readonly draftTarget?: ComposerThreadTarget; readonly force?: boolean; }): void { - if (!readEnvironmentScope(input.environmentId, AuthOrchestrationOperateScope)) return; - const existingJob = jobsByImageId.get(input.image.id); - if (existingJob?.environmentId === input.environmentId) { + if (!canOperate(input.environmentId)) return; + const existingHold = heldUploads.get(input.image.id); + if (existingHold?.job.environmentId === input.environmentId) { return; } @@ -433,14 +377,10 @@ export function startAttachmentUpload(input: { return; } - if (existingJob) { + if (existingHold) { cancelAttachmentUpload(input.image.id); } const previous = existing?.status === "ready" ? existing : existing?.previous; - let resolveSettled: () => void = () => {}; - const settled = new Promise((resolve) => { - resolveSettled = resolve; - }); const job: UploadJob = { image: input.image, environmentId: input.environmentId, @@ -452,65 +392,68 @@ export function startAttachmentUpload(input: { input.image.uploadedAttachmentId ? { persistedAttachmentId: input.image.uploadedAttachmentId } : {}), - settled, - resolveSettled, - attachmentId: null, - cancelled: false, - abort: null, - stopWatchingConnection: () => {}, }; - jobsByImageId.set(input.image.id, job); + setUploadState(input.image.id, { + status: "uploading", + environmentId: input.environmentId, + progress: 0, + ...(previous ? { previous } : {}), + }); const connectionAtom = environmentCatalog.stateAtom(job.environmentId); const isConnected = () => Option.exists( AsyncResult.value(appAtomRegistry.get(connectionAtom)), (state) => state.phase === "connected", ); + const ownsImage = () => heldUploads.get(input.image.id)?.job === job; let wasConnected = isConnected(); - job.stopWatchingConnection = appAtomRegistry.subscribe(connectionAtom, () => { + const stopWatchingConnection = appAtomRegistry.subscribe(connectionAtom, () => { const connected = isConnected(); const reconnected = connected && !wasConnected; wasConnected = connected; if (!reconnected) return; // HTTP failures can follow reconnection; await the attempt and retry only if this job still owns the file. - void job.settled.then(() => { - if ( - jobsByImageId.get(job.image.id) === job && - readAttachmentUpload(job.image.id)?.status === "failed" && - isConnected() - ) { - retryAttachmentUpload(input); - } - }); + Effect.runFork( + awaitHeldUpload(input.image.id).pipe( + Effect.andThen( + Effect.sync(() => { + if ( + ownsImage() && + readAttachmentUpload(input.image.id)?.status === "failed" && + isConnected() + ) { + retryAttachmentUpload(input); + } + }), + ), + ), + ); }); - queue.push(job); - setUploadState(input.image.id, { - status: "uploading", - environmentId: input.environmentId, - progress: 0, - ...(previous ? { previous } : {}), + heldUploads.set(input.image.id, { + job, + release: uploads.hold(job), + stopWatchingConnection, + }); + // A finished upload lets go of its image; a failed one stays held for its retry. + uploads.current(jobKey(job))?.addObserver((exit) => { + if (exit._tag !== "Success" || !ownsImage()) return; + const held = heldUploads.get(input.image.id)!; + heldUploads.delete(input.image.id); + held.stopWatchingConnection(); + held.release(); }); - pumpUploads(); } function cancelAttachmentUpload(imageId: string): void { - const job = jobsByImageId.get(imageId); - if (!job) { + const held = heldUploads.get(imageId); + if (!held) { return; } - job.cancelled = true; - job.stopWatchingConnection(); - jobsByImageId.delete(imageId); - const queuedIndex = queue.indexOf(job); - if (queuedIndex !== -1) { - queue.splice(queuedIndex, 1); - } - job.abort?.(); - if (job.attachmentId) { - deletePendingUpload(job.environmentId, job.attachmentId); - } - job.resolveSettled(); + heldUploads.delete(imageId); + held.stopWatchingConnection(); + // Releasing interrupts a running upload, which aborts its transfer and deletes its mint. + held.release(); } export function releaseAttachmentUpload(imageId: string): void { @@ -543,10 +486,10 @@ export function releasePersistedAttachmentUpload(input: { releaseAttachmentUpload(input.id); return; } - const job = jobsByImageId.get(input.id); + const held = heldUploads.get(input.id); if ( - job?.environmentId === input.environmentId && - job.persistedAttachmentId === input.attachmentId + held?.job.environmentId === input.environmentId && + held.job.persistedAttachmentId === input.attachmentId ) { releaseAttachmentUpload(input.id); } @@ -558,9 +501,12 @@ export function retryAttachmentUpload(input: { readonly image: ComposerImageAttachment | ComposerFileAttachment; readonly draftTarget?: ComposerThreadTarget; }): void { - if (!readEnvironmentScope(input.environmentId, AuthOrchestrationOperateScope)) return; + if (!canOperate(input.environmentId)) return; const previous = readAttachmentUpload(input.image.id); + const held = heldUploads.get(input.image.id); cancelAttachmentUpload(input.image.id); + // The failed upload stays cached until dropped, and the retry must start a fresh one. + if (held) uploads.invalidate(jobKey(held.job)); if (previous?.status === "failed" && previous.attachmentId) { deletePendingUpload(previous.environmentId, previous.attachmentId); } @@ -577,16 +523,21 @@ export function verifyStashedAttachmentUpload(input: { readonly environmentId: EnvironmentId; readonly attachmentId: string; }): Promise { - return verifyPersistedAttachmentUpload({ - registry: appAtomRegistry, - createAssetUrl: assetEnvironment.createUrl, - environmentId: input.environmentId, - attachmentId: input.attachmentId, - }); + return Effect.runPromise( + verifyPersistedAttachmentUpload({ + registry: appAtomRegistry, + createAssetUrl: assetEnvironment.createUrl, + environmentId: input.environmentId, + attachmentId: input.attachmentId, + }), + ); } -export async function awaitAttachmentUploads(imageIds: ReadonlyArray): Promise { - await Promise.all(imageIds.map((imageId) => jobsByImageId.get(imageId)?.settled)); +/** Resolves once these images' uploads finish, fail or are cancelled. */ +export function awaitAttachmentUploads(imageIds: ReadonlyArray): Promise { + return Effect.runPromise( + Effect.forEach(imageIds, awaitHeldUpload, { concurrency: "unbounded", discard: true }), + ); } /** Revalidate ready uploads at send time; a long-lived draft can outlive server cleanup. */ @@ -685,7 +636,7 @@ export function releaseDraftAttachment( environmentId: attachment.uploadEnvironmentId, attachmentId: attachment.uploadedAttachmentId, }); - if (jobsByImageId.has(attachment.id) || readAttachmentUpload(attachment.id)) { + if (heldUploads.has(attachment.id) || readAttachmentUpload(attachment.id)) { releaseAttachmentUpload(attachment.id); } return; diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 6fa96cdd493ae..ec826c4861668 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -175,6 +175,10 @@ "types": "./src/state/attachments.ts", "default": "./src/state/attachments.ts" }, + "./state/attachment-uploads": { + "types": "./src/state/attachmentUploads.ts", + "default": "./src/state/attachmentUploads.ts" + }, "./text-paste": { "types": "./src/textPaste.ts", "default": "./src/textPaste.ts" diff --git a/packages/client-runtime/src/state/attachmentUploads.test.ts b/packages/client-runtime/src/state/attachmentUploads.test.ts new file mode 100644 index 0000000000000..559bef0c4d7ae --- /dev/null +++ b/packages/client-runtime/src/state/attachmentUploads.test.ts @@ -0,0 +1,289 @@ +import { EnvironmentId } from "@t3tools/contracts"; +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import { vi } from "vite-plus/test"; + +import { + abortableTransfer, + createAttachmentUploadRegistry, + type AttachmentUploadError, + type AttachmentUploadStatus, +} from "./attachmentUploads.ts"; + +interface UploadRequest { + readonly environmentId: EnvironmentId; + readonly attachment: { readonly id: string; readonly fileUri: string }; +} + +const environmentId = EnvironmentId.make("environment-1"); +const uploadKey = (environment: EnvironmentId, id: string) => `${environment}:${id}`; + +function request(id: string, environment = environmentId): UploadRequest { + return { environmentId: environment, attachment: { id, fileUri: `file:///documents/${id}.pdf` } }; +} + +/** The registry over a promise-based transfer, as each client wires its own. */ +function createQueue(options: { + readonly upload: ( + request: UploadRequest, + signal: AbortSignal, + onProgress: (progress: number) => void, + ) => Promise; + readonly onUploaded?: ( + request: UploadRequest, + uploadedId: string, + ) => Effect.Effect; + readonly onChange: (statuses: Readonly>) => void; + readonly concurrencyPerEnvironment?: number; +}) { + return createAttachmentUploadRegistry({ + key: (candidate) => uploadKey(candidate.environmentId, candidate.attachment.id), + environmentId: (candidate) => candidate.environmentId, + concurrencyPerEnvironment: options.concurrencyPerEnvironment ?? 3, + upload: (candidate, report) => + abortableTransfer((signal) => options.upload(candidate, signal, report)), + ...(options.onUploaded ? { onUploaded: options.onUploaded } : {}), + onChange: options.onChange, + }); +} + +describe("attachment upload registry", () => { + it("bounds concurrency, deduplicates updates, and drains all attachments", async () => { + const gates = new Map>>(); + const fourthStarted = Promise.withResolvers(); + const firstThreeStarted = Promise.withResolvers(); + let active = 0; + let maximum = 0; + const upload = vi.fn(async (input: UploadRequest) => { + active += 1; + maximum = Math.max(maximum, active); + const gate = Promise.withResolvers(); + gates.set(input.attachment.id, gate); + if (gates.size === 3) firstThreeStarted.resolve(); + if (gates.size === 4) fourthStarted.resolve(); + try { + return await gate.promise; + } finally { + active -= 1; + } + }); + const queue = createQueue({ upload, onChange: () => {} }); + const requests = [request("one"), request("two"), request("three"), request("four")]; + queue.sync(requests); + queue.sync(requests); + await firstThreeStarted.promise; + expect(upload).toHaveBeenCalledTimes(3); + gates.get("one")!.resolve("uploaded"); + await fourthStarted.promise; + for (const gate of gates.values()) gate.resolve("uploaded"); + await queue.settled(); + queue.sync(requests); + await queue.settled(); + expect(maximum).toBe(3); + expect(upload).toHaveBeenCalledTimes(4); + queue.dispose(); + }); + + it("cancels on disconnect and resumes from the same local draft on reconnect", async () => { + const started = Promise.withResolvers(); + let states: Readonly> = {}; + let signal: AbortSignal | undefined; + const upload = vi.fn(async (_request: UploadRequest, currentSignal: AbortSignal) => { + signal = currentSignal; + started.resolve(); + return new Promise((resolve) => + currentSignal.addEventListener("abort", () => resolve("aborted"), { once: true }), + ); + }); + const queue = createQueue({ + upload, + onChange: (next) => { + states = next; + }, + }); + const local = request("offline-draft"); + queue.sync([local]); + await started.promise; + queue.sync([]); + await queue.settled(); + expect(signal?.aborted).toBe(true); + expect(states).toEqual({}); + upload.mockResolvedValueOnce("uploaded"); + queue.sync([local]); + await queue.settled(); + expect(upload.mock.calls[1]?.[0]).toBe(local); + expect(states[uploadKey(environmentId, local.attachment.id)]).toEqual({ + status: "ready", + }); + expect(local.attachment).toMatchObject({ fileUri: "file:///documents/offline-draft.pdf" }); + queue.dispose(); + }); + + it("retries a failed attachment when the worker restores requests after reconnect", async () => { + let states: Readonly> = {}; + const upload = vi + .fn() + .mockRejectedValueOnce(new Error("Disconnected")) + .mockResolvedValue("uploaded"); + const queue = createQueue({ + upload, + onChange: (next) => { + states = next; + }, + }); + const local = request("failed-upload"); + const key = uploadKey(environmentId, local.attachment.id); + queue.sync([local]); + await queue.settled(); + expect(states[key]?.status).toBe("failed"); + queue.sync([local]); + await queue.settled(); + expect(upload).toHaveBeenCalledTimes(1); + queue.sync([]); + queue.sync([local]); + await queue.settled(); + expect(upload).toHaveBeenCalledTimes(2); + expect(states[key]?.status).toBe("ready"); + queue.dispose(); + }); + + it("ignores a late completion after removal or environment switch", async () => { + const gate = Promise.withResolvers(); + const started = Promise.withResolvers(); + let states: Readonly> = {}; + const upload = vi.fn(async () => { + started.resolve(); + return gate.promise; + }); + const queue = createQueue({ + upload, + onChange: (next) => { + states = next; + }, + }); + queue.sync([request("photo")]); + await started.promise; + upload.mockResolvedValueOnce("uploaded"); + const other = EnvironmentId.make("environment-2"); + queue.sync([request("photo", other)]); + gate.resolve("uploaded"); + await queue.settled(); + expect(states).toEqual({ [uploadKey(other, "photo")]: { status: "ready" } }); + queue.sync([]); + expect(states).toEqual({}); + queue.dispose(); + }); + + it("restarts a re-added attachment after its aborted transfer finishes settling", async () => { + const firstStarted = Promise.withResolvers(); + const firstSettled = Promise.withResolvers(); + const secondStarted = Promise.withResolvers(); + const secondSettled = Promise.withResolvers(); + let states: Readonly> = {}; + let firstSignal: AbortSignal | undefined; + const upload = vi.fn(async (_request: UploadRequest, signal: AbortSignal) => { + if (!firstSignal) { + firstSignal = signal; + firstStarted.resolve(); + return firstSettled.promise; + } + secondStarted.resolve(); + return secondSettled.promise; + }); + const queue = createQueue({ + upload, + onChange: (next) => { + states = next; + }, + }); + const local = request("re-added"); + queue.sync([local]); + await firstStarted.promise; + queue.sync([]); + queue.sync([local]); + expect(firstSignal?.aborted).toBe(true); + expect(upload).toHaveBeenCalledOnce(); + firstSettled.resolve("aborted"); + await secondStarted.promise; + expect(upload).toHaveBeenCalledTimes(2); + secondSettled.resolve("uploaded"); + await queue.settled(); + expect(states[uploadKey(environmentId, local.attachment.id)]).toEqual({ + status: "ready", + }); + queue.dispose(); + }); + + it("keeps failures stable until retry and reports bounded progress", async () => { + let states: Readonly> = {}; + const progress: number[] = []; + const upload = vi.fn( + async ( + _request: UploadRequest, + _signal: AbortSignal, + report: (value: number) => void, + ): Promise => { + report(0.12); + report(0.13); + report(1.1); + throw new Error("Server unavailable"); + }, + ); + const queue = createQueue({ + upload, + onChange: (next) => { + states = next; + const state = next[uploadKey(environmentId, "file")]; + if (state?.status === "uploading") progress.push(state.progress); + }, + }); + queue.sync([request("file")]); + await queue.settled(); + queue.sync([request("file")]); + expect(upload).toHaveBeenCalledOnce(); + expect(states[uploadKey(environmentId, "file")]).toEqual({ + status: "failed", + reason: "Server unavailable", + }); + expect(progress).toEqual([0, 0.1, 1]); + upload.mockImplementationOnce(async () => "uploaded"); + queue.retry(uploadKey(environmentId, "file")); + await queue.settled(); + expect(states[uploadKey(environmentId, "file")]).toEqual({ status: "ready" }); + queue.dispose(); + }); + + it.effect("joins a send to the running upload instead of uploading again", () => + Effect.gen(function* () { + const gate = Promise.withResolvers(); + const started = Promise.withResolvers(); + const uploaded = vi.fn(); + const upload = vi.fn(async () => { + started.resolve(); + return gate.promise; + }); + let states: Readonly> = {}; + const queue = createQueue({ + upload, + onUploaded: uploaded, + onChange: (next) => { + states = next; + }, + }); + const local = request("video"); + queue.sync([local]); + yield* Effect.promise(() => started.promise); + // Sending hands the attachment from the draft to the outbox, which uploads it too. + const sent = yield* Effect.forkChild(queue.upload(local), { startImmediately: true }); + queue.sync([]); + gate.resolve("uploaded-1"); + expect(yield* Fiber.join(sent)).toBe("uploaded-1"); + expect(upload).toHaveBeenCalledOnce(); + expect(uploaded).toHaveBeenCalledExactlyOnceWith(local, "uploaded-1"); + yield* Effect.promise(() => queue.settled()); + expect(states).toEqual({}); + queue.dispose(); + }), + ); +}); diff --git a/packages/client-runtime/src/state/attachmentUploads.ts b/packages/client-runtime/src/state/attachmentUploads.ts new file mode 100644 index 0000000000000..7a64b8592c7aa --- /dev/null +++ b/packages/client-runtime/src/state/attachmentUploads.ts @@ -0,0 +1,250 @@ +import type { EnvironmentId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Equal from "effect/Equal"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as Hash from "effect/Hash"; +import * as RcMap from "effect/RcMap"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; + +export class AttachmentUploadError extends Schema.TaggedError()( + "AttachmentUploadError", + { message: Schema.String, cause: Schema.Defect() }, +) {} + +const isAttachmentUploadError = Schema.is(AttachmentUploadError); + +export function attachmentUploadError(cause: unknown): AttachmentUploadError { + if (isAttachmentUploadError(cause)) return cause; + return new AttachmentUploadError({ + message: cause instanceof Error ? cause.message : "Upload failed. Tap to retry.", + cause, + }); +} + +/** + * Runs a promise-based transfer as an Effect. Interrupting it aborts the transfer and waits for + * it to settle, so the same bytes are never sent twice at once. + */ +export function abortableTransfer( + start: (signal: AbortSignal) => Promise, +): Effect.Effect { + return Effect.suspend(() => { + const controller = new AbortController(); + const done = start(controller.signal); + return Effect.tryPromise({ try: () => done, catch: attachmentUploadError }).pipe( + Effect.onInterrupt(() => + Effect.sync(() => controller.abort()).pipe( + Effect.andThen(Effect.tryPromise({ try: () => done, catch: attachmentUploadError })), + Effect.ignore, + ), + ), + ); + }); +} + +export type AttachmentUploadStatus = + | { readonly status: "uploading"; readonly progress: number } + | { readonly status: "ready" } + | { readonly status: "failed"; readonly reason: string }; + +/** Equal by key; the request rides along for the upload. */ +class UploadKey implements Equal.Equal { + readonly id: string; + readonly request: Request | null; + constructor(id: string, request: Request | null) { + this.id = id; + this.request = request; + } + [Equal.symbol](that: unknown): boolean { + return that instanceof UploadKey && that.id === this.id; + } + [Hash.symbol](): number { + return Hash.string(this.id); + } +} + +/** + * Every upload of an attachment, shared by key. Holders keep an upload running, and anyone who + * needs its result joins the running upload instead of starting another. An upload is + * interrupted once nothing holds it, and the same attachment does not upload again until that + * interrupted upload has settled. + */ +export function createAttachmentUploadRegistry(options: { + readonly key: (request: Request) => string; + readonly environmentId: (request: Request) => EnvironmentId; + /** Uploads running at once against one environment; the rest wait their turn. */ + readonly concurrencyPerEnvironment: number; + /** + * Uploads one attachment and returns its uploaded id. Interrupting it must cancel the + * transfer and wait for it to settle. + */ + readonly upload: ( + request: Request, + reportProgress: (progress: number) => void, + ) => Effect.Effect; + /** Saves a finished upload's id; the upload is ready once this succeeds. */ + readonly onUploaded?: ( + request: Request, + uploadedId: string, + ) => Effect.Effect; + readonly onChange?: (statuses: Readonly>) => void; +}) { + let statuses: Readonly> = {}; + function setStatus(key: string, status: AttachmentUploadStatus | undefined) { + const previous = statuses[key]; + if ( + previous === status || + (previous?.status === "uploading" && + status?.status === "uploading" && + previous.progress === status.progress) + ) + return; + const next = { ...statuses }; + if (status) next[key] = status; + else delete next[key]; + statuses = next; + options.onChange?.(statuses); + } + + const permits = new Map(); + const permitsFor = (environmentId: EnvironmentId) => { + let semaphore = permits.get(environmentId); + if (semaphore === undefined) { + semaphore = Semaphore.makeUnsafe(options.concurrencyPerEnvironment); + permits.set(environmentId, semaphore); + } + return semaphore; + }; + const running = new Set>(); + // The latest upload per key: a re-added attachment waits for an interrupted one to settle. + const latest = new Map>(); + + const transfer = ( + key: UploadKey, + previous: Fiber.Fiber | undefined, + ) => { + const request = key.request!; + return (previous === undefined ? Effect.void : Fiber.await(previous)).pipe( + Effect.andThen( + Effect.suspend(() => { + if (statuses[key.id]?.status !== "uploading") { + setStatus(key.id, { status: "uploading", progress: 0 }); + } + return options.upload(request, (progress) => + setStatus(key.id, { + status: "uploading", + progress: Math.floor(Math.max(0, Math.min(1, progress)) * 20) / 20, + }), + ); + }).pipe(Semaphore.withPermits(permitsFor(options.environmentId(request)), 1)), + ), + Effect.tap((uploadedId) => options.onUploaded?.(request, uploadedId) ?? Effect.void), + Effect.tap(() => Effect.sync(() => setStatus(key.id, { status: "ready" }))), + Effect.tapError((error) => + Effect.sync(() => setStatus(key.id, { status: "failed", reason: error.message })), + ), + ); + }; + + const scope = Scope.makeUnsafe(); + const uploads = Effect.runSync( + Scope.provide(scope)( + RcMap.make({ + lookup: Effect.fn("AttachmentUploads.lookup")(function* (key: UploadKey) { + setStatus(key.id, { status: "uploading", progress: 0 }); + yield* Effect.addFinalizer(() => Effect.sync(() => setStatus(key.id, undefined))); + // Starts in the holder's call, so an upload begins as soon as something holds it. + const fiber = yield* Effect.forkScoped(transfer(key, latest.get(key.id)), { + startImmediately: true, + }); + latest.set(key.id, fiber); + running.add(fiber); + fiber.addObserver(() => { + running.delete(fiber); + if (latest.get(key.id) === fiber) latest.delete(key.id); + }); + return fiber; + }), + }), + ), + ); + const keyOf = (request: Request) => new UploadKey(options.key(request), request); + + let disposed = false; + /** Keeps the request's upload running until the returned release is called. */ + function hold(request: Request): () => void { + if (disposed) return () => {}; + const holder = Scope.makeUnsafe(); + Effect.runSync(Scope.provide(holder)(RcMap.get(uploads, keyOf(request)))); + let released = false; + return () => { + if (released) return; + released = true; + Effect.runFork(Scope.close(holder, Exit.void)); + }; + } + + const synced = new Map void>(); + let requested: ReadonlyArray = []; + /** Holds exactly these uploads; anything no longer requested and held nowhere else stops. */ + function sync(requests: ReadonlyArray) { + if (disposed) return; + requested = requests; + const keys = new Map(requests.map((request) => [options.key(request), request])); + for (const [id, release] of synced) { + if (keys.has(id)) continue; + synced.delete(id); + release(); + } + for (const [id, request] of keys) { + if (!synced.has(id)) synced.set(id, hold(request)); + } + } + + /** Drops a failed upload so the next holder starts it again. */ + function invalidate(key: string) { + Effect.runSync(RcMap.invalidate(uploads, new UploadKey(key, null))); + } + + return { + hold, + sync, + invalidate, + /** Uploads the request, joining its running upload when there is one. */ + upload(request: Request): Effect.Effect { + return disposed + ? Effect.fail(attachmentUploadError(new Error("Upload cancelled."))) + : RcMap.get(uploads, keyOf(request)).pipe(Effect.flatMap(Fiber.join), Effect.scoped); + }, + /** Starts a failed upload that `sync` holds again. */ + retry(key: string) { + if (statuses[key]?.status !== "failed") return; + invalidate(key); + synced.get(key)?.(); + synced.delete(key); + sync(requested); + }, + /** The upload running or last run for the key, to await it without holding it. */ + current(key: string): Fiber.Fiber | undefined { + return latest.get(key); + }, + status(key: string): AttachmentUploadStatus | undefined { + return statuses[key]; + }, + async settled() { + while (running.size > 0) { + await Effect.runPromise(Fiber.awaitAll([...running])); + } + }, + dispose() { + disposed = true; + synced.clear(); + Effect.runFork(Scope.close(scope, Exit.void)); + statuses = {}; + options.onChange?.(statuses); + }, + }; +} diff --git a/packages/client-runtime/src/state/attachments.test.ts b/packages/client-runtime/src/state/attachments.test.ts index a5396dcc0c66c..5a467311ad5a0 100644 --- a/packages/client-runtime/src/state/attachments.test.ts +++ b/packages/client-runtime/src/state/attachments.test.ts @@ -6,6 +6,7 @@ import { type AttachmentDeleteInput, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import { AsyncResult, Atom, AtomRegistry } from "effect/reactivity"; import type { AtomCommand } from "./runtime.ts"; @@ -59,79 +60,87 @@ const uploadInput: AttachmentCreateUploadUrlInput = { }; describe("runAttachmentUploadCycle", () => { - it("deletes the fresh mint when the caller cancels at onMinted", async () => { - removeCalls.length = 0; - const result = await runAttachmentUploadCycle({ - registry, - createUploadUrl: makeCreateUploadUrl("pending-cancelled"), - remove, - environmentId, - upload: uploadInput, - resolveUploadUrl: () => "https://environment.test/upload", - transport: () => { - throw new Error("transport must not run after cancel"); - }, - onMinted: () => "cancel", - }); - - expect(result).toEqual({ status: "cancelled", attachmentId: "pending-cancelled" }); - expect(removeCalls).toEqual(["pending-cancelled"]); - }); - - it("keeps the minted id on transfer failure so the caller can retry or release", async () => { - removeCalls.length = 0; - const result = await runAttachmentUploadCycle({ - registry, - createUploadUrl: makeCreateUploadUrl("pending-failed"), - remove, - environmentId, - upload: uploadInput, - resolveUploadUrl: () => "https://environment.test/upload", - transport: () => ({ - done: Promise.reject(new Error("Upload rejected (413)")), - abort: () => {}, - }), - }); + it.effect("aborts the transfer and deletes the fresh mint when interrupted", () => + Effect.gen(function* () { + removeCalls.length = 0; + const transferStarted = Promise.withResolvers(); + let aborted = false; + const cycle = yield* Effect.forkChild( + runAttachmentUploadCycle({ + registry, + createUploadUrl: makeCreateUploadUrl("pending-cancelled"), + remove, + environmentId, + upload: uploadInput, + resolveUploadUrl: () => "https://environment.test/upload", + transport: () => + Effect.callback(() => { + transferStarted.resolve(); + return Effect.sync(() => { + aborted = true; + }); + }), + }), + ); + yield* Effect.promise(() => transferStarted.promise); + yield* Fiber.interrupt(cycle); + + expect(aborted).toBe(true); + expect(removeCalls).toEqual(["pending-cancelled"]); + }), + ); + + it.effect("keeps the minted id on transfer failure so the caller can retry or release", () => + Effect.gen(function* () { + removeCalls.length = 0; + const result = yield* runAttachmentUploadCycle({ + registry, + createUploadUrl: makeCreateUploadUrl("pending-failed"), + remove, + environmentId, + upload: uploadInput, + resolveUploadUrl: () => "https://environment.test/upload", + transport: () => Effect.fail({ _tag: "UploadRejected", status: 413 } as const), + }); - expect(result).toMatchObject({ - status: "failed", - step: "transfer", - attachmentId: "pending-failed", - }); - expect(removeCalls).toEqual([]); - }); + expect(result).toMatchObject({ + status: "failed", + step: "transfer", + attachmentId: "pending-failed", + }); + expect(removeCalls).toEqual([]); + }), + ); }); describe("verifyPersistedAttachmentUpload", () => { - it("hits the server on every verification instead of reusing a cached failure", async () => { - let lookups = 0; - const assetUrlAtom = Atom.make( - Effect.promise(() => Promise.resolve()).pipe( - Effect.flatMap(() => { - lookups += 1; - return lookups === 1 - ? Effect.fail({ _tag: "TransportError" } as const) - : Effect.succeed({ url: "/api/assets/pending-1" }); - }), - ), - ).pipe(Atom.swr({ staleTime: 60_000 }), Atom.keepAlive); - const liveRegistry = AtomRegistry.make(); - - const verify = () => - verifyPersistedAttachmentUpload({ + it.effect("hits the server on every verification instead of reusing a cached failure", () => + Effect.gen(function* () { + let lookups = 0; + const assetUrlAtom = Atom.make( + Effect.promise(() => Promise.resolve()).pipe( + Effect.flatMap(() => { + lookups += 1; + return lookups === 1 + ? Effect.fail({ _tag: "TransportError" } as const) + : Effect.succeed({ url: "/api/assets/pending-1" }); + }), + ), + ).pipe(Atom.swr({ staleTime: 60_000 }), Atom.keepAlive); + const liveRegistry = AtomRegistry.make(); + + const verify = verifyPersistedAttachmentUpload({ registry: liveRegistry, createAssetUrl: () => assetUrlAtom, environmentId, attachmentId: "pending-1", }); - const first = await verify(); - expect(first).toMatchObject({ status: "failed" }); - - const second = await verify(); - expect(second).toEqual({ status: "verified" }); - expect(lookups).toBe(2); - }); + expect(yield* verify).toMatchObject({ status: "failed" }); + expect(yield* verify).toEqual({ status: "verified" }); + expect(lookups).toBe(2); + }), + ); }); describe("formatAttachmentSize", () => { diff --git a/packages/client-runtime/src/state/attachments.ts b/packages/client-runtime/src/state/attachments.ts index 800393470ffad..24d5b7723c062 100644 --- a/packages/client-runtime/src/state/attachments.ts +++ b/packages/client-runtime/src/state/attachments.ts @@ -6,6 +6,7 @@ import { type AttachmentDeleteInput, type EnvironmentId, } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; import type { AsyncResult, Atom, AtomRegistry } from "effect/reactivity"; import type { EnvironmentRegistry } from "../connection/registry.ts"; @@ -51,7 +52,7 @@ export type PersistedAttachmentVerification = | { readonly status: "missing" } | { readonly status: "failed"; readonly error: unknown }; -export async function verifyPersistedAttachmentUpload(input: { +export function verifyPersistedAttachmentUpload(input: { readonly registry: AtomRegistry.AtomRegistry; readonly createAssetUrl: (query: { readonly environmentId: EnvironmentId; @@ -61,25 +62,29 @@ export async function verifyPersistedAttachmentUpload(input: { }) => Atom.Atom>; readonly environmentId: EnvironmentId; readonly attachmentId: string; -}): Promise { - const result = await executeAtomQuery( - input.registry, - input.createAssetUrl({ - environmentId: input.environmentId, - input: { resource: { _tag: "attachment", attachmentId: input.attachmentId } }, +}): Effect.Effect { + return Effect.promise(() => + executeAtomQuery( + input.registry, + input.createAssetUrl({ + environmentId: input.environmentId, + input: { resource: { _tag: "attachment", attachmentId: input.attachmentId } }, + }), + // `refresh` forces a server round trip: the asset URL query atom caches + // results (SWR), so a retry right after a transient failure would + // otherwise re-observe the cached failure and never ask the server. + { reportFailure: false, reportDefect: false, refresh: true }, + ), + ).pipe( + Effect.map((result): PersistedAttachmentVerification => { + if (result._tag === "Success") return { status: "verified" }; + const error = squashAtomCommandFailure(result); + return isAssetAttachmentNotFoundFailure(error) + ? { status: "missing" } + : { status: "failed", error }; }), - // `refresh` forces a server round trip: the asset URL query atom caches - // results (SWR), so a retry right after a transient failure would - // otherwise re-observe the cached failure and never ask the server. - { reportFailure: false, reportDefect: false, refresh: true }, + Effect.withSpan("Attachments.verifyPersistedUpload"), ); - if (result._tag === "Success") { - return { status: "verified" }; - } - const error = squashAtomCommandFailure(result); - return isAssetAttachmentNotFoundFailure(error) - ? { status: "missing" } - : { status: "failed", error }; } type AttachmentCreateUploadUrlCommand = AtomCommand< @@ -108,14 +113,8 @@ export function deletePendingAttachmentUpload(input: { ); } -export interface AttachmentByteUpload { - readonly done: Promise; - readonly abort: () => void; -} - export type AttachmentUploadCycleResult = | { readonly status: "uploaded"; readonly attachmentId: string } - | { readonly status: "cancelled"; readonly attachmentId: string | null } | { readonly status: "failed"; readonly step: "mint" | "resolve-url" | "transfer"; @@ -124,65 +123,76 @@ export type AttachmentUploadCycleResult = }; /** - * The cycle never deletes the minted pending upload on failure: callers keep the returned - * `attachmentId` and decide between retry and release. The one exception is `onMinted` - * returning `"cancel"`, where the caller already abandoned the upload and the fresh mint is - * deleted before returning. + * Mints an upload slot and sends the bytes. A failure keeps the minted id in the result so the + * caller decides between retry and release. Interrupting the cycle aborts the transfer and + * deletes the fresh mint, including when the interruption arrives while minting. */ -export async function runAttachmentUploadCycle(input: { +export function runAttachmentUploadCycle(input: { readonly registry: AtomRegistry.AtomRegistry; readonly createUploadUrl: AttachmentCreateUploadUrlCommand; readonly remove: AttachmentRemoveCommand; readonly environmentId: EnvironmentId; readonly upload: AttachmentCreateUploadUrlInput; readonly resolveUploadUrl: (relativeUrl: string) => string | null; - readonly transport: (url: string) => AttachmentByteUpload; - readonly onMinted?: (attachmentId: string) => "continue" | "cancel"; - readonly onTransferStart?: (abort: () => void) => void; -}): Promise { - const minted = await runAtomCommand( - input.registry, - input.createUploadUrl, - { environmentId: input.environmentId, input: input.upload }, - { reportFailure: false }, - ); - if (minted._tag !== "Success") { - return { - status: "failed", - step: "mint", - attachmentId: null, - error: squashAtomCommandFailure(minted), - }; - } - const attachmentId = minted.value.attachmentId; - if (input.onMinted?.(attachmentId) === "cancel") { - deletePendingAttachmentUpload({ - registry: input.registry, - remove: input.remove, - environmentId: input.environmentId, - attachmentId, - }); - return { status: "cancelled", attachmentId }; - } - - const url = input.resolveUploadUrl(minted.value.relativeUrl); - if (!url) { - return { - status: "failed", - step: "resolve-url", - attachmentId, - error: new Error("The environment is not connected."), - }; - } - - const transfer = input.transport(url); - input.onTransferStart?.(transfer.abort); - try { - await transfer.done; - } catch (error) { - return { status: "failed", step: "transfer", attachmentId, error }; - } - return { status: "uploaded", attachmentId }; + /** Sends the bytes; interrupting it aborts the transfer. */ + readonly transport: (url: string) => Effect.Effect; +}): Effect.Effect { + return Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const minted = yield* Effect.promise(() => + runAtomCommand( + input.registry, + input.createUploadUrl, + { environmentId: input.environmentId, input: input.upload }, + { reportFailure: false }, + ), + ); + if (minted._tag !== "Success") { + return { + status: "failed", + step: "mint", + attachmentId: null, + error: squashAtomCommandFailure(minted), + } as const; + } + const attachmentId = minted.value.attachmentId; + return yield* restore( + Effect.suspend(() => { + const url = input.resolveUploadUrl(minted.value.relativeUrl); + if (!url) { + return Effect.succeed({ + status: "failed", + step: "resolve-url", + attachmentId, + error: new Error("The environment is not connected."), + }); + } + return input.transport(url).pipe( + Effect.as({ status: "uploaded", attachmentId }), + Effect.catch((error) => + Effect.succeed({ + status: "failed", + step: "transfer", + attachmentId, + error, + }), + ), + ); + }), + ).pipe( + Effect.onInterrupt(() => + Effect.sync(() => + deletePendingAttachmentUpload({ + registry: input.registry, + remove: input.remove, + environmentId: input.environmentId, + attachmentId, + }), + ), + ), + ); + }), + ).pipe(Effect.withSpan("Attachments.runUploadCycle")); } /** From 56d2a29615441f258d8d8b2408248fcfb030f579 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 16 Nov 2026 11:36:04 -0800 Subject: [PATCH 2/4] refactor(client-runtime): drop abortableTransfer for Effect.tryPromise Effect.tryPromise already aborts its signal on interruption. The helper only added waiting for an aborted transfer to settle, which the registry used to delay re-uploading a re-added attachment by the few milliseconds an abort takes. Both go: transports use Effect.tryPromise directly, and the registry keeps a plain map of running uploads for awaiting one without holding it. Co-Authored-By: Claude Opus 5.5 --- apps/mobile/src/lib/attachmentUpload.ts | 8 +- apps/web/src/lib/attachmentUploadQueue.ts | 41 +++++----- .../src/state/attachmentUploads.test.ts | 47 ++--------- .../src/state/attachmentUploads.ts | 80 +++++-------------- 4 files changed, 52 insertions(+), 124 deletions(-) diff --git a/apps/mobile/src/lib/attachmentUpload.ts b/apps/mobile/src/lib/attachmentUpload.ts index e51005bd1e474..ab102a059934e 100644 --- a/apps/mobile/src/lib/attachmentUpload.ts +++ b/apps/mobile/src/lib/attachmentUpload.ts @@ -33,7 +33,6 @@ import { import { imageMimeType } from "@t3tools/shared/image"; import { uuidv4 } from "./uuid"; import { - abortableTransfer, attachmentUploadError, type AttachmentUploadError, } from "@t3tools/client-runtime/state/attachment-uploads"; @@ -317,9 +316,10 @@ export const uploadComposerAttachment = Effect.fn("uploadComposerAttachment")(fu : resolveAssetUrl(connection.value.httpBaseUrl, relativeUrl); }, transport: (url) => - abortableTransfer((signal) => - uploadFileBytes(environmentId, attachment, url, signal, onProgress), - ), + Effect.tryPromise({ + try: (signal) => uploadFileBytes(environmentId, attachment, url, signal, onProgress), + catch: attachmentUploadError, + }), }); if (result.status === "uploaded") return result.attachmentId; if (result.attachmentId !== null) { diff --git a/apps/web/src/lib/attachmentUploadQueue.ts b/apps/web/src/lib/attachmentUploadQueue.ts index bf89566499e32..0e4dffcfb193f 100644 --- a/apps/web/src/lib/attachmentUploadQueue.ts +++ b/apps/web/src/lib/attachmentUploadQueue.ts @@ -8,7 +8,6 @@ import { import { parseScopedThreadKey } from "@t3tools/client-runtime/environment"; import { resolveAssetUrl } from "@t3tools/client-runtime/state/assets"; import { - abortableTransfer, attachmentUploadError, createAttachmentUploadRegistry, } from "@t3tools/client-runtime/state/attachment-uploads"; @@ -268,25 +267,27 @@ const uploadJob = Effect.fn("WebAttachmentUploads.upload")(function* (job: Uploa return connection ? resolveAssetUrl(connection.httpBaseUrl, relativeUrl) : null; }, transport: (url) => - abortableTransfer((signal) => - uploadBytes({ - url, - file, - mimeType, - signal, - onProgress: (progress) => { - const step = Math.floor(progress * 20); - if (step === lastStep || signal.aborted) return; - lastStep = step; - setUploadState(job.image.id, { - status: "uploading", - environmentId: job.environmentId, - progress, - ...(job.previous ? { previous: job.previous } : {}), - }); - }, - }), - ), + Effect.tryPromise({ + try: (signal) => + uploadBytes({ + url, + file, + mimeType, + signal, + onProgress: (progress) => { + const step = Math.floor(progress * 20); + if (step === lastStep || signal.aborted) return; + lastStep = step; + setUploadState(job.image.id, { + status: "uploading", + environmentId: job.environmentId, + progress, + ...(job.previous ? { previous: job.previous } : {}), + }); + }, + }), + catch: attachmentUploadError, + }), }); if (result.status === "uploaded") { setUploadState(job.image.id, { diff --git a/packages/client-runtime/src/state/attachmentUploads.test.ts b/packages/client-runtime/src/state/attachmentUploads.test.ts index 559bef0c4d7ae..7886a51a81f62 100644 --- a/packages/client-runtime/src/state/attachmentUploads.test.ts +++ b/packages/client-runtime/src/state/attachmentUploads.test.ts @@ -5,7 +5,7 @@ import * as Fiber from "effect/Fiber"; import { vi } from "vite-plus/test"; import { - abortableTransfer, + attachmentUploadError, createAttachmentUploadRegistry, type AttachmentUploadError, type AttachmentUploadStatus, @@ -42,7 +42,10 @@ function createQueue(options: { environmentId: (candidate) => candidate.environmentId, concurrencyPerEnvironment: options.concurrencyPerEnvironment ?? 3, upload: (candidate, report) => - abortableTransfer((signal) => options.upload(candidate, signal, report)), + Effect.tryPromise({ + try: (signal) => options.upload(candidate, signal, report), + catch: attachmentUploadError, + }), ...(options.onUploaded ? { onUploaded: options.onUploaded } : {}), onChange: options.onChange, }); @@ -175,46 +178,6 @@ describe("attachment upload registry", () => { queue.dispose(); }); - it("restarts a re-added attachment after its aborted transfer finishes settling", async () => { - const firstStarted = Promise.withResolvers(); - const firstSettled = Promise.withResolvers(); - const secondStarted = Promise.withResolvers(); - const secondSettled = Promise.withResolvers(); - let states: Readonly> = {}; - let firstSignal: AbortSignal | undefined; - const upload = vi.fn(async (_request: UploadRequest, signal: AbortSignal) => { - if (!firstSignal) { - firstSignal = signal; - firstStarted.resolve(); - return firstSettled.promise; - } - secondStarted.resolve(); - return secondSettled.promise; - }); - const queue = createQueue({ - upload, - onChange: (next) => { - states = next; - }, - }); - const local = request("re-added"); - queue.sync([local]); - await firstStarted.promise; - queue.sync([]); - queue.sync([local]); - expect(firstSignal?.aborted).toBe(true); - expect(upload).toHaveBeenCalledOnce(); - firstSettled.resolve("aborted"); - await secondStarted.promise; - expect(upload).toHaveBeenCalledTimes(2); - secondSettled.resolve("uploaded"); - await queue.settled(); - expect(states[uploadKey(environmentId, local.attachment.id)]).toEqual({ - status: "ready", - }); - queue.dispose(); - }); - it("keeps failures stable until retry and reports bounded progress", async () => { let states: Readonly> = {}; const progress: number[] = []; diff --git a/packages/client-runtime/src/state/attachmentUploads.ts b/packages/client-runtime/src/state/attachmentUploads.ts index 7a64b8592c7aa..12064d110153c 100644 --- a/packages/client-runtime/src/state/attachmentUploads.ts +++ b/packages/client-runtime/src/state/attachmentUploads.ts @@ -24,27 +24,6 @@ export function attachmentUploadError(cause: unknown): AttachmentUploadError { }); } -/** - * Runs a promise-based transfer as an Effect. Interrupting it aborts the transfer and waits for - * it to settle, so the same bytes are never sent twice at once. - */ -export function abortableTransfer( - start: (signal: AbortSignal) => Promise, -): Effect.Effect { - return Effect.suspend(() => { - const controller = new AbortController(); - const done = start(controller.signal); - return Effect.tryPromise({ try: () => done, catch: attachmentUploadError }).pipe( - Effect.onInterrupt(() => - Effect.sync(() => controller.abort()).pipe( - Effect.andThen(Effect.tryPromise({ try: () => done, catch: attachmentUploadError })), - Effect.ignore, - ), - ), - ); - }); -} - export type AttachmentUploadStatus = | { readonly status: "uploading"; readonly progress: number } | { readonly status: "ready" } @@ -69,18 +48,14 @@ class UploadKey implements Equal.Equal { /** * Every upload of an attachment, shared by key. Holders keep an upload running, and anyone who * needs its result joins the running upload instead of starting another. An upload is - * interrupted once nothing holds it, and the same attachment does not upload again until that - * interrupted upload has settled. + * interrupted once nothing holds it. */ export function createAttachmentUploadRegistry(options: { readonly key: (request: Request) => string; readonly environmentId: (request: Request) => EnvironmentId; /** Uploads running at once against one environment; the rest wait their turn. */ readonly concurrencyPerEnvironment: number; - /** - * Uploads one attachment and returns its uploaded id. Interrupting it must cancel the - * transfer and wait for it to settle. - */ + /** Uploads one attachment and returns its uploaded id; interrupting it cancels the transfer. */ readonly upload: ( request: Request, reportProgress: (progress: number) => void, @@ -118,29 +93,22 @@ export function createAttachmentUploadRegistry(options: { } return semaphore; }; - const running = new Set>(); - // The latest upload per key: a re-added attachment waits for an interrupted one to settle. - const latest = new Map>(); + const running = new Map>(); - const transfer = ( - key: UploadKey, - previous: Fiber.Fiber | undefined, - ) => { + const transfer = (key: UploadKey) => { const request = key.request!; - return (previous === undefined ? Effect.void : Fiber.await(previous)).pipe( - Effect.andThen( - Effect.suspend(() => { - if (statuses[key.id]?.status !== "uploading") { - setStatus(key.id, { status: "uploading", progress: 0 }); - } - return options.upload(request, (progress) => - setStatus(key.id, { - status: "uploading", - progress: Math.floor(Math.max(0, Math.min(1, progress)) * 20) / 20, - }), - ); - }).pipe(Semaphore.withPermits(permitsFor(options.environmentId(request)), 1)), - ), + return Effect.suspend(() => { + if (statuses[key.id]?.status !== "uploading") { + setStatus(key.id, { status: "uploading", progress: 0 }); + } + return options.upload(request, (progress) => + setStatus(key.id, { + status: "uploading", + progress: Math.floor(Math.max(0, Math.min(1, progress)) * 20) / 20, + }), + ); + }).pipe( + Semaphore.withPermits(permitsFor(options.environmentId(request)), 1), Effect.tap((uploadedId) => options.onUploaded?.(request, uploadedId) ?? Effect.void), Effect.tap(() => Effect.sync(() => setStatus(key.id, { status: "ready" }))), Effect.tapError((error) => @@ -157,14 +125,10 @@ export function createAttachmentUploadRegistry(options: { setStatus(key.id, { status: "uploading", progress: 0 }); yield* Effect.addFinalizer(() => Effect.sync(() => setStatus(key.id, undefined))); // Starts in the holder's call, so an upload begins as soon as something holds it. - const fiber = yield* Effect.forkScoped(transfer(key, latest.get(key.id)), { - startImmediately: true, - }); - latest.set(key.id, fiber); - running.add(fiber); + const fiber = yield* Effect.forkScoped(transfer(key), { startImmediately: true }); + running.set(key.id, fiber); fiber.addObserver(() => { - running.delete(fiber); - if (latest.get(key.id) === fiber) latest.delete(key.id); + if (running.get(key.id) === fiber) running.delete(key.id); }); return fiber; }), @@ -227,16 +191,16 @@ export function createAttachmentUploadRegistry(options: { synced.delete(key); sync(requested); }, - /** The upload running or last run for the key, to await it without holding it. */ + /** The key's running upload, to await it without holding it. */ current(key: string): Fiber.Fiber | undefined { - return latest.get(key); + return running.get(key); }, status(key: string): AttachmentUploadStatus | undefined { return statuses[key]; }, async settled() { while (running.size > 0) { - await Effect.runPromise(Fiber.awaitAll([...running])); + await Effect.runPromise(Fiber.awaitAll([...running.values()])); } }, dispose() { From ecaa924d1a7910d3fb584008279e526f25981c7b Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 16 Nov 2026 11:41:30 -0800 Subject: [PATCH 3/4] refactor(mobile): configure the shared upload registry where it is created The composer worker creates the client-runtime registry directly with its key, environment and concurrency, so the one-caller createComposerAttachmentUploadQueue wrapper goes. Co-Authored-By: Claude Opus 5.5 --- .../src/lib/composerAttachmentUploadQueue.ts | 42 +------------------ .../src/state/composer-attachment-uploads.ts | 20 ++++++--- 2 files changed, 15 insertions(+), 47 deletions(-) diff --git a/apps/mobile/src/lib/composerAttachmentUploadQueue.ts b/apps/mobile/src/lib/composerAttachmentUploadQueue.ts index 5e6318a31c62a..7c0e56dae08de 100644 --- a/apps/mobile/src/lib/composerAttachmentUploadQueue.ts +++ b/apps/mobile/src/lib/composerAttachmentUploadQueue.ts @@ -1,11 +1,6 @@ import { EnvironmentId, type ServerConfig } from "@t3tools/contracts"; -import type * as Effect from "effect/Effect"; import { clampFileAttachmentUploadBytes } from "@t3tools/client-runtime/state/attachments"; -import { - createAttachmentUploadRegistry, - type AttachmentUploadError, - type AttachmentUploadStatus, -} from "@t3tools/client-runtime/state/attachment-uploads"; +import type { AttachmentUploadStatus } from "@t3tools/client-runtime/state/attachment-uploads"; import { parseLegacyNewTaskDraftKey } from "../state/new-task-draft-key"; import type { DraftComposerAttachment } from "./composerImages"; @@ -107,38 +102,3 @@ export function composerAttachmentsStillUploading(input: { return state?.status !== "ready" && state?.status !== "failed"; }); } - -/** - * The composer's uploads, on the shared registry: keyed by environment and attachment, three at - * a time per environment. - */ -export function createComposerAttachmentUploadQueue(options: { - /** Uploads one attachment; interrupting it must abort the transfer and wait for it to settle. */ - readonly upload: ( - request: ComposerAttachmentUploadRequest, - reportProgress: (progress: number) => void, - ) => Effect.Effect; - /** Saves a finished upload's id; the upload is ready once this succeeds. */ - readonly onUploaded?: ( - request: ComposerAttachmentUploadRequest, - uploadedId: string, - ) => Effect.Effect; - readonly onChange: (states: Readonly>) => void; -}) { - const registry = createAttachmentUploadRegistry({ - key: (request) => composerAttachmentUploadKey(request.environmentId, request.attachment.id), - environmentId: (request) => request.environmentId, - concurrencyPerEnvironment: 3, - upload: options.upload, - ...(options.onUploaded ? { onUploaded: options.onUploaded } : {}), - onChange: options.onChange, - }); - return { - sync: registry.sync, - upload: registry.upload, - retry: (environmentId: EnvironmentId, attachmentId: string) => - registry.retry(composerAttachmentUploadKey(environmentId, attachmentId)), - settled: registry.settled, - dispose: registry.dispose, - }; -} diff --git a/apps/mobile/src/state/composer-attachment-uploads.ts b/apps/mobile/src/state/composer-attachment-uploads.ts index c80a9436636ea..2ba993dfa7b82 100644 --- a/apps/mobile/src/state/composer-attachment-uploads.ts +++ b/apps/mobile/src/state/composer-attachment-uploads.ts @@ -2,7 +2,10 @@ import { useAtomValue } from "@effect/atom-react"; import { AuthOrchestrationOperateScope, type EnvironmentId } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; -import { attachmentUploadError } from "@t3tools/client-runtime/state/attachment-uploads"; +import { + attachmentUploadError, + createAttachmentUploadRegistry, +} from "@t3tools/client-runtime/state/attachment-uploads"; import { Atom } from "effect/reactivity"; import { uploadComposerAttachment } from "../lib/attachmentUpload"; @@ -15,7 +18,6 @@ import { composerAttachmentUploadKey, composerDraftEnvironmentId, canUploadComposerAttachment, - createComposerAttachmentUploadQueue, type ComposerAttachmentUploadState, } from "../lib/composerAttachmentUploadQueue"; import { appAtomRegistry } from "./atom-registry"; @@ -52,7 +54,7 @@ export const composerAttachmentUploadsAtom = Atom.make< const uploadStateAtom = Atom.family((key: string) => Atom.map(composerAttachmentUploadsAtom, (states) => states[key]), ); -let uploadQueue: ReturnType | null = null; +let uploadQueue: ReturnType | null = null; export function useComposerAttachmentUploadState( environmentId: EnvironmentId | undefined, @@ -65,7 +67,7 @@ export function useComposerAttachmentUploadState( export function retryComposerAttachmentUpload(environmentId: EnvironmentId, attachmentId: string) { if (!readEnvironmentScope(environmentId, AuthOrchestrationOperateScope)) return; - uploadQueue?.retry(environmentId, attachmentId); + uploadQueue?.retry(composerAttachmentUploadKey(environmentId, attachmentId)); } type ComposerAttachmentUploadRequest = { @@ -238,8 +240,11 @@ const recordComposerAttachmentUpload = Effect.fn("ComposerAttachments.recordUplo ); }); -function composerAttachmentUploads() { - uploadQueue ??= createComposerAttachmentUploadQueue({ +const createComposerUploadRegistry = () => + createAttachmentUploadRegistry({ + key: (request) => composerAttachmentUploadKey(request.environmentId, request.attachment.id), + environmentId: (request) => request.environmentId, + concurrencyPerEnvironment: 3, onChange: (states) => appAtomRegistry.set(composerAttachmentUploadsAtom, states), // The file stays on disk while its upload reads it, even if its draft lets go meanwhile. upload: ({ environmentId, attachment }, reportProgress) => @@ -254,6 +259,9 @@ function composerAttachmentUploads() { ), onUploaded: recordComposerAttachmentUpload, }); + +function composerAttachmentUploads() { + uploadQueue ??= createComposerUploadRegistry(); return uploadQueue; } From b84d04a21e79ceb71e0f818cd5281c83162e1a80 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 16 Nov 2026 11:50:28 -0800 Subject: [PATCH 4/4] fix(client-runtime): restart failed uploads for sends and verify swept ones A send that joins a failed upload restarts it instead of rejoining the cached failure, so a queued message recovers once the transport does. Each upload entry owns its key's status, so an invalidated entry finishing or closing late cannot clear its replacement's. On mobile, a send that joins an upload finished before it verifies the id with the server and uploads again when the pending-upload sweep removed it, as sends did before the shared registry. Co-Authored-By: Claude Opus 5.5 --- .../state/composer-attachment-uploads.test.ts | 22 ++++++- .../src/state/composer-attachment-uploads.ts | 30 ++++++++-- .../src/state/attachmentUploads.test.ts | 49 ++++++++++++++++ .../src/state/attachmentUploads.ts | 58 ++++++++++++++----- 4 files changed, 136 insertions(+), 23 deletions(-) diff --git a/apps/mobile/src/state/composer-attachment-uploads.test.ts b/apps/mobile/src/state/composer-attachment-uploads.test.ts index c931dd1a87aaa..7d8a431150d26 100644 --- a/apps/mobile/src/state/composer-attachment-uploads.test.ts +++ b/apps/mobile/src/state/composer-attachment-uploads.test.ts @@ -143,7 +143,9 @@ vi.mock("@t3tools/client-runtime/state/runtime", async (importOriginal) => ({ ) => native.server.get(query.input.resource.attachmentId) ? { _tag: "Success", value: {} } - : { _tag: "Failure", error: { _tag: "AssetAttachmentNotFoundError" } }, + : (await import("effect/reactivity")).AsyncResult.failure( + (await import("effect/Cause")).fail({ _tag: "AssetAttachmentNotFoundError" }), + ), runAtomCommand: async ( _registry: unknown, command: unknown, @@ -324,6 +326,24 @@ it.effect.each(["decode", "write"] as const)( }), ); +it.effect("uploads again at send time when the server swept a finished upload", () => + Effect.gen(function* () { + const queue = worker(); + yield* Effect.promise(() => queue.settled()); + expect(uploadState()).toEqual({ status: "ready" }); + // The server sweeps pending uploads nothing claimed, here while the draft sat unsent. + native.server.delete("pending-1"); + + const message = queuedMessage(); + yield* Effect.promise(() => enqueueThreadOutboxMessage(message)); + const prepared = yield* prepareQueuedMessageAttachments(message); + expect(prepared.status).toBe("ready"); + if (prepared.status !== "ready") throw new Error("Outbox preparation abandoned"); + expect(prepared.prepared.pendingAttachmentIds).toEqual(["pending-2"]); + expect(native.transfer).toHaveBeenCalledTimes(2); + }), +); + it("keeps an upload running when its message is sent mid-upload and records it on the message", async () => { const transferStarted = Promise.withResolvers(); const finishTransfer = Promise.withResolvers<{ status: number }>(); diff --git a/apps/mobile/src/state/composer-attachment-uploads.ts b/apps/mobile/src/state/composer-attachment-uploads.ts index 2ba993dfa7b82..0779d153d3f23 100644 --- a/apps/mobile/src/state/composer-attachment-uploads.ts +++ b/apps/mobile/src/state/composer-attachment-uploads.ts @@ -6,6 +6,7 @@ import { attachmentUploadError, createAttachmentUploadRegistry, } from "@t3tools/client-runtime/state/attachment-uploads"; +import { verifyPersistedAttachmentUpload } from "@t3tools/client-runtime/state/attachments"; import { Atom } from "effect/reactivity"; import { uploadComposerAttachment } from "../lib/attachmentUpload"; @@ -21,6 +22,7 @@ import { type ComposerAttachmentUploadState, } from "../lib/composerAttachmentUploadQueue"; import { appAtomRegistry } from "./atom-registry"; +import { assetEnvironment } from "./assets"; import { environmentServerConfigsAtom } from "./server"; import { flattenQueuedThreadMessages, @@ -266,12 +268,28 @@ function composerAttachmentUploads() { } /** Every send uploads through here, so it joins the composer's upload of the same attachment. */ -export function uploadComposerAttachmentShared( - environmentId: EnvironmentId, - attachment: DraftComposerAttachment, -) { - return composerAttachmentUploads().upload({ environmentId, attachment }); -} +export const uploadComposerAttachmentShared = Effect.fn("ComposerAttachments.uploadForSend")( + function* (environmentId: EnvironmentId, attachment: DraftComposerAttachment) { + const uploads = composerAttachmentUploads(); + const request = { environmentId, attachment }; + const key = composerAttachmentUploadKey(environmentId, attachment.id); + // An upload that finished earlier can outlive the server's pending-upload sweep. + const finishedEarlier = uploads.status(key)?.status === "ready"; + const uploadedId = yield* uploads.upload(request); + if (!finishedEarlier) return uploadedId; + const verification = yield* verifyPersistedAttachmentUpload({ + registry: appAtomRegistry, + createAssetUrl: assetEnvironment.createUrl, + environmentId, + attachmentId: uploadedId, + }); + if (verification.status === "verified") return uploadedId; + if (verification.status === "failed") return yield* attachmentUploadError(verification.error); + // Starting over verifies the saved id again and uploads the bytes when it is gone. + uploads.restart(key); + return yield* uploads.upload(request); + }, +); /** * Uploads draft and queued attachments to their environment as soon as it is connected and diff --git a/packages/client-runtime/src/state/attachmentUploads.test.ts b/packages/client-runtime/src/state/attachmentUploads.test.ts index 7886a51a81f62..53df58ba9bdb3 100644 --- a/packages/client-runtime/src/state/attachmentUploads.test.ts +++ b/packages/client-runtime/src/state/attachmentUploads.test.ts @@ -249,4 +249,53 @@ describe("attachment upload registry", () => { queue.dispose(); }), ); + it.effect("restarts a failed upload when a send joins it instead of rejoining the failure", () => + Effect.gen(function* () { + const upload = vi + .fn() + .mockRejectedValueOnce(new Error("Server unavailable")) + .mockResolvedValue("uploaded-2"); + const queue = createQueue({ upload, onChange: () => {} }); + const local = request("photo"); + queue.sync([local]); + yield* Effect.promise(() => queue.settled()); + expect(queue.status(uploadKey(environmentId, "photo"))?.status).toBe("failed"); + + expect(yield* queue.upload(local)).toBe("uploaded-2"); + expect(upload).toHaveBeenCalledTimes(2); + queue.dispose(); + }), + ); + + it.effect("keeps a replacement upload's status when the upload it replaced is released", () => + Effect.gen(function* () { + const replacement = Promise.withResolvers(); + const upload = vi + .fn() + .mockRejectedValueOnce(new Error("Server unavailable")) + .mockImplementationOnce(() => replacement.promise); + let statuses: Readonly> = {}; + const queue = createQueue({ + upload, + onChange: (next) => { + statuses = next; + }, + }); + const local = request("photo"); + const releaseFailed = queue.hold(local); + yield* Effect.promise(() => queue.settled()); + queue.invalidate(uploadKey(environmentId, "photo")); + const releaseReplacement = queue.hold(local); + releaseFailed(); + expect(statuses[uploadKey(environmentId, "photo")]).toEqual({ + status: "uploading", + progress: 0, + }); + replacement.resolve("uploaded-2"); + yield* Effect.promise(() => queue.settled()); + expect(statuses[uploadKey(environmentId, "photo")]).toEqual({ status: "ready" }); + releaseReplacement(); + queue.dispose(); + }), + ); }); diff --git a/packages/client-runtime/src/state/attachmentUploads.ts b/packages/client-runtime/src/state/attachmentUploads.ts index 12064d110153c..a3afa29d24273 100644 --- a/packages/client-runtime/src/state/attachmentUploads.ts +++ b/packages/client-runtime/src/state/attachmentUploads.ts @@ -94,15 +94,21 @@ export function createAttachmentUploadRegistry(options: { return semaphore; }; const running = new Map>(); + // The entry that owns each key's status: an invalidated entry finishing or closing late must + // not overwrite or clear the status of the entry that replaced it. + const owners = new Map(); + const setOwnStatus = (owner: object, key: string, status: AttachmentUploadStatus | undefined) => { + if (owners.get(key) === owner) setStatus(key, status); + }; - const transfer = (key: UploadKey) => { + const transfer = (key: UploadKey, owner: object) => { const request = key.request!; return Effect.suspend(() => { if (statuses[key.id]?.status !== "uploading") { - setStatus(key.id, { status: "uploading", progress: 0 }); + setOwnStatus(owner, key.id, { status: "uploading", progress: 0 }); } return options.upload(request, (progress) => - setStatus(key.id, { + setOwnStatus(owner, key.id, { status: "uploading", progress: Math.floor(Math.max(0, Math.min(1, progress)) * 20) / 20, }), @@ -110,9 +116,9 @@ export function createAttachmentUploadRegistry(options: { }).pipe( Semaphore.withPermits(permitsFor(options.environmentId(request)), 1), Effect.tap((uploadedId) => options.onUploaded?.(request, uploadedId) ?? Effect.void), - Effect.tap(() => Effect.sync(() => setStatus(key.id, { status: "ready" }))), + Effect.tap(() => Effect.sync(() => setOwnStatus(owner, key.id, { status: "ready" }))), Effect.tapError((error) => - Effect.sync(() => setStatus(key.id, { status: "failed", reason: error.message })), + Effect.sync(() => setOwnStatus(owner, key.id, { status: "failed", reason: error.message })), ), ); }; @@ -122,10 +128,19 @@ export function createAttachmentUploadRegistry(options: { Scope.provide(scope)( RcMap.make({ lookup: Effect.fn("AttachmentUploads.lookup")(function* (key: UploadKey) { + const owner = {}; + owners.set(key.id, owner); setStatus(key.id, { status: "uploading", progress: 0 }); - yield* Effect.addFinalizer(() => Effect.sync(() => setStatus(key.id, undefined))); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + setOwnStatus(owner, key.id, undefined); + if (owners.get(key.id) === owner) owners.delete(key.id); + }), + ); // Starts in the holder's call, so an upload begins as soon as something holds it. - const fiber = yield* Effect.forkScoped(transfer(key), { startImmediately: true }); + const fiber = yield* Effect.forkScoped(transfer(key, owner), { + startImmediately: true, + }); running.set(key.id, fiber); fiber.addObserver(() => { if (running.get(key.id) === fiber) running.delete(key.id); @@ -173,23 +188,34 @@ export function createAttachmentUploadRegistry(options: { Effect.runSync(RcMap.invalidate(uploads, new UploadKey(key, null))); } + /** Drops the key's upload and starts it again for whatever `sync` holds. */ + function restart(key: string) { + invalidate(key); + synced.get(key)?.(); + synced.delete(key); + sync(requested); + } + return { hold, sync, invalidate, - /** Uploads the request, joining its running upload when there is one. */ + restart, + /** + * Uploads the request, joining its running or finished upload. A failed one starts again, + * so a retrying send never just rejoins the failure. + */ upload(request: Request): Effect.Effect { - return disposed - ? Effect.fail(attachmentUploadError(new Error("Upload cancelled."))) - : RcMap.get(uploads, keyOf(request)).pipe(Effect.flatMap(Fiber.join), Effect.scoped); + return Effect.suspend(() => { + if (disposed) return Effect.fail(attachmentUploadError(new Error("Upload cancelled."))); + const key = options.key(request); + if (statuses[key]?.status === "failed") restart(key); + return RcMap.get(uploads, keyOf(request)).pipe(Effect.flatMap(Fiber.join), Effect.scoped); + }); }, /** Starts a failed upload that `sync` holds again. */ retry(key: string) { - if (statuses[key]?.status !== "failed") return; - invalidate(key); - synced.get(key)?.(); - synced.delete(key); - sync(requested); + if (statuses[key]?.status === "failed") restart(key); }, /** The key's running upload, to await it without holding it. */ current(key: string): Fiber.Fiber | undefined {