diff --git a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx index c6fa629d18b43..70770d681c89c 100644 --- a/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx +++ b/apps/mobile/src/features/threads/NewTaskDraftScreen.tsx @@ -42,7 +42,7 @@ import { } from "@t3tools/contracts"; import { ComposerEditor, type ComposerEditorHandle } from "../../components/ComposerEditor"; -import { composerContextImportsAtom } from "../../state/use-composer-drafts"; +import { composerContextImportsAtom, composerDraftBusy } from "../../state/use-composer-drafts"; import { abortComposerDraftOperations } from "../../state/composerDraftOperations"; import { composerContextSendBlockReason } from "../../lib/composerContext"; import { ComposerInlineControl } from "../../components/ComposerToolbar"; @@ -601,7 +601,7 @@ export function NewTaskDraftScreen(props: { return null; } const draft = getComposerDraftSnapshot(draftKey); - if (appAtomRegistry.get(composerContextImportsAtom)[draftKey]) return null; + if (composerDraftBusy(draftKey)) return null; const modelSelection = resolveSelectableModelSelection( selectedEnvironmentServerConfig, diff --git a/apps/mobile/src/features/threads/ThreadComposer.tsx b/apps/mobile/src/features/threads/ThreadComposer.tsx index a28c03843b163..7db707a376bed 100644 --- a/apps/mobile/src/features/threads/ThreadComposer.tsx +++ b/apps/mobile/src/features/threads/ThreadComposer.tsx @@ -46,6 +46,7 @@ import { getComposerDraftSnapshot, setComposerDraftText, composerContextImportsAtom, + composerPendingMediaAtom, countComposerDraftAttachmentsAfterSelection, useComposerDraft, useComposerDraftResetGeneration, @@ -304,9 +305,11 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps context: draftContext, } = useComposerDraft(composerDraftKey); const queuedEdit = props.queuedEdit ?? null; + const pendingMedia = useAtomValue(composerPendingMediaAtom)[composerDraftKey]; const hasContent = draftMessage.trim().length > 0 || draftAttachments.length > 0 || + pendingMedia !== undefined || (queuedEdit?.existingAttachments.length ?? 0) > 0; const showStopAction = !hasContent && props.canStopThread && queuedEdit === null; const onResumeQueue = props.onResumeQueue; @@ -325,10 +328,16 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps AuthOrchestrationOperateScope, ); const uploadStates = useAtomValue(composerAttachmentUploadsAtom); - const stripAttachments = composerStripAttachments( - draftAttachments, - (attachment) => uploadStates[composerAttachmentUploadKey(props.environmentId, attachment.id)], - ); + // A finished attachment keeps its placeholder's id; skip the placeholder once it lands. + const stripAttachments = [ + ...composerStripAttachments( + draftAttachments, + (attachment) => uploadStates[composerAttachmentUploadKey(props.environmentId, attachment.id)], + ), + ...(pendingMedia ?? []).filter( + (placeholder) => !draftAttachments.some(({ id }) => id === placeholder.id), + ), + ]; const attachmentsUploading = props.connectionState === "connected" && composerAttachmentsStillUploading({ @@ -499,6 +508,7 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps (props.canOperateThread || props.connectionState !== "connected") && hasContent && !contextImports[composerDraftKey] && + pendingMedia === undefined && !isSubmissionLocked && sendBlockedReason === null && !modelUnavailable; @@ -514,7 +524,7 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps (current) => current ?? { ...source, - gallery: filePreviewGallery(source, composerImageGallery(draftAttachments)), + gallery: filePreviewGallery(source, composerImageGallery(stripAttachments)), }, ); }; diff --git a/apps/mobile/src/lib/composerFiles.test.ts b/apps/mobile/src/lib/composerFiles.test.ts index bdcd35c36ff75..311bd68866a0b 100644 --- a/apps/mobile/src/lib/composerFiles.test.ts +++ b/apps/mobile/src/lib/composerFiles.test.ts @@ -160,7 +160,7 @@ describe("composer file attachments", () => { }); it.each(["image/heic", "image/heif", undefined])( - "renders a %s photo to JPEG natively and previews the rendered file", + "renders a %s photo to JPEG natively and previews the picker copy", async (mimeType) => { mocks.pickMedia.mockResolvedValue({ canceled: false, @@ -181,7 +181,7 @@ describe("composer file attachments", () => { mimeType: "image/jpeg", sizeBytes: 4, dataUrl: `data:image/jpeg;base64,${rendered.base64}`, - previewUri: rendered.uri, + previewUri: photo.uri, }, ], error: null, @@ -248,12 +248,14 @@ describe("composer file attachments", () => { name: "IMG_9280.png", mimeType: "image/png", sizeBytes: last.width * last.height * 4, - previewUri: `file:///cache/fitted-${last.width}x${last.height}.png`, + previewUri: photo.uri, }), ); expect(image!.sizeBytes).toBeLessThanOrEqual(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES); expect(mocks.delete).toHaveBeenCalledWith("file:///cache/fitted-1397x2000.png"); - expect(mocks.delete).not.toHaveBeenCalledWith(image!.previewUri); + expect(mocks.delete).toHaveBeenCalledWith( + `file:///cache/fitted-${last.width}x${last.height}.png`, + ); }); it("measures the picker file instead of trusting the reported size", async () => { @@ -362,8 +364,17 @@ describe("composer file attachments", () => { uri.endsWith("clip.mov") ? video.fileSize : 3, ); - const result = await pickComposerMedia({ existingCount: 0, maxVideoBytes: 50 * 1024 * 1024 }); + const onPicked = vi.fn(); + const result = await pickComposerMedia({ + existingCount: 0, + maxVideoBytes: 50 * 1024 * 1024, + onPicked, + }); + expect(onPicked).toHaveBeenCalledWith([ + expect.objectContaining({ id: "attachment-id", type: "image", previewUri: image.uri }), + expect.objectContaining({ id: "attachment-id", type: "file", fileUri: video.uri }), + ]); expect(mocks.pickMedia).toHaveBeenCalledWith( expect.objectContaining({ mediaTypes: ["images", "videos"], diff --git a/apps/mobile/src/lib/composerImages.ts b/apps/mobile/src/lib/composerImages.ts index 69f26103134b9..90ef6e3fdcc28 100644 --- a/apps/mobile/src/lib/composerImages.ts +++ b/apps/mobile/src/lib/composerImages.ts @@ -400,6 +400,8 @@ export async function pickComposerImages(input: { readonly existingCount: number export async function pickComposerMedia(input: { readonly existingCount: number; readonly maxVideoBytes?: number; + /** Receives placeholders over the picker's local copies before processing; each keeps its attachment's id. */ + readonly onPicked?: (placeholders: ReadonlyArray) => void; readonly acceptSelection?: () => boolean; }): Promise<{ readonly attachments: ReadonlyArray; @@ -455,11 +457,28 @@ export async function pickComposerMedia(input: { error: null, }; } + const ids = result.assets.map(() => uuidv4()); + input.onPicked?.( + result.assets.flatMap((asset, index): DraftComposerAttachment[] => { + const common = { + id: ids[index]!, + name: asset.fileName ?? "", + mimeType: asset.mimeType ?? "", + sizeBytes: asset.fileSize ?? 0, + }; + if (asset.type === "image" || asset.mimeType?.startsWith("image/")) { + return [{ ...common, type: "image", previewUri: asset.uri }]; + } + return input.maxVideoBytes === undefined + ? [] + : [{ ...common, type: "file", fileUri: asset.uri }]; + }), + ); const attachments: DraftComposerAttachment[] = []; let error: string | null = null; - for (const asset of result.assets) { + for (const [index, asset] of result.assets.entries()) { if (attachments.length >= remainingSlots) { error = m( msg("You can attach up to {max} attachments per message.", { @@ -477,15 +496,16 @@ export async function pickComposerMedia(input: { try { const { File } = await import("expo-file-system"); const file = new File(asset.uri); - attachments.push( - await createComposerFileAttachment({ + attachments.push({ + ...(await createComposerFileAttachment({ uri: asset.uri, name: asset.fileName?.trim() || file.name || "video", mimeType: mimeType || file.type || "application/octet-stream", sizeBytes: asset.fileSize ?? null, maxBytes: clampFileAttachmentUploadBytes(input.maxVideoBytes), - }), - ); + })), + id: ids[index]!, + }); } catch (cause) { error = errorDisplayMessage(cause, m) ?? @@ -517,7 +537,8 @@ export async function pickComposerMedia(input: { ? mimeType : null; - let image: { base64: string; mimeType: string; name: string; previewUri: string }; + let image: { base64: string; mimeType: string; name: string }; + let renderedUri: string | undefined; try { if (originalMimeType !== null) { const { File } = await import("expo-file-system"); @@ -527,25 +548,34 @@ export async function pickComposerMedia(input: { mimeType: originalMimeType, base64, }); + renderedUri = fitted?.uri; image = { base64: fitted?.base64 ?? base64, mimeType: originalMimeType, name, - previewUri: fitted?.uri ?? asset.uri, }; } else { const rendered = await renderPhoto(asset.uri); + renderedUri = rendered.uri; image = { base64: rendered.base64, mimeType: "image/jpeg", name: /\.jpe?g$/i.test(name) ? name : `${name.replace(/\.[^.]+$/, "")}.jpg`, - previewUri: rendered.uri, }; } } catch { error = m(msg("Failed to read ''{name}''.", { name })); continue; } + // The bytes now live in the data URL, and the preview uses the picker copy. + if (renderedUri !== undefined) { + try { + const { File } = await import("expo-file-system"); + new File(renderedUri).delete(); + } catch (cause) { + console.warn("[composer-images] could not remove rendered photo", cause); + } + } const sizeBytes = estimateBase64ByteSize(image.base64); if (sizeBytes <= 0 || sizeBytes > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { @@ -554,13 +584,14 @@ export async function pickComposerMedia(input: { } attachments.push({ - id: uuidv4(), + id: ids[index]!, type: "image", name: image.name, mimeType: image.mimeType, sizeBytes, dataUrl: `data:${image.mimeType};base64,${image.base64}`, - previewUri: image.previewUri, + // The picker copy is already on screen as the placeholder; reusing it avoids a reload flash. + previewUri: asset.uri, }); } diff --git a/apps/mobile/src/state/thread-task-permissions.test.ts b/apps/mobile/src/state/thread-task-permissions.test.ts index 0be50e8c2280e..944a00e31b68d 100644 --- a/apps/mobile/src/state/thread-task-permissions.test.ts +++ b/apps/mobile/src/state/thread-task-permissions.test.ts @@ -206,6 +206,7 @@ vi.mock("./use-composer-drafts", async () => { return { composerDraftsAtom: Atom.make({}), composerContextImportsAtom: Atom.make({}), + composerDraftBusy: () => false, getComposerDraftSnapshot: () => state.draft, isFreshSessionPending: () => state.freshPending, setFreshSessionPending: state.setFreshPending, diff --git a/apps/mobile/src/state/use-composer-attachment-actions.ts b/apps/mobile/src/state/use-composer-attachment-actions.ts index 3e9c3439d4164..ec13b98a53e42 100644 --- a/apps/mobile/src/state/use-composer-attachment-actions.ts +++ b/apps/mobile/src/state/use-composer-attachment-actions.ts @@ -20,6 +20,9 @@ import { captureComposerDraftInsertion, countComposerDraftAttachmentsAfterSelection, getComposerDraftSnapshot, + addComposerPendingMedia, + keepPendingComposerMedia, + removeComposerPendingMedia, type ComposerDraftInsertion, } from "./use-composer-drafts"; import { startComposerDraftOperation } from "./composerDraftOperations"; @@ -72,15 +75,24 @@ export function useComposerAttachmentActions(props: { if (!key || props.locked) return; const insertion = captureComposerDraftInsertion(key); await withDraftOperation(key, async (signal) => { - const result = await pickComposerMedia({ + let placeholderIds: ReadonlyArray = []; + await pickComposerMedia({ existingCount: countComposerDraftAttachmentsAfterSelection(key, insertion), maxVideoBytes: capabilities?.fileAttachments?.maxUploadBytes, - }); - report( - m(msg("Could not attach photo or video")), - result.error, - await append(key, signal, result.attachments, insertion), - ); + onPicked: (placeholders) => { + placeholderIds = placeholders.map(({ id }) => id); + addComposerPendingMedia(key, placeholders); + }, + }) + .then(async (result) => { + // append writes the draft before its first await, so no removal lands in between. + report( + m(msg("Could not attach photo or video")), + result.error, + await append(key, signal, keepPendingComposerMedia(key, result.attachments), insertion), + ); + }) + .finally(() => removeComposerPendingMedia(key, placeholderIds)); }); }; const onPickDraftFiles = async () => { diff --git a/apps/mobile/src/state/use-composer-drafts.test.ts b/apps/mobile/src/state/use-composer-drafts.test.ts index fd1c248f5de9b..66d0eb6d42bbc 100644 --- a/apps/mobile/src/state/use-composer-drafts.test.ts +++ b/apps/mobile/src/state/use-composer-drafts.test.ts @@ -210,6 +210,9 @@ import { findNewTaskDraftKeys, flushComposerDrafts, getComposerDraftSnapshot, + addComposerPendingMedia, + keepPendingComposerMedia, + removeComposerDraftAttachment, mergeComposerDraftContentState, migrateLegacyNewTaskDraft, modelOptionMemoryAtom, @@ -1166,6 +1169,36 @@ describe("mobile composer drafts", () => { expect(appAtomRegistry.get(composerDraftsAtom)[draftKey]?.attachments).toHaveLength(101); }); + it("releases picked media whose placeholder was removed while it processed", async () => { + const outboxLoad = vi.spyOn(threadOutboxManager, "load").mockResolvedValue(true); + onTestFinished(() => outboxLoad.mockRestore()); + const cleanup = Promise.withResolvers(); + composerAttachmentCleanupMocks.remove.mockImplementationOnce(async () => { + cleanup.resolve(); + }); + const video = (id: string) => ({ + id, + type: "file" as const, + name: `${id}.mov`, + mimeType: "video/quicktime", + sizeBytes: 42, + fileUri: `file:///documents/t3-composer-attachments/${id}.mov`, + }); + const draftKey = "thread-pending-media"; + appAtomRegistry.set(composerDraftsAtom, { [draftKey]: { text: "", attachments: [] } }); + addComposerPendingMedia(draftKey, [video("kept"), video("removed")]); + + removeComposerDraftAttachment(draftKey, "removed"); + + expect(keepPendingComposerMedia(draftKey, [video("kept"), video("removed")])).toEqual([ + video("kept"), + ]); + await cleanup.promise; + expect(composerAttachmentCleanupMocks.remove).toHaveBeenCalledExactlyOnceWith( + video("removed").fileUri, + ); + }); + it("drops duplicate attachments, releases their files, and keeps them on restore", async () => { const outboxLoad = vi.spyOn(threadOutboxManager, "load").mockResolvedValue(true); onTestFinished(() => outboxLoad.mockRestore()); diff --git a/apps/mobile/src/state/use-composer-drafts.ts b/apps/mobile/src/state/use-composer-drafts.ts index 0020defdc5915..472d59e182ef6 100644 --- a/apps/mobile/src/state/use-composer-drafts.ts +++ b/apps/mobile/src/state/use-composer-drafts.ts @@ -91,6 +91,59 @@ export function setComposerContextImporting(draftKey: string, importing: boolean appAtomRegistry.set(composerContextImportsAtom, next); } +/** Placeholders for picked media still being processed, shown in the strip until their attachments land. */ +export const composerPendingMediaAtom = Atom.make< + Record> +>({}).pipe(Atom.keepAlive); + +function updateComposerPendingMedia( + draftKey: string, + update: (current: ReadonlyArray) => DraftComposerAttachment[], +): void { + const all = appAtomRegistry.get(composerPendingMediaAtom); + const current = all[draftKey] ?? []; + const placeholders = update(current); + if (placeholders.length === current.length) return; + const next = { ...all }; + if (placeholders.length > 0) next[draftKey] = placeholders; + else delete next[draftKey]; + appAtomRegistry.set(composerPendingMediaAtom, next); +} + +export function addComposerPendingMedia( + draftKey: string, + placeholders: ReadonlyArray, +): void { + updateComposerPendingMedia(draftKey, (current) => [...current, ...placeholders]); +} + +export function removeComposerPendingMedia(draftKey: string, ids: ReadonlyArray): void { + updateComposerPendingMedia(draftKey, (current) => + current.filter((placeholder) => !ids.includes(placeholder.id)), + ); +} + +/** Picked attachments whose placeholder the user removed while processing are released, not added. */ +export function keepPendingComposerMedia( + draftKey: string, + attachments: ReadonlyArray, +): ReadonlyArray { + const pending = appAtomRegistry.get(composerPendingMediaAtom)[draftKey] ?? []; + const kept = attachments.filter(({ id }) => pending.some((placeholder) => placeholder.id === id)); + scheduleUnusedComposerAttachmentCleanup( + attachments.filter((attachment) => !kept.includes(attachment)), + ); + return kept; +} + +/** A draft can't be sent while context is importing or picked media is still processing. */ +export function composerDraftBusy(draftKey: string): boolean { + return ( + appAtomRegistry.get(composerContextImportsAtom)[draftKey] === true || + appAtomRegistry.get(composerPendingMediaAtom)[draftKey] !== undefined + ); +} + interface ComposerDraftSelection { readonly text: string; readonly start: number; @@ -1548,6 +1601,7 @@ export function replaceComposerDraftAttachments( } export function removeComposerDraftAttachment(draftKey: string, imageId: string): void { + removeComposerPendingMedia(draftKey, [imageId]); const previousAttachments = getComposerDraftSnapshot(draftKey).attachments; updateComposerDrafts((current) => { const existing = normalizeDraft(current[draftKey]); diff --git a/apps/mobile/src/state/use-thread-composer-state.ts b/apps/mobile/src/state/use-thread-composer-state.ts index 48e7fc02b5b32..4059ee0c46ce7 100644 --- a/apps/mobile/src/state/use-thread-composer-state.ts +++ b/apps/mobile/src/state/use-thread-composer-state.ts @@ -92,7 +92,7 @@ import { insertComposerDraftContext, clearComposerDraft, clearComposerDraftContent, - composerContextImportsAtom, + composerDraftBusy, getComposerDraftSnapshot, isFreshSessionPending, mergeComposerDraftContent, @@ -508,7 +508,9 @@ export function useThreadComposerState(input: { const threadKey = scopedThreadKey(thread.environmentId, thread.id); const edit = getQueuedRunEdit(threadKey); if (edit === null) return; - const draft = getComposerDraftSnapshot(queuedEditDraftKey(threadKey, edit.runId)); + const editDraftKey = queuedEditDraftKey(threadKey, edit.runId); + if (composerDraftBusy(editDraftKey)) return; + const draft = getComposerDraftSnapshot(editDraftKey); const text = draft.text.trim(); if ( text.length === 0 && @@ -649,7 +651,7 @@ export function useThreadComposerState(input: { if (startingSideChatRef.current) return null; const threadKey = scopedThreadKey(selectedThreadShell.environmentId, selectedThreadShell.id); const draft = getComposerDraftSnapshot(threadKey); - if (appAtomRegistry.get(composerContextImportsAtom)[threadKey]) return null; + if (composerDraftBusy(threadKey)) return null; const thread = selectedThreadShell; const text = draft.text.trim(); const attachments = draft.attachments; @@ -851,7 +853,7 @@ export function useThreadComposerState(input: { if (!response) return null; // A context paste still landing, or any draft or settings change while approvals were // declined, means this snapshot is no longer what the user sent. - if (appAtomRegistry.get(composerContextImportsAtom)[threadKey]) return null; + if (composerDraftBusy(threadKey)) return null; const current = getComposerDraftSnapshot(threadKey); if ( current.text !== draft.text ||