From 7de47548cbf532b914d12c05fd4d3078e618a798 Mon Sep 17 00:00:00 2001 From: Raj Patel Date: Wed, 21 Oct 2026 18:27:52 -0700 Subject: [PATCH] Add admin import queue sections, dismissal, and safe retry controls --- app/(site)/admin/actions.test.ts | 43 ++++++++++++++++++ app/(site)/admin/actions.ts | 17 +++++++ app/(site)/admin/imports/ImportForms.tsx | 20 ++++++++- app/(site)/admin/imports/page.tsx | 31 +++++++++---- .../20261026000000_import_queue_dismissal.sql | 2 + lib/importer/admin.ts | 2 +- lib/importer/importer.ts | 35 ++++++++++++--- lib/importer/queue.test.ts | 44 +++++++++++++++++++ lib/importer/queue.ts | 40 +++++++++++++++++ lib/importer/retry.test.ts | 39 ++++++++++++++++ scripts/seed.mjs | 2 +- 11 files changed, 258 insertions(+), 17 deletions(-) create mode 100644 app/(site)/admin/actions.test.ts create mode 100644 db/migrations/20261026000000_import_queue_dismissal.sql create mode 100644 lib/importer/queue.test.ts create mode 100644 lib/importer/queue.ts create mode 100644 lib/importer/retry.test.ts diff --git a/app/(site)/admin/actions.test.ts b/app/(site)/admin/actions.test.ts new file mode 100644 index 0000000..a5c47c2 --- /dev/null +++ b/app/(site)/admin/actions.test.ts @@ -0,0 +1,43 @@ +import { beforeEach, expect, it, vi } from "vitest"; +const mocks = vi.hoisted(() => ({ user: vi.fn(), admin: vi.fn(), dismiss: vi.fn(), retry: vi.fn(), after: vi.fn(), revalidate: vi.fn() })); +vi.mock("server-only", () => ({})); +vi.mock("next/navigation", () => ({ notFound: () => { throw new Error("404"); }, redirect: vi.fn() })); +vi.mock("next/cache", () => ({ revalidatePath: mocks.revalidate })); +vi.mock("next/server", () => ({ after: mocks.after })); +vi.mock("@/lib/auth", () => ({ requireUser: mocks.user })); +vi.mock("@/lib/importer/admin", () => ({ isAdmin: mocks.admin, approve: vi.fn(), reject: vi.fn(), sendInvite: vi.fn() })); +vi.mock("@/lib/importer/queue", () => ({ dismissSource: mocks.dismiss, retrySource: mocks.retry })); +vi.mock("@/lib/importer/importer", () => ({ finishListservRun: vi.fn(), readDeck: vi.fn(), startDeck: vi.fn(), startListservRun: vi.fn() })); +import { sourceAction } from "./actions"; +const id = "10000000-0000-4000-8000-000000000001"; +const form = (decision: string, sourceId = id) => { const f = new FormData(); f.set("decision", decision); f.set("sourceId", sourceId); return f; }; +beforeEach(() => { vi.resetAllMocks(); mocks.user.mockResolvedValue({ netid: "rp4348" }); mocks.admin.mockReturnValue(true); }); +it("blocks non-admins before changing imports", async () => { + mocks.admin.mockReturnValue(false); + await expect(sourceAction(undefined, form("done"))).rejects.toThrow("404"); + expect(mocks.dismiss).not.toHaveBeenCalled(); expect(mocks.retry).not.toHaveBeenCalled(); +}); +it("rejects malformed source IDs and unknown decisions", async () => { + expect(await sourceAction(undefined, form("done", "bad"))).toHaveProperty("error"); + expect(await sourceAction(undefined, form("delete"))).toHaveProperty("error"); + expect(mocks.dismiss).not.toHaveBeenCalled(); expect(mocks.retry).not.toHaveBeenCalled(); +}); +it("clears completed sources and refreshes the queue", async () => { + mocks.dismiss.mockResolvedValue(undefined); + expect(await sourceAction(undefined, form("done"))).toHaveProperty("message"); + expect(mocks.dismiss).toHaveBeenCalledWith(id); expect(mocks.revalidate).toHaveBeenCalledWith("/admin/imports"); +}); +it("claims a retry before scheduling its slow work after the response", async () => { + const read = vi.fn(); mocks.retry.mockResolvedValue(read); + expect(await sourceAction(undefined, form("retry"))).toHaveProperty("message"); + expect(mocks.retry).toHaveBeenCalledWith(id); expect(mocks.after).toHaveBeenCalledWith(read); expect(read).not.toHaveBeenCalled(); +}); +it("shows errors without scheduling another retry", async () => { + mocks.retry.mockRejectedValue(new Error("Already being read.")); + expect(await sourceAction(undefined, form("retry"))).toEqual({error: "Already being read."}); + expect(mocks.after).not.toHaveBeenCalled(); +}); +it("keeps rows with unreviewed listings visible", async () => { + mocks.dismiss.mockRejectedValue(new Error("Review the remaining items first.")); + expect(await sourceAction(undefined, form("done"))).toEqual({error: "Review the remaining items first."}); +}); diff --git a/app/(site)/admin/actions.ts b/app/(site)/admin/actions.ts index ea72110..ddb0edb 100644 --- a/app/(site)/admin/actions.ts +++ b/app/(site)/admin/actions.ts @@ -7,6 +7,7 @@ import { after } from "next/server"; import { requireUser } from "@/lib/auth"; import { approve, isAdmin, reject, sendInvite } from "@/lib/importer/admin"; import { finishListservRun, readDeck, startDeck, startListservRun } from "@/lib/importer/importer"; +import { dismissSource, retrySource } from "@/lib/importer/queue"; import { deckIdsIn } from "@/lib/importer/rules"; async function requireAdmin() { @@ -74,3 +75,19 @@ export async function reviewAction(f: FormData) { // from a listing's preview page: back to the queue if (f.get("back") === "queue") redirect("/admin/imports"); } + +export async function sourceAction(_: ImportState, f: FormData): Promise { + await requireAdmin(); + const id = String(f.get("sourceId") ?? ""); + const decision = f.get("decision"); + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)) return { error: "Invalid import." }; + if (decision !== "done" && decision !== "retry") return { error: "Invalid action." }; + try { + if (decision === "done") await dismissSource(id); + else after(await retrySource(id)); + revalidatePath("/admin/imports"); + return { message: decision === "done" ? "Cleared from the queue. Listings are unchanged." : "Retry started. This page will update when it finishes." }; + } catch (e) { + return { error: e instanceof Error ? e.message : String(e) }; + } +} diff --git a/app/(site)/admin/imports/ImportForms.tsx b/app/(site)/admin/imports/ImportForms.tsx index 909f757..8a2761f 100644 --- a/app/(site)/admin/imports/ImportForms.tsx +++ b/app/(site)/admin/imports/ImportForms.tsx @@ -1,6 +1,6 @@ "use client"; import { useActionState } from "react"; -import { importLinkAction, inviteAction, runNowAction, type ImportState } from "../actions"; +import { importLinkAction, inviteAction, sourceAction, runNowAction, type ImportState } from "../actions"; export function ImportForms({ engineConfigured }: { engineConfigured: boolean }) { const [linkState, linkAction, linkPending] = useActionState(importLinkAction, undefined); @@ -63,3 +63,21 @@ export function InviteButton({ sourceId, invited }: { sourceId: string; invited: ); } + +export function SourceButton({ sourceId, decision, disabled = false }: { sourceId: string; decision: "done" | "retry"; disabled?: boolean }) { + const [state, action, pending] = useActionState(sourceAction, undefined); + return ( +
+ + + + {disabled && review items first} + {state?.message && {state.message}} + {state?.error && {state.error}} +
+ ); +} diff --git a/app/(site)/admin/imports/page.tsx b/app/(site)/admin/imports/page.tsx index 467201f..7b33560 100644 --- a/app/(site)/admin/imports/page.tsx +++ b/app/(site)/admin/imports/page.tsx @@ -8,7 +8,7 @@ import { netidFromEmail } from "@/lib/importer/rules"; import { formatCents } from "@/lib/rules"; import { AutoRefresh } from "../../_components/AutoRefresh"; import { reviewAction } from "../actions"; -import { ImportForms, InviteButton } from "./ImportForms"; +import { ImportForms, InviteButton, SourceButton } from "./ImportForms"; /* Listserv imports: run them, approve or reject each item before anyone sees it (the admin's approval is the consent to republish), and invite each @@ -55,11 +55,15 @@ async function Imports() { {String(lastRun.summary ?? (lastRun.finished_at ? "" : runGoing ? "still running…" : "cut off before it finished (the server restarted); run it again"))}

)} -

- waiting for review ({waiting}) -

- {queue.length === 0 &&
nothing imported yet.
} - {queue.map(({ source, items }) => ( + {[ + { title: "pending", entries: queue.filter(({ source, items }) => source.status !== "done" || items.length > 0) }, + { title: "reviewed", entries: queue.filter(({ source, items }) => source.status === "done" && items.length === 0) }, + ].map(({ title, entries }) => ( +
+

{title} ({entries.length})

+ {title === "pending" &&

{waiting} items waiting for approval

} + {entries.length === 0 &&

{title === "pending" ? "nothing pending." : "nothing reviewed yet."}

} + {entries.map(({ source, items }) => (
@@ -77,9 +81,16 @@ async function Imports() { )}
- - {source.status === "pending" ? (!olderThan(source.createdAt, STALE_PENDING_MS) ? "reading…" : "cut off") : source.status} - + {source.status === "done" ? ( + 0} /> + ) : source.status === "failed" ? ( +
+ failed + +
+ ) : ( + {!olderThan(source.createdAt, STALE_PENDING_MS) ? "reading…" : "cut off"} + )}
{source.error &&

{source.status === "failed" ? source.error : `skipped: ${source.error}`}

}
@@ -156,6 +167,8 @@ async function Imports() { )}
+ ))} +
))} ); diff --git a/db/migrations/20261026000000_import_queue_dismissal.sql b/db/migrations/20261026000000_import_queue_dismissal.sql new file mode 100644 index 0000000..3c23ea1 --- /dev/null +++ b/db/migrations/20261026000000_import_queue_dismissal.sql @@ -0,0 +1,2 @@ +-- Keep import history and deduplication while clearing completed admin rows. +alter table import_sources add column dismissed_at timestamptz; diff --git a/lib/importer/admin.ts b/lib/importer/admin.ts index c19f177..7e4fdbf 100644 --- a/lib/importer/admin.ts +++ b/lib/importer/admin.ts @@ -79,7 +79,7 @@ const toSource = (r: Row): ImportSource => ({ /* Decks newest first, each with the listings still waiting for review. */ export async function reviewQueue(): Promise<{ source: ImportSource; items: QueueItem[] }[]> { - const sources = await step("load decks", () => sql()`select * from import_sources order by created_at desc limit 100`); + const sources = await step("load decks", () => sql()`select * from import_sources where dismissed_at is null and submitted_by is null order by created_at desc limit 100`); const listings = await step("load review queue", () => sql()` select id, import_source_id, import_slide, title, price_cents, size, section, sub, condition, description, photo_paths, status from listings where status = 'review' and import_source_id is not null order by import_slide`); diff --git a/lib/importer/importer.ts b/lib/importer/importer.ts index 4b8ccfe..65bd434 100644 --- a/lib/importer/importer.ts +++ b/lib/importer/importer.ts @@ -214,7 +214,17 @@ export async function startDeck(deckId: string, email: EmailMeta, owner?: User): } return { skipped: { deckId, status: "skipped", detail: "already imported", created: 0 } }; } - if (existing) await step("clear failed attempt", () => sql()`delete from import_sources where id = ${existing.id as string}`); + if (existing) { + const [claimed] = await sql()`update import_sources set status = 'pending', error = null, created_at = now(), + message_id = ${email.messageId}, subject = ${email.subject}, listserv = ${email.listserv}, + author_name = ${email.authorName}, author_email = ${email.authorEmail?.toLowerCase() ?? null}, + sent_at = ${email.sentAt}, submitted_by = ${owner?.netid ?? null} + where id = ${existing.id as string} + and (status = 'failed' or (status = 'pending' and created_at < now() - interval '15 minutes')) + and not exists (select 1 from listings where import_source_id = ${existing.id as string}) + returning id`; + return claimed ? { sourceId: claimed.id as string } : { skipped: { deckId, status: "skipped", detail: "already reading or has imported items", created: 0 } }; + } const record = { deck_id: deckId, @@ -366,10 +376,17 @@ async function fetchPhoto(url: string): Promise { /* An email selling items in its body, with photos attached. Read once: emails that turn out not to be sales are recorded too, so they're never paid for twice. */ -export async function importEmailSale(m: EngineMessage, photoUrls: string[]): Promise { - const [existing] = await step("find email", () => sql()`select id, status from import_sources where kind = 'email' and message_id = ${m.id}`); +export async function importEmailSale(m: EngineMessage, photoUrls: string[], claimedSourceId?: string): Promise { + const [existing] = claimedSourceId ? [] : await step("find email", () => sql()`select id, status from import_sources where kind = 'email' and message_id = ${m.id}`); if (existing && existing.status !== "failed") return { messageId: m.id, status: "skipped", detail: "already read", created: 0 }; - if (existing) await step("clear failed attempt", () => sql()`delete from import_sources where id = ${existing.id as string}`); + if (existing) { + const [claimed] = await sql()`update import_sources set status = 'pending', error = null, created_at = now() + where id = ${existing.id as string} and status = 'failed' + and not exists (select 1 from listings where import_source_id = ${existing.id as string}) + returning id`; + if (!claimed) return { messageId: m.id, status: "skipped", detail: "already reading or has imported items", created: 0 }; + claimedSourceId = claimed.id as string; + } const author = saleAuthor(m.sender, m.senderEmail); const record = { @@ -382,7 +399,7 @@ export async function importEmailSale(m: EngineMessage, photoUrls: string[]): Pr author_email: author.email, sent_at: m.sentAt, }; - const [source] = await step("record email", () => sql()`insert into import_sources ${sql()(record)} returning id`); + const [source] = claimedSourceId ? [{ id: claimedSourceId }] : await step("record email", () => sql()`insert into import_sources ${sql()(record)} returning id`); const finish = async (found: number, why: string | null, contact?: SaleContact, notes: string[] = []) => { const patch = { status: "done", slides: null, items_found: found, error: why, ...(contact ? await contactColumns(contact, notes) : {}) }; await step("finish email", () => sql()`update import_sources set ${sql()(patch)} where id = ${source.id as string}`); @@ -776,3 +793,11 @@ export async function finishListservRun(run: { id: string; since: string }): Pro throw e; } } + +// Retry a specific email using its original source row and fresh InboxEngine data. +export async function readEmailSource(sourceId: string, messageId: string): Promise { + const message = await engine()(`/v1/messages/${encodeURIComponent(messageId)}`); + const candidate = bodySaleCandidate(message); + if (!candidate.ok) throw new Error("This email no longer has sale photos that can be imported."); + await importEmailSale(message, candidate.photos, sourceId); +} diff --git a/lib/importer/queue.test.ts b/lib/importer/queue.test.ts new file mode 100644 index 0000000..a68c2fc --- /dev/null +++ b/lib/importer/queue.test.ts @@ -0,0 +1,44 @@ +import { beforeEach, expect, it, vi } from "vitest"; +const m = vi.hoisted(() => ({ query: vi.fn(), deck: vi.fn(), email: vi.fn() })); +vi.mock("server-only", () => ({})); +vi.mock("../db", () => ({ sql: () => m.query })); +vi.mock("./importer", () => ({ readDeck: m.deck, readEmailSource: m.email })); +import { dismissSource, retrySource } from "./queue"; +beforeEach(() => vi.resetAllMocks()); +it("does not read anything when another request has claimed the source", async () => { + m.query.mockResolvedValue([]); + await expect(retrySource("id")).rejects.toThrow("already running"); + expect(m.deck).not.toHaveBeenCalled(); expect(m.email).not.toHaveBeenCalled(); +}); +it("retains the source ID and seller metadata on a deck retry", async () => { + m.query.mockResolvedValue([{kind: "deck", deck_id: "slides", message_id: "mail", subject: "Sale", listserv: "butler", author_name: "Seller", author_email: "seller@princeton.edu", sent_at: "2026-10-21"}]); + const read = await retrySource("source-id"); + expect(m.deck).not.toHaveBeenCalled(); + await read(); + expect(m.deck).toHaveBeenCalledWith("source-id", "slides", {messageId: "mail", subject: "Sale", text: "", listserv: "butler", authorName: "Seller", authorEmail: "seller@princeton.edu", sentAt: "2026-10-21"}); +}); +it("retries email sources through the email reader with the same ID", async () => { + m.query.mockResolvedValue([{kind: "email", message_id: "mail"}]); + await (await retrySource("source-id"))(); + expect(m.email).toHaveBeenCalledWith("source-id", "mail"); expect(m.deck).not.toHaveBeenCalled(); +}); +it("records background fetch failures so they can be retried again", async () => { + m.query.mockResolvedValue([{kind: "email", message_id: "mail"}]); + m.email.mockRejectedValue(new Error("Inbox unavailable")); + await (await retrySource("source-id"))(); + expect(m.query).toHaveBeenCalledTimes(2); + expect(m.query.mock.calls[1].slice(1)).toEqual(["Inbox unavailable", "source-id"]); +}); +it("rejects dismissal when the database guard does not match", async () => { + m.query.mockResolvedValue([]); + await expect(dismissSource("source-id")).rejects.toThrow("Review the remaining items"); +}); +it("clears a completed row with one update, without deleting listings", async () => { + m.query.mockResolvedValue([{id: "source-id"}]); + await dismissSource("source-id"); + expect(m.query).toHaveBeenCalledTimes(1); + const statement = m.query.mock.calls[0][0].join("?"); + expect(statement).toContain("status = 'done'"); + expect(statement).toContain("status = 'review'"); + expect(statement).not.toMatch(/delete/i); +}); diff --git a/lib/importer/queue.ts b/lib/importer/queue.ts new file mode 100644 index 0000000..9d71b42 --- /dev/null +++ b/lib/importer/queue.ts @@ -0,0 +1,40 @@ +import "server-only"; +import { sql, type Row } from "../db"; +import { readDeck, readEmailSource } from "./importer"; + +export async function dismissSource(id: string): Promise { + const rows = await sql()`update import_sources set dismissed_at = now() + where id = ${id} and status = 'done' and submitted_by is null + and not exists (select 1 from listings where import_source_id = ${id} and status = 'review') + returning id`; + if (!rows.length) throw new Error("Review the remaining items first; only completed imports can be cleared."); +} + +// Claim before returning to the browser. Concurrent clicks cannot start two readers. +// Never rerun a partial import with listings: that could duplicate published items. +export async function retrySource(id: string): Promise<() => Promise> { + const [source] = await sql()`update import_sources + set status = 'pending', error = null, created_at = now(), dismissed_at = null + where id = ${id} and status = 'failed' and submitted_by is null + and not exists (select 1 from listings where import_source_id = ${id}) + returning *`; + if (!source) throw new Error("This import is already running, completed, or has items that need review. Refresh the page."); + return async () => { + try { + if (source.kind === "email") { + await readEmailSource(id, source.message_id as string); + } else { + await readDeck(id, source.deck_id as string, { + messageId: source.message_id as string | null, + subject: (source.subject as string) ?? "", text: "", + listserv: source.listserv as string | null, + authorName: source.author_name as string | null, + authorEmail: source.author_email as string | null, + sentAt: source.sent_at as string | null, + }); + } + } catch (e) { + await sql()`update import_sources set status = 'failed', error = ${e instanceof Error ? e.message : String(e)} where id = ${id}`; + } + }; +} diff --git a/lib/importer/retry.test.ts b/lib/importer/retry.test.ts new file mode 100644 index 0000000..6411925 --- /dev/null +++ b/lib/importer/retry.test.ts @@ -0,0 +1,39 @@ +import { beforeEach, expect, it, vi } from "vitest"; +const { query } = vi.hoisted(() => ({ query: vi.fn() })); +vi.mock("server-only", () => ({})); +vi.mock("../db", () => ({ sql: () => query })); +import type { User } from "../data"; +import { startDeck, importEmailSale } from "./importer"; +const meta = {messageId: null, subject: "Sale", text: "", listserv: null, authorName: null, authorEmail: null, sentAt: null}; +beforeEach(() => vi.resetAllMocks()); +it("skips a deck if a manual retry claims it between lookup and update", async () => { + query.mockResolvedValueOnce([{id: "old", status: "failed"}]).mockResolvedValueOnce([]); + expect(await startDeck("deck", meta)).toHaveProperty("skipped.status", "skipped"); + expect(query).toHaveBeenCalledTimes(2); + const statement = query.mock.calls[1][0].join("?"); + expect(statement).toContain("update import_sources"); + expect(statement).toContain("not exists"); + expect(statement).not.toContain("delete"); +}); +it("reuses the existing deck source when the claim succeeds", async () => { + query.mockResolvedValueOnce([{id: "old", status: "failed"}]).mockResolvedValueOnce([{id: "old"}]); + expect(await startDeck("deck", meta)).toEqual({sourceId: "old"}); + expect(query).toHaveBeenCalledTimes(2); +}); +it("skips an email if another reader wins the claim", async () => { + query.mockResolvedValueOnce([{id: "old", status: "failed"}]).mockResolvedValueOnce([]); + const message = {id: "mail", subject: "Sale", body: "", sender: "Seller", senderEmail: "seller@princeton.edu", sentAt: "2026-10-21", url: "https://example.com", listservs: [], links: [], media: []}; + expect(await importEmailSale(message, [])).toHaveProperty("status", "skipped"); + expect(query).toHaveBeenCalledTimes(2); + expect(query.mock.calls[1][0].join("?")).toContain("status = 'failed'"); +}); + +it("records seller ownership when a seller retries a failed admin deck", async () => { + query.mockResolvedValueOnce([{id: "old", status: "failed"}]).mockResolvedValueOnce([{id: "old"}]); + const owner = {netid: "seller"} as User; + await startDeck("deck", {...meta, authorEmail: "seller@princeton.edu"}, owner); + const claim = query.mock.calls[1]; + expect(claim[0].join("?")).toContain("submitted_by ="); + expect(claim.slice(1)).toContain("seller"); + expect(claim.slice(1)).toContain("seller@princeton.edu"); +}); diff --git a/scripts/seed.mjs b/scripts/seed.mjs index c6793d7..8f1807a 100644 --- a/scripts/seed.mjs +++ b/scripts/seed.mjs @@ -104,7 +104,7 @@ for (const [i, l] of listings.entries()) { } const [row] = await insert("listings", { seller_netid: seller, - title: l.title, + title: `[seed] ${l.title}`, description: l.desc ?? "", price_cents: Math.round(l.price * 100), section: l.cat,