Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions app/(site)/admin/actions.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import { beforeEach, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({ user: vi.fn(), admin: vi.fn(), dismiss: vi.fn(), retry: vi.fn(), after: vi.fn(), revalidate: vi.fn() }));
vi.mock("server-only", () => ({}));
vi.mock("next/navigation", () => ({ notFound: () => { throw new Error("404"); }, redirect: vi.fn() }));
vi.mock("next/cache", () => ({ revalidatePath: mocks.revalidate }));
vi.mock("next/server", () => ({ after: mocks.after }));
vi.mock("@/lib/auth", () => ({ requireUser: mocks.user }));
vi.mock("@/lib/importer/admin", () => ({ isAdmin: mocks.admin, approve: vi.fn(), reject: vi.fn(), sendInvite: vi.fn() }));
vi.mock("@/lib/importer/queue", () => ({ dismissSource: mocks.dismiss, retrySource: mocks.retry }));
vi.mock("@/lib/importer/importer", () => ({ finishListservRun: vi.fn(), readDeck: vi.fn(), startDeck: vi.fn(), startListservRun: vi.fn() }));
import { sourceAction } from "./actions";
const id = "10000000-0000-4000-8000-000000000001";
const form = (decision: string, sourceId = id) => { const f = new FormData(); f.set("decision", decision); f.set("sourceId", sourceId); return f; };
beforeEach(() => { vi.resetAllMocks(); mocks.user.mockResolvedValue({ netid: "rp4348" }); mocks.admin.mockReturnValue(true); });
it("blocks non-admins before changing imports", async () => {
mocks.admin.mockReturnValue(false);
await expect(sourceAction(undefined, form("done"))).rejects.toThrow("404");
expect(mocks.dismiss).not.toHaveBeenCalled(); expect(mocks.retry).not.toHaveBeenCalled();
});
it("rejects malformed source IDs and unknown decisions", async () => {
expect(await sourceAction(undefined, form("done", "bad"))).toHaveProperty("error");
expect(await sourceAction(undefined, form("delete"))).toHaveProperty("error");
expect(mocks.dismiss).not.toHaveBeenCalled(); expect(mocks.retry).not.toHaveBeenCalled();
});
it("clears completed sources and refreshes the queue", async () => {
mocks.dismiss.mockResolvedValue(undefined);
expect(await sourceAction(undefined, form("done"))).toHaveProperty("message");
expect(mocks.dismiss).toHaveBeenCalledWith(id); expect(mocks.revalidate).toHaveBeenCalledWith("/admin/imports");
});
it("claims a retry before scheduling its slow work after the response", async () => {
const read = vi.fn(); mocks.retry.mockResolvedValue(read);
expect(await sourceAction(undefined, form("retry"))).toHaveProperty("message");
expect(mocks.retry).toHaveBeenCalledWith(id); expect(mocks.after).toHaveBeenCalledWith(read); expect(read).not.toHaveBeenCalled();
});
it("shows errors without scheduling another retry", async () => {
mocks.retry.mockRejectedValue(new Error("Already being read."));
expect(await sourceAction(undefined, form("retry"))).toEqual({error: "Already being read."});
expect(mocks.after).not.toHaveBeenCalled();
});
it("keeps rows with unreviewed listings visible", async () => {
mocks.dismiss.mockRejectedValue(new Error("Review the remaining items first."));
expect(await sourceAction(undefined, form("done"))).toEqual({error: "Review the remaining items first."});
});
17 changes: 17 additions & 0 deletions app/(site)/admin/actions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { after } from "next/server";
import { requireUser } from "@/lib/auth";
import { approve, isAdmin, reject, sendInvite } from "@/lib/importer/admin";
import { finishListservRun, readDeck, startDeck, startListservRun } from "@/lib/importer/importer";
import { dismissSource, retrySource } from "@/lib/importer/queue";
import { deckIdsIn } from "@/lib/importer/rules";

async function requireAdmin() {
Expand Down Expand Up @@ -74,3 +75,19 @@ export async function reviewAction(f: FormData) {
// from a listing's preview page: back to the queue
if (f.get("back") === "queue") redirect("/admin/imports");
}

export async function sourceAction(_: ImportState, f: FormData): Promise<ImportState> {
await requireAdmin();
const id = String(f.get("sourceId") ?? "");
const decision = f.get("decision");
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)) return { error: "Invalid import." };
if (decision !== "done" && decision !== "retry") return { error: "Invalid action." };
try {
if (decision === "done") await dismissSource(id);
else after(await retrySource(id));
revalidatePath("/admin/imports");
return { message: decision === "done" ? "Cleared from the queue. Listings are unchanged." : "Retry started. This page will update when it finishes." };
} catch (e) {
return { error: e instanceof Error ? e.message : String(e) };
}
}
20 changes: 19 additions & 1 deletion app/(site)/admin/imports/ImportForms.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"use client";
import { useActionState } from "react";
import { importLinkAction, inviteAction, runNowAction, type ImportState } from "../actions";
import { importLinkAction, inviteAction, sourceAction, runNowAction, type ImportState } from "../actions";

export function ImportForms({ engineConfigured }: { engineConfigured: boolean }) {
const [linkState, linkAction, linkPending] = useActionState<ImportState, FormData>(importLinkAction, undefined);
Expand Down Expand Up @@ -63,3 +63,21 @@ export function InviteButton({ sourceId, invited }: { sourceId: string; invited:
</form>
);
}

export function SourceButton({ sourceId, decision, disabled = false }: { sourceId: string; decision: "done" | "retry"; disabled?: boolean }) {
const [state, action, pending] = useActionState<ImportState, FormData>(sourceAction, undefined);
return (
<form action={action} style={{ display: "flex", gap: 8, alignItems: "center", flexWrap: "wrap" }}>
<input type="hidden" name="sourceId" value={sourceId} />
<input type="hidden" name="decision" value={decision} />
<button className="btn btn-outline" disabled={disabled || pending}
title={decision === "done" ? (disabled ? "Approve or reject the remaining items first" : "Clear this completed import from the queue") : "Retry this failed import"}
style={{ padding: "6px 14px" }}>
{pending ? (decision === "done" ? "clearing…" : "starting…") : decision}
</button>
{disabled && <span className="sub">review items first</span>}
{state?.message && <span role="status" className="note">{state.message}</span>}
{state?.error && <span role="alert" className="err">{state.error}</span>}
</form>
);
}
31 changes: 22 additions & 9 deletions app/(site)/admin/imports/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { netidFromEmail } from "@/lib/importer/rules";
import { formatCents } from "@/lib/rules";
import { AutoRefresh } from "../../_components/AutoRefresh";
import { reviewAction } from "../actions";
import { ImportForms, InviteButton } from "./ImportForms";
import { ImportForms, InviteButton, SourceButton } from "./ImportForms";

/* Listserv imports: run them, approve or reject each item before anyone sees
it (the admin's approval is the consent to republish), and invite each
Expand Down Expand Up @@ -55,11 +55,15 @@ async function Imports() {
{String(lastRun.summary ?? (lastRun.finished_at ? "" : runGoing ? "still running…" : "cut off before it finished (the server restarted); run it again"))}
</p>
)}
<h2 className="sec-title" style={{ fontSize: 20, margin: "18px 0 10px" }}>
waiting for review ({waiting})
</h2>
{queue.length === 0 && <div className="empty">nothing imported yet.</div>}
{queue.map(({ source, items }) => (
{[
{ title: "pending", entries: queue.filter(({ source, items }) => source.status !== "done" || items.length > 0) },
{ title: "reviewed", entries: queue.filter(({ source, items }) => source.status === "done" && items.length === 0) },
].map(({ title, entries }) => (
<section key={title} aria-label={title}>
<h2 className="sec-title" style={{ fontSize: 20, margin: "18px 0 10px" }}>{title} ({entries.length})</h2>
{title === "pending" && <p className="sub">{waiting} items waiting for approval</p>}
{entries.length === 0 && <p className="sub">{title === "pending" ? "nothing pending." : "nothing reviewed yet."}</p>}
{entries.map(({ source, items }) => (
<div key={source.id} className="sect review-deck">
<div className="sec-head">
<div>
Expand All @@ -77,9 +81,16 @@ async function Imports() {
)}
</span>
</div>
<span className={`ostat ${source.status === "done" ? "accepted" : ""}`}>
{source.status === "pending" ? (!olderThan(source.createdAt, STALE_PENDING_MS) ? "reading…" : "cut off") : source.status}
</span>
{source.status === "done" ? (
<SourceButton sourceId={source.id} decision="done" disabled={items.length > 0} />
) : source.status === "failed" ? (
<div style={{ display: "flex", gap: 8, alignItems: "center" }}>
<span className="err">failed</span>
<SourceButton sourceId={source.id} decision="retry" />
</div>
) : (
<span className="ostat">{!olderThan(source.createdAt, STALE_PENDING_MS) ? "reading…" : "cut off"}</span>
)}
</div>
{source.error && <p className={source.status === "failed" ? "err" : "sub"}>{source.status === "failed" ? source.error : `skipped: ${source.error}`}</p>}
<div style={{ display: "flex", gap: 10, alignItems: "center", flexWrap: "wrap", marginTop: 6 }}>
Expand Down Expand Up @@ -156,6 +167,8 @@ async function Imports() {
</>
)}
</div>
))}
</section>
))}
</>
);
Expand Down
2 changes: 2 additions & 0 deletions db/migrations/20261026000000_import_queue_dismissal.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
-- Keep import history and deduplication while clearing completed admin rows.
alter table import_sources add column dismissed_at timestamptz;
2 changes: 1 addition & 1 deletion lib/importer/admin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ const toSource = (r: Row): ImportSource => ({

/* Decks newest first, each with the listings still waiting for review. */
export async function reviewQueue(): Promise<{ source: ImportSource; items: QueueItem[] }[]> {
const sources = await step("load decks", () => sql()<Row[]>`select * from import_sources order by created_at desc limit 100`);
const sources = await step("load decks", () => sql()<Row[]>`select * from import_sources where dismissed_at is null and submitted_by is null order by created_at desc limit 100`);
const listings = await step("load review queue", () => sql()<Row[]>`
select id, import_source_id, import_slide, title, price_cents, size, section, sub, condition, description, photo_paths, status
from listings where status = 'review' and import_source_id is not null order by import_slide`);
Expand Down
35 changes: 30 additions & 5 deletions lib/importer/importer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,17 @@ export async function startDeck(deckId: string, email: EmailMeta, owner?: User):
}
return { skipped: { deckId, status: "skipped", detail: "already imported", created: 0 } };
}
if (existing) await step("clear failed attempt", () => sql()`delete from import_sources where id = ${existing.id as string}`);
if (existing) {
const [claimed] = await sql()<Row[]>`update import_sources set status = 'pending', error = null, created_at = now(),
message_id = ${email.messageId}, subject = ${email.subject}, listserv = ${email.listserv},
author_name = ${email.authorName}, author_email = ${email.authorEmail?.toLowerCase() ?? null},
sent_at = ${email.sentAt}, submitted_by = ${owner?.netid ?? null}
where id = ${existing.id as string}
and (status = 'failed' or (status = 'pending' and created_at < now() - interval '15 minutes'))
and not exists (select 1 from listings where import_source_id = ${existing.id as string})
returning id`;
return claimed ? { sourceId: claimed.id as string } : { skipped: { deckId, status: "skipped", detail: "already reading or has imported items", created: 0 } };
}

const record = {
deck_id: deckId,
Expand Down Expand Up @@ -366,10 +376,17 @@ async function fetchPhoto(url: string): Promise<Buffer | null> {
/* An email selling items in its body, with photos attached. Read once:
emails that turn out not to be sales are recorded too, so they're never
paid for twice. */
export async function importEmailSale(m: EngineMessage, photoUrls: string[]): Promise<EmailSaleResult> {
const [existing] = await step("find email", () => sql()<Row[]>`select id, status from import_sources where kind = 'email' and message_id = ${m.id}`);
export async function importEmailSale(m: EngineMessage, photoUrls: string[], claimedSourceId?: string): Promise<EmailSaleResult> {
const [existing] = claimedSourceId ? [] : await step("find email", () => sql()<Row[]>`select id, status from import_sources where kind = 'email' and message_id = ${m.id}`);
if (existing && existing.status !== "failed") return { messageId: m.id, status: "skipped", detail: "already read", created: 0 };
if (existing) await step("clear failed attempt", () => sql()`delete from import_sources where id = ${existing.id as string}`);
if (existing) {
const [claimed] = await sql()<Row[]>`update import_sources set status = 'pending', error = null, created_at = now()
where id = ${existing.id as string} and status = 'failed'
and not exists (select 1 from listings where import_source_id = ${existing.id as string})
returning id`;
if (!claimed) return { messageId: m.id, status: "skipped", detail: "already reading or has imported items", created: 0 };
claimedSourceId = claimed.id as string;
}

const author = saleAuthor(m.sender, m.senderEmail);
const record = {
Expand All @@ -382,7 +399,7 @@ export async function importEmailSale(m: EngineMessage, photoUrls: string[]): Pr
author_email: author.email,
sent_at: m.sentAt,
};
const [source] = await step("record email", () => sql()<Row[]>`insert into import_sources ${sql()(record)} returning id`);
const [source] = claimedSourceId ? [{ id: claimedSourceId }] : await step("record email", () => sql()<Row[]>`insert into import_sources ${sql()(record)} returning id`);
const finish = async (found: number, why: string | null, contact?: SaleContact, notes: string[] = []) => {
const patch = { status: "done", slides: null, items_found: found, error: why, ...(contact ? await contactColumns(contact, notes) : {}) };
await step("finish email", () => sql()`update import_sources set ${sql()(patch)} where id = ${source.id as string}`);
Expand Down Expand Up @@ -776,3 +793,11 @@ export async function finishListservRun(run: { id: string; since: string }): Pro
throw e;
}
}

// Retry a specific email using its original source row and fresh InboxEngine data.
export async function readEmailSource(sourceId: string, messageId: string): Promise<void> {
const message = await engine()<EngineMessage>(`/v1/messages/${encodeURIComponent(messageId)}`);
const candidate = bodySaleCandidate(message);
if (!candidate.ok) throw new Error("This email no longer has sale photos that can be imported.");
await importEmailSale(message, candidate.photos, sourceId);
}
44 changes: 44 additions & 0 deletions lib/importer/queue.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import { beforeEach, expect, it, vi } from "vitest";
const m = vi.hoisted(() => ({ query: vi.fn(), deck: vi.fn(), email: vi.fn() }));
vi.mock("server-only", () => ({}));
vi.mock("../db", () => ({ sql: () => m.query }));
vi.mock("./importer", () => ({ readDeck: m.deck, readEmailSource: m.email }));
import { dismissSource, retrySource } from "./queue";
beforeEach(() => vi.resetAllMocks());
it("does not read anything when another request has claimed the source", async () => {
m.query.mockResolvedValue([]);
await expect(retrySource("id")).rejects.toThrow("already running");
expect(m.deck).not.toHaveBeenCalled(); expect(m.email).not.toHaveBeenCalled();
});
it("retains the source ID and seller metadata on a deck retry", async () => {
m.query.mockResolvedValue([{kind: "deck", deck_id: "slides", message_id: "mail", subject: "Sale", listserv: "butler", author_name: "Seller", author_email: "seller@princeton.edu", sent_at: "2026-10-21"}]);
const read = await retrySource("source-id");
expect(m.deck).not.toHaveBeenCalled();
await read();
expect(m.deck).toHaveBeenCalledWith("source-id", "slides", {messageId: "mail", subject: "Sale", text: "", listserv: "butler", authorName: "Seller", authorEmail: "seller@princeton.edu", sentAt: "2026-10-21"});
});
it("retries email sources through the email reader with the same ID", async () => {
m.query.mockResolvedValue([{kind: "email", message_id: "mail"}]);
await (await retrySource("source-id"))();
expect(m.email).toHaveBeenCalledWith("source-id", "mail"); expect(m.deck).not.toHaveBeenCalled();
});
it("records background fetch failures so they can be retried again", async () => {
m.query.mockResolvedValue([{kind: "email", message_id: "mail"}]);
m.email.mockRejectedValue(new Error("Inbox unavailable"));
await (await retrySource("source-id"))();
expect(m.query).toHaveBeenCalledTimes(2);
expect(m.query.mock.calls[1].slice(1)).toEqual(["Inbox unavailable", "source-id"]);
});
it("rejects dismissal when the database guard does not match", async () => {
m.query.mockResolvedValue([]);
await expect(dismissSource("source-id")).rejects.toThrow("Review the remaining items");
});
it("clears a completed row with one update, without deleting listings", async () => {
m.query.mockResolvedValue([{id: "source-id"}]);
await dismissSource("source-id");
expect(m.query).toHaveBeenCalledTimes(1);
const statement = m.query.mock.calls[0][0].join("?");
expect(statement).toContain("status = 'done'");
expect(statement).toContain("status = 'review'");
expect(statement).not.toMatch(/delete/i);
});
40 changes: 40 additions & 0 deletions lib/importer/queue.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import "server-only";
import { sql, type Row } from "../db";
import { readDeck, readEmailSource } from "./importer";

export async function dismissSource(id: string): Promise<void> {
const rows = await sql()`update import_sources set dismissed_at = now()
where id = ${id} and status = 'done' and submitted_by is null
and not exists (select 1 from listings where import_source_id = ${id} and status = 'review')
returning id`;
if (!rows.length) throw new Error("Review the remaining items first; only completed imports can be cleared.");
}

// Claim before returning to the browser. Concurrent clicks cannot start two readers.
// Never rerun a partial import with listings: that could duplicate published items.
export async function retrySource(id: string): Promise<() => Promise<void>> {
const [source] = await sql()<Row[]>`update import_sources
set status = 'pending', error = null, created_at = now(), dismissed_at = null
where id = ${id} and status = 'failed' and submitted_by is null
and not exists (select 1 from listings where import_source_id = ${id})
returning *`;
if (!source) throw new Error("This import is already running, completed, or has items that need review. Refresh the page.");
return async () => {
try {
if (source.kind === "email") {
await readEmailSource(id, source.message_id as string);
} else {
await readDeck(id, source.deck_id as string, {
messageId: source.message_id as string | null,
subject: (source.subject as string) ?? "", text: "",
listserv: source.listserv as string | null,
authorName: source.author_name as string | null,
authorEmail: source.author_email as string | null,
sentAt: source.sent_at as string | null,
});
}
} catch (e) {
await sql()`update import_sources set status = 'failed', error = ${e instanceof Error ? e.message : String(e)} where id = ${id}`;
}
};
}
Loading
Loading