From 02515eab063d1823fe7cbd08064d35d88443f920 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Sat, 21 Nov 2026 21:55:53 +0200 Subject: [PATCH 1/4] fix(release-mirror): publish a release in one run, and on demand A release reached the update feeds two or three runs (20-30 minutes) after it was published. The worker logs show why: - Runs were stopped at the 30 s CPU limit. The Stable feed step read back every installer the download-page step had just copied, to hash it with the manifest's SHA-512 (30.6 s of CPU for 1.3 GB on 11-19). - GitHub answered some runs with 429 at the round ten minutes. Changes: - limits.cpu_ms = 300000; a channel's release needs about 40 s of CPU. - The Stable feed runs before the download-page mirror, so the mirror step finds every installer already verified and hashes nothing again. - GitHub fetches retry 429 and gateway errors, up to three attempts, honoring Retry-After up to 30 s. - The schedule moves to three minutes past every ten. - POST /sync?channel=stable|beta (workers.dev, bearer token in the MIRROR_SYNC_TOKEN secret, 404 while unset) runs the mirror now and streams one JSON line per step, for the desktop release workflow. Co-Authored-By: Claude Opus 5.5 --- README.md | 31 ++-- docs/architecture/desktop-downloads.md | 31 +++- workers/release-mirror/src/github.test.ts | 69 +++++++++ workers/release-mirror/src/github.ts | 37 +++++ workers/release-mirror/src/index.test.ts | 138 ++++++++++++++++++ workers/release-mirror/src/index.ts | 167 ++++++++++++++++++---- workers/release-mirror/src/mirror.test.ts | 37 ++++- workers/release-mirror/src/mirror.ts | 19 +-- workers/release-mirror/src/releaseFeed.ts | 11 +- workers/release-mirror/src/updateFeed.ts | 19 +-- workers/release-mirror/wrangler.jsonc | 12 +- 11 files changed, 504 insertions(+), 67 deletions(-) create mode 100644 workers/release-mirror/src/github.test.ts create mode 100644 workers/release-mirror/src/github.ts create mode 100644 workers/release-mirror/src/index.test.ts diff --git a/README.md b/README.md index 153af6d..970670a 100644 --- a/README.md +++ b/README.md @@ -58,21 +58,35 @@ Stable and Beta electron-updater sets; see deployments do not deploy it. 1. After merge, from `main`: `bun run mirror:types:check && bun run mirror:deploy`. -2. Watch the next runs (every ten minutes) with +2. Watch the next runs (at three minutes past every ten, and whenever a desktop release is + published) with `bunx wrangler tail scientfactory-release-mirror --config workers/release-mirror/wrangler.jsonc`. Expect `Stable update feed synchronized` and `Beta update feed synchronized` with `promoted` naming three manifests each. The first run per release copies up to about 1.8 GB per channel and - hashes it; on the Workers Free plan (50 external subrequests, 10 ms CPU) it cannot finish, - so confirm the account is on Workers Paid. A run cut short resumes from what is already in the - bucket. -3. In the `scientfactory.com` zone, the `Scient download cache policy` ruleset (C1) + hashes it in about 40 seconds of CPU. It needs Workers Paid and the configured + `limits.cpu_ms` of 300000; on the Workers Free plan (50 external subrequests, 10 ms CPU) it + cannot finish. A run cut short resumes from what is already in the bucket. +3. The release workflow's trigger needs one shared random token, set in both places (never + printed or committed): + + ```sh + token=$(openssl rand -hex 32) + printf %s "$token" | bunx wrangler secret put MIRROR_SYNC_TOKEN --config workers/release-mirror/wrangler.jsonc + printf %s "$token" | gh secret set RELEASE_MIRROR_SYNC_TOKEN --repo ScientFactory/scient-desktop + gh secret set RELEASE_MIRROR_SYNC_URL --repo ScientFactory/scient-desktop \ + --body "https://scientfactory-release-mirror..workers.dev/sync" + ``` + + Without the secret, `POST /sync` answers 404 and only the schedule runs. + +4. In the `scientfactory.com` zone, the `Scient download cache policy` ruleset (C1) applies only to the `downloads.scientfactory.com` hostname: - `/scient-desktop/updates/*` and `/scient-desktop/stable.json`: **Bypass cache**. - `/scient-desktop/v*`: **Eligible for cache**, edge and browser TTLs **respect origin**, with **no-store** for HTTP 300–599. Keep errors and redirects out of immutable caches. Without such a rule, Cloudflare caches some 404s and does not cache `.AppImage`, `.blockmap` or `.yml` by default. Files above 512 MB are not cached on non-Enterprise plans. -4. Verify each channel (`stable`/`latest`, then `beta`/`beta`): +5. Verify each channel (`stable`/`latest`, then `beta`/`beta`): ```sh feed=https://downloads.scientfactory.com/scient-desktop/updates/stable @@ -90,10 +104,11 @@ deployments do not deploy it. curl -sS -o /dev/null -w '%{http_code}\n' https://downloads.scientfactory.com/scient-desktop/v0.0.0/missing.exe # 404, twice: never HIT ``` -5. Only then merge the desktop change that makes these feeds the updater's default, and qualify +6. Only then merge the desktop change that makes these feeds the updater's default, and qualify real updates on each platform from a build that uses them. -To stop publishing, remove the cron trigger and redeploy; the bucket's objects stay valid. Never +To stop publishing, remove the cron trigger, delete the `MIRROR_SYNC_TOKEN` secret and redeploy; +the bucket's objects stay valid. Never delete or rewrite a versioned object: installed clients and caches may still use it. ## Scient Docs publishing diff --git a/docs/architecture/desktop-downloads.md b/docs/architecture/desktop-downloads.md index 20176de..ae22eae 100644 --- a/docs/architecture/desktop-downloads.md +++ b/docs/architecture/desktop-downloads.md @@ -28,9 +28,15 @@ download path, or `downloads.scientfactory.com/scient-desktop/v//` whose size @@ -94,6 +100,25 @@ how long a 404 is kept (see the runbook in the README). Deploy the worker with `bun run mirror:deploy`. The Pages project reads the bucket through its `DESKTOP_DOWNLOADS` binding in `wrangler.jsonc`. +## Run on publish + +The desktop release workflow (`.github/workflows/release.yml` in `scient-desktop`) calls +`POST https://.workers.dev/sync?channel=` right after it publishes a Stable +or Beta release, with `Authorization: Bearer` and the token held as the worker secret +`MIRROR_SYNC_TOKEN` and the desktop repository secret `RELEASE_MIRROR_SYNC_TOKEN` (the address is +`RELEASE_MIRROR_SYNC_URL`). The worker answers 404 while the secret is unset, 401 for any other +token, and otherwise streams one JSON object per line: each step's result, a heartbeat every 15 +seconds while it copies, and finally `{"message":"Release mirror finished","failures":n}`. The +run lasts only while the caller stays connected, since an HTTP invocation has no wall-time limit +but stops when its client leaves; whatever it copied stays, and the next scheduled run continues +from the bucket. The scheduled runs remain the fallback whenever the call fails. + +The worker allows 5 minutes of CPU per invocation (`limits.cpu_ms`). Hashing one channel's +release (about 1.8 GB, SHA-256 by R2 and SHA-512 while streaming) takes about 40 seconds; with +the default 30 seconds, and the Stable pointer step's installers hashed a second time by the +feed step, the first run per release was cut short, and a release reached the feeds two or three +runs (20–30 minutes) after it was published. + ## Why Before this, every cold edge cache asked GitHub for the handoff while the visitor waited, diff --git a/workers/release-mirror/src/github.test.ts b/workers/release-mirror/src/github.test.ts new file mode 100644 index 0000000..6d93484 --- /dev/null +++ b/workers/release-mirror/src/github.test.ts @@ -0,0 +1,69 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { fetchFromGitHub } from "./github"; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +function answers(...statuses: Array]>) { + const fetchMock = vi.fn(); + for (const status of statuses) { + const [code, headers] = Array.isArray(status) ? status : [status, {}]; + fetchMock.mockResolvedValueOnce( + new Response(code === 200 ? "ok" : null, { status: code, headers }), + ); + } + vi.stubGlobal("fetch", fetchMock); + return fetchMock; +} + +describe("fetchFromGitHub", () => { + it("waits as long as GitHub asks, then tries again", async () => { + const fetchMock = answers([429, { "Retry-After": "2" }], 200); + const sleep = vi.fn(async () => undefined); + + const response = await fetchFromGitHub("https://github.com/x", {}, 1_000, sleep); + + expect(response.status).toBe(200); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(sleep).toHaveBeenCalledWith(2_000); + }); + + it("caps a long Retry-After and backs off when GitHub gives none", async () => { + answers([503, { "Retry-After": "600" }], 502, 200); + const sleep = vi.fn(async () => undefined); + + await fetchFromGitHub("https://github.com/x", {}, 1_000, sleep); + + expect(sleep.mock.calls).toEqual([[30_000], [10_000]]); + }); + + it("returns the third refusal for the caller to report", async () => { + const fetchMock = answers(429, 429, 429); + + const response = await fetchFromGitHub( + "https://github.com/x", + {}, + 1_000, + async () => undefined, + ); + + expect(response.status).toBe(429); + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it("does not retry an answer that will not change", async () => { + const fetchMock = answers(404); + + const response = await fetchFromGitHub( + "https://github.com/x", + {}, + 1_000, + async () => undefined, + ); + + expect(response.status).toBe(404); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/workers/release-mirror/src/github.ts b/workers/release-mirror/src/github.ts new file mode 100644 index 0000000..4105d49 --- /dev/null +++ b/workers/release-mirror/src/github.ts @@ -0,0 +1,37 @@ +// FILE: github.ts +// Purpose: Fetches from github.com for the release mirror, retrying GitHub's brief refusals. +// Layer: Cloudflare Worker + +/** Statuses GitHub uses for a refusal or outage that usually clears within seconds. */ +const RETRYABLE = new Set([429, 502, 503, 504]); +const ATTEMPTS = 3; +const MAX_RETRY_AFTER_MS = 30_000; + +/** How long GitHub asked us to wait, or a short growing pause when it did not say. */ +function retryDelay(response: Response, attempt: number): number { + const header = response.headers.get("Retry-After"); + const seconds = header === null ? Number.NaN : Number(header); + if (Number.isFinite(seconds) && seconds >= 0) { + return Math.min(seconds * 1000, MAX_RETRY_AFTER_MS); + } + return 5_000 * attempt; +} + +/** + * `fetch` that tries again, up to three attempts in all, when GitHub answers 429 or a gateway + * error. Each attempt gets its own timeout. Any other answer, including the last refusal, is + * returned for the caller to judge. + */ +export async function fetchFromGitHub( + url: string, + headers: Record, + timeoutMs: number, + sleep: (ms: number) => Promise = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), +): Promise { + for (let attempt = 1; ; attempt += 1) { + const response = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) }); + if (!RETRYABLE.has(response.status) || attempt >= ATTEMPTS) return response; + await response.body?.cancel(); + await sleep(retryDelay(response, attempt)); + } +} diff --git a/workers/release-mirror/src/index.test.ts b/workers/release-mirror/src/index.test.ts new file mode 100644 index 0000000..63cea01 --- /dev/null +++ b/workers/release-mirror/src/index.test.ts @@ -0,0 +1,138 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const calls: string[] = []; +const failing = new Set(); + +vi.mock("./mirror", () => ({ + syncDesktopMirror: vi.fn(async () => { + calls.push("mirror"); + if (failing.has("mirror")) throw new Error("GitHub returned 429 for the release handoff."); + return { tag: "v1.0.0", copied: [], promoted: false, superseded: false }; + }), +})); +vi.mock("./updateFeed", () => ({ + syncUpdateFeed: vi.fn(async (_bucket: unknown, channel: string) => { + calls.push(channel); + return { channel, tag: "v1.0.0", copied: [], promoted: [], superseded: [] }; + }), +})); + +const { default: worker, runMirror } = await import("./index"); + +const bucket = {} as R2Bucket; +const TOKEN = "s3cret-token"; + +function env(token: string | undefined = TOKEN) { + return { + DESKTOP_DOWNLOADS: bucket, + ...(token === undefined ? {} : { MIRROR_SYNC_TOKEN: token }), + }; +} + +function sync(path: string, init: RequestInit = {}, token?: string) { + const request = new Request(`https://mirror.example${path}`, { method: "POST", ...init }); + return worker.fetch(request as never, env(token) as never); +} + +async function lines(response: Response) { + return (await response.text()) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); +} + +beforeEach(() => { + calls.length = 0; + failing.clear(); + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("runMirror", () => { + it("publishes the Stable feed before the download-page mirror, so installers are hashed once", async () => { + await runMirror(bucket, new Date(), ["stable", "beta"], () => undefined); + expect(calls).toEqual(["stable", "mirror", "beta"]); + }); + + it("keeps going after a failed step and counts it", async () => { + failing.add("mirror"); + const report: Array> = []; + + const failures = await runMirror(bucket, new Date(), ["stable", "beta"], (line) => + report.push(line), + ); + + expect(failures).toBe(1); + expect(calls).toEqual(["stable", "mirror", "beta"]); + expect(report.map((line) => line.message)).toEqual([ + "Stable update feed synchronized", + "Desktop mirror failed", + "Beta update feed synchronized", + ]); + expect(report[1]!.error).toMatch(/429/); + }); + + it("runs only the requested channel", async () => { + await runMirror(bucket, new Date(), ["beta"], () => undefined); + expect(calls).toEqual(["beta"]); + }); +}); + +describe("scheduled run", () => { + it("runs every channel and fails the invocation when a step failed", async () => { + failing.add("mirror"); + const controller = { + scheduledTime: Date.now(), + cron: "3-59/10 * * * *", + } as ScheduledController; + + await expect(worker.scheduled(controller, env() as never)).rejects.toThrow( + /1 release mirror step/, + ); + expect(calls).toEqual(["stable", "mirror", "beta"]); + }); +}); + +describe("POST /sync", () => { + const auth = { headers: { Authorization: `Bearer ${TOKEN}` } }; + + it("streams each step and a final summary", async () => { + const response = await sync("/sync?channel=stable", auth); + + expect(response.status).toBe(200); + const report = await lines(response); + expect(report.map((line) => line.message)).toEqual([ + "Release mirror started", + "Stable update feed synchronized", + "Desktop mirror synchronized", + "Release mirror finished", + ]); + expect(report.at(-1)).toMatchObject({ failures: 0 }); + expect(calls).toEqual(["stable", "mirror"]); + }); + + it("does not exist while no token is configured", async () => { + const response = await sync("/sync", auth, ""); + expect(response.status).toBe(404); + expect(calls).toEqual([]); + }); + + it("refuses a missing or wrong token", async () => { + expect((await sync("/sync")).status).toBe(401); + expect( + (await sync("/sync", { headers: { Authorization: "Bearer s3cret-tokem" } })).status, + ).toBe(401); + expect(calls).toEqual([]); + }); + + it("answers only POST /sync with a known channel", async () => { + expect((await sync("/other", auth)).status).toBe(404); + expect((await sync("/sync", { ...auth, method: "GET" })).status).toBe(405); + expect((await sync("/sync?channel=nightly", auth)).status).toBe(400); + expect(calls).toEqual([]); + }); +}); diff --git a/workers/release-mirror/src/index.ts b/workers/release-mirror/src/index.ts index 35c6d1d..1bc5b8a 100644 --- a/workers/release-mirror/src/index.ts +++ b/workers/release-mirror/src/index.ts @@ -1,40 +1,147 @@ // FILE: index.ts // Purpose: Keeps downloads.scientfactory.com's copy of the Stable desktop installers and the // desktop app's Stable and Beta update feeds current. -// Layer: Cloudflare Worker (cron only; no public routes) +// Layer: Cloudflare Worker (cron, plus one authenticated trigger for the release workflow) import { syncDesktopMirror } from "./mirror"; import { syncUpdateFeed } from "./updateFeed"; -export default { - async scheduled(controller, env) { - const bucket = env.DESKTOP_DOWNLOADS; - // Each step is independent: one failing (a release still uploading, GitHub refusing) must not - // hold back the others. Steps run in turn so a release's shared installers are copied once. - const steps = [ - [ - "Desktop mirror synchronized", - () => syncDesktopMirror(bucket, new Date(controller.scheduledTime)), - ], - ["Stable update feed synchronized", () => syncUpdateFeed(bucket, "stable")], - ["Beta update feed synchronized", () => syncUpdateFeed(bucket, "beta")], - ] as const; - const failures: unknown[] = []; - for (const [message, run] of steps) { - try { - console.log(JSON.stringify({ message, ...(await run()) })); - } catch (error) { - failures.push(error); - console.error( - JSON.stringify({ - message: message.replace("synchronized", "failed"), - error: error instanceof Error ? error.message : String(error), - }), - ); - } +export type MirrorChannel = "stable" | "beta"; + +interface MirrorEnv extends Env { + /** + * Bearer token the desktop release workflow sends to `POST /sync` right after it publishes. + * Unset, the trigger answers 404 and only the schedule runs. + */ + readonly MIRROR_SYNC_TOKEN?: string; +} + +/** One line of a run's report, as logged and as streamed to the release workflow. */ +export type MirrorReportLine = Readonly> & { readonly message: string }; + +/** + * Runs the steps for `channels` in turn and reports each one. Each step is independent: one + * failing (a release still uploading, GitHub refusing) must not hold back the others. + * + * The Stable update feed runs before the download-page mirror. The feed copies every Stable + * installer with its update manifest's SHA-512 as well as the handoff's SHA-256, so the mirror + * step then finds each one already verified. In the other order, the feed step had to read every + * installer back from R2 to hash it again, which pushed a release past the CPU limit. + */ +export async function runMirror( + bucket: R2Bucket, + now: Date, + channels: readonly MirrorChannel[], + report: (line: MirrorReportLine) => void, +): Promise { + const steps: Array Promise]> = []; + if (channels.includes("stable")) { + steps.push(["Stable update feed synchronized", () => syncUpdateFeed(bucket, "stable")]); + steps.push(["Desktop mirror synchronized", () => syncDesktopMirror(bucket, now)]); + } + if (channels.includes("beta")) { + steps.push(["Beta update feed synchronized", () => syncUpdateFeed(bucket, "beta")]); + } + let failures = 0; + for (const [message, run] of steps) { + try { + report({ message, ...(await run()) }); + } catch (error) { + failures += 1; + report({ + message: message.replace("synchronized", "failed"), + error: error instanceof Error ? error.message : String(error), + }); } - if (failures.length > 0) { - throw new AggregateError(failures, `${failures.length} release mirror step(s) failed.`); + } + return failures; +} + +const ALL_CHANNELS: readonly MirrorChannel[] = ["stable", "beta"]; +const HEARTBEAT_MS = 15_000; + +function logLine(line: MirrorReportLine): void { + const text = JSON.stringify(line); + if ("error" in line) console.error(text); + else console.log(text); +} + +async function sameToken(given: string, expected: string): Promise { + const encoder = new TextEncoder(); + const [a, b] = await Promise.all([ + crypto.subtle.digest("SHA-256", encoder.encode(given)), + crypto.subtle.digest("SHA-256", encoder.encode(expected)), + ]); + // Equal-length digests compared without an early exit, so timing reveals nothing of the token. + const x = new Uint8Array(a); + const y = new Uint8Array(b); + let difference = 0; + for (let index = 0; index < x.length; index += 1) difference |= x[index]! ^ y[index]!; + return difference === 0; +} + +/** + * `POST /sync?channel=stable|beta` runs the mirror now instead of at the next scheduled tick. + * The report streams back as one JSON object per line, with a heartbeat while a step copies, + * and ends with `{"message":"Release mirror finished","failures":n}`. The run lasts only while + * the caller stays connected; a run cut short resumes from the bucket at the next tick. + */ +async function handleSync(request: Request, env: MirrorEnv): Promise { + const url = new URL(request.url); + const expected = env.MIRROR_SYNC_TOKEN; + if (url.pathname !== "/sync" || !expected) return new Response(null, { status: 404 }); + if (request.method !== "POST") { + return new Response(null, { status: 405, headers: { Allow: "POST" } }); + } + const given = /^Bearer (.+)$/u.exec(request.headers.get("Authorization") ?? "")?.[1]; + if (given === undefined || !(await sameToken(given, expected))) { + return new Response(null, { status: 401 }); + } + const requested = url.searchParams.get("channel"); + if (requested !== null && requested !== "stable" && requested !== "beta") { + return new Response("channel must be stable or beta\n", { status: 400 }); + } + const channels: readonly MirrorChannel[] = requested === null ? ALL_CHANNELS : [requested]; + + const { readable, writable } = new TransformStream(); + const writer = writable.getWriter(); + const encoder = new TextEncoder(); + // Writes are serialized, and a caller that went away only stops the report, not the logs. + let pending = Promise.resolve(); + const send = (line: MirrorReportLine) => { + pending = pending + .then(() => writer.write(encoder.encode(`${JSON.stringify(line)}\n`))) + .catch(() => undefined); + }; + void (async () => { + send({ message: "Release mirror started", channels }); + const heartbeat = setInterval(() => send({ message: "Release mirror working" }), HEARTBEAT_MS); + try { + const failures = await runMirror(env.DESKTOP_DOWNLOADS, new Date(), channels, (line) => { + logLine(line); + send(line); + }); + send({ message: "Release mirror finished", failures }); + } finally { + clearInterval(heartbeat); + await pending; + await writer.close().catch(() => undefined); } + })(); + return new Response(readable, { + headers: { "Content-Type": "application/x-ndjson; charset=utf-8", "Cache-Control": "no-store" }, + }); +} + +export default { + async scheduled(controller, env) { + const failures = await runMirror( + env.DESKTOP_DOWNLOADS, + new Date(controller.scheduledTime), + ALL_CHANNELS, + logLine, + ); + if (failures > 0) throw new Error(`${failures} release mirror step(s) failed.`); }, -} satisfies ExportedHandler; + fetch: handleSync, +} satisfies ExportedHandler; diff --git a/workers/release-mirror/src/mirror.test.ts b/workers/release-mirror/src/mirror.test.ts index 3549e7c..687e0eb 100644 --- a/workers/release-mirror/src/mirror.test.ts +++ b/workers/release-mirror/src/mirror.test.ts @@ -320,14 +320,43 @@ describe("desktop release mirror", () => { expect(objects.size).toBe(0); }); - it("does nothing when GitHub's handoff is unavailable", async () => { + it("does nothing when GitHub keeps refusing the handoff", async () => { + vi.useFakeTimers(); const { bucket, writes } = memoryBucket(); + const refusals = vi.fn(() => new Response(null, { status: 429 })); + vi.stubGlobal("fetch", githubFetch(undefined, { [HANDOFF_URL]: refusals })); + + const run = expect(syncDesktopMirror(bucket, NOW)).rejects.toThrow(/429/); + await vi.runAllTimersAsync(); + await run; + expect(refusals).toHaveBeenCalledTimes(3); + expect(writes).toEqual([]); + vi.useRealTimers(); + }); + + it("tries again when GitHub briefly refuses the handoff", async () => { + vi.useFakeTimers(); + const { bucket, objects } = memoryBucket(); + let refused = false; vi.stubGlobal( "fetch", - githubFetch(undefined, { [HANDOFF_URL]: () => new Response(null, { status: 429 }) }), + githubFetch(undefined, { + [HANDOFF_URL]: () => { + if (refused) { + return Response.json(handoff, { + headers: { "Last-Modified": "Tue, 10 Nov 2026 16:14:08 GMT" }, + }); + } + refused = true; + return new Response(null, { status: 429, headers: { "Retry-After": "2" } }); + }, + }), ); - await expect(syncDesktopMirror(bucket, NOW)).rejects.toThrow(/429/); - expect(writes).toEqual([]); + const run = syncDesktopMirror(bucket, NOW); + await vi.runAllTimersAsync(); + expect((await run).promoted).toBe(true); + expect(pointerTag(objects)).toBe("v0.6.28"); + vi.useRealTimers(); }); }); diff --git a/workers/release-mirror/src/mirror.ts b/workers/release-mirror/src/mirror.ts index 751dded..1023422 100644 --- a/workers/release-mirror/src/mirror.ts +++ b/workers/release-mirror/src/mirror.ts @@ -18,6 +18,7 @@ import { desktopMirrorKey, GITHUB_RELEASE_HANDOFF_URL, } from "../../../src/lib/release-source"; +import { fetchFromGitHub } from "./github"; export const USER_AGENT = "ScientFactory-release-mirror"; const HANDOFF_TIMEOUT_MS = 30_000; @@ -36,10 +37,11 @@ export interface MirrorSyncResult { } async function fetchLatestHandoff(): Promise<{ raw: unknown; publishedAt: string | null }> { - const response = await fetch(GITHUB_RELEASE_HANDOFF_URL, { - headers: { Accept: "application/octet-stream, application/json", "User-Agent": USER_AGENT }, - signal: AbortSignal.timeout(HANDOFF_TIMEOUT_MS), - }); + const response = await fetchFromGitHub( + GITHUB_RELEASE_HANDOFF_URL, + { Accept: "application/octet-stream, application/json", "User-Agent": USER_AGENT }, + HANDOFF_TIMEOUT_MS, + ); if (!response.ok) { await response.body?.cancel(); throw new Error(`GitHub returned ${response.status} for the release handoff.`); @@ -130,10 +132,11 @@ export async function mirrorReleaseAsset( const existing = await bucket.head(key); if (existing !== null) return verifyExisting(existing); - const response = await fetch(githubReleaseDownloadUrl(tag, asset.name, repository), { - headers: { "User-Agent": USER_AGENT }, - signal: AbortSignal.timeout(INSTALLER_TIMEOUT_MS), - }); + const response = await fetchFromGitHub( + githubReleaseDownloadUrl(tag, asset.name, repository), + { "User-Agent": USER_AGENT }, + INSTALLER_TIMEOUT_MS, + ); if (!response.ok || response.body === null) { await response.body?.cancel(); throw new Error(`GitHub returned ${response.status} for ${asset.name}.`); diff --git a/workers/release-mirror/src/releaseFeed.ts b/workers/release-mirror/src/releaseFeed.ts index b80d067..cb03724 100644 --- a/workers/release-mirror/src/releaseFeed.ts +++ b/workers/release-mirror/src/releaseFeed.ts @@ -3,6 +3,8 @@ // GitHub provider reads for prerelease discovery, release names and release notes. // Layer: Cloudflare Worker (scheduled) +import { fetchFromGitHub } from "./github"; + const FEED_TIMEOUT_MS = 30_000; const MAX_FEED_BYTES = 4 * 1024 * 1024; /** Notes larger than this are left out of the feed copy rather than truncated mid-markup. */ @@ -61,10 +63,11 @@ export async function fetchReleaseFeed( repository: string, userAgent: string, ): Promise { - const response = await fetch(`https://github.com/${repository}/releases.atom`, { - headers: { Accept: "application/atom+xml, application/xml", "User-Agent": userAgent }, - signal: AbortSignal.timeout(FEED_TIMEOUT_MS), - }); + const response = await fetchFromGitHub( + `https://github.com/${repository}/releases.atom`, + { Accept: "application/atom+xml, application/xml", "User-Agent": userAgent }, + FEED_TIMEOUT_MS, + ); if (!response.ok) { await response.body?.cancel(); throw new Error(`GitHub returned ${response.status} for the ${repository} release feed.`); diff --git a/workers/release-mirror/src/updateFeed.ts b/workers/release-mirror/src/updateFeed.ts index 46e53db..0352699 100644 --- a/workers/release-mirror/src/updateFeed.ts +++ b/workers/release-mirror/src/updateFeed.ts @@ -17,6 +17,7 @@ import { DESKTOP_UPDATE_FEED_PREFIX, type DesktopUpdateFeedChannel, } from "../../../src/lib/release-source"; +import { fetchFromGitHub } from "./github"; import { mirrorReleaseAsset, USER_AGENT } from "./mirror"; import { fetchReleaseFeed, MAX_RELEASE_NOTES_BYTES, type ReleaseFeedEntry } from "./releaseFeed"; import { feedUpdateManifest, parseUpdateManifest } from "./updateManifest"; @@ -89,10 +90,11 @@ export function newestBetaEntry(entries: readonly ReleaseFeedEntry[]): ReleaseFe } async function fetchHandoff(url: string): Promise { - const response = await fetch(url, { - headers: { Accept: "application/octet-stream, application/json", "User-Agent": USER_AGENT }, - signal: AbortSignal.timeout(HANDOFF_TIMEOUT_MS), - }); + const response = await fetchFromGitHub( + url, + { Accept: "application/octet-stream, application/json", "User-Agent": USER_AGENT }, + HANDOFF_TIMEOUT_MS, + ); if (!response.ok) { await response.body?.cancel(); throw new Error(`GitHub returned ${response.status} for the release handoff.`); @@ -111,10 +113,11 @@ async function fetchManifest( asset: ReleaseHandoffAsset, ): Promise { if (asset.size > MAX_MANIFEST_BYTES) throw new Error(`${asset.name} is too large.`); - const response = await fetch(githubReleaseDownloadUrl(tag, asset.name, repository), { - headers: { "User-Agent": USER_AGENT }, - signal: AbortSignal.timeout(MANIFEST_TIMEOUT_MS), - }); + const response = await fetchFromGitHub( + githubReleaseDownloadUrl(tag, asset.name, repository), + { "User-Agent": USER_AGENT }, + MANIFEST_TIMEOUT_MS, + ); if (!response.ok) { await response.body?.cancel(); throw new Error(`GitHub returned ${response.status} for ${asset.name}.`); diff --git a/workers/release-mirror/wrangler.jsonc b/workers/release-mirror/wrangler.jsonc index 07dbfd2..d4e4930 100644 --- a/workers/release-mirror/wrangler.jsonc +++ b/workers/release-mirror/wrangler.jsonc @@ -3,10 +3,18 @@ "name": "scientfactory-release-mirror", "main": "src/index.ts", "compatibility_date": "2026-07-20", - "workers_dev": false, + // The workers.dev address serves only the release workflow's authenticated `POST /sync`. + "workers_dev": true, "preview_urls": false, "triggers": { - "crons": ["*/10 * * * *"], + // Three minutes past each ten: crons on the round ten minutes all reach GitHub at once from + // Cloudflare's shared addresses, and GitHub answered some of those runs with 429. + "crons": ["3-59/10 * * * *"], + }, + // Hashing a whole release (about 1.8 GB per channel) needs about 40 s of CPU; the default + // 30 s limit cut runs short, so a release took two or three runs to reach the feeds. + "limits": { + "cpu_ms": 300000, }, "r2_buckets": [ { From 057c0f757d45aa9661689d67f925ce989ec019e4 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Sat, 21 Nov 2026 22:03:45 +0200 Subject: [PATCH 2/4] fix(release-mirror): never move the Stable pointer back; sturdier retries From Codex's review of #80: - The pointer step refuses a handoff older than the pointer it read: a stale GitHub answer after the trigger promoted a newer release could otherwise swap the pointer back, since the pointer had not changed. - Retry-After is also read as an HTTP date (capped at 30 s), and cancelling a refused response's body can no longer stop the retry. - Tests: a whole Stable run reads no installer back; when the feed step could not run first, the next run reads them back once (documented). Co-Authored-By: Claude Opus 5.5 --- functions/_lib/release-mirror.ts | 8 ++- workers/release-mirror/src/github.test.ts | 36 ++++++++++- workers/release-mirror/src/github.ts | 20 ++++--- workers/release-mirror/src/index.ts | 4 +- workers/release-mirror/src/mirror.test.ts | 15 +++++ workers/release-mirror/src/mirror.ts | 36 +++++++++-- workers/release-mirror/src/updateFeed.test.ts | 59 +++++++++++++++++++ 7 files changed, 161 insertions(+), 17 deletions(-) diff --git a/functions/_lib/release-mirror.ts b/functions/_lib/release-mirror.ts index 33825d1..4fbdef4 100644 --- a/functions/_lib/release-mirror.ts +++ b/functions/_lib/release-mirror.ts @@ -59,7 +59,13 @@ export function parseMirrorPointer(value: unknown): MirroredRelease { /** The current Stable pointer, null when nothing has been mirrored yet. Throws when unreadable. */ export async function readMirroredRelease(bucket: R2Bucket): Promise { const object = await bucket.get(DESKTOP_MIRROR_STABLE_KEY); - if (object === null) return null; + return object === null ? null : parseMirrorPointerObject(object); +} + +/** The pointer in an object already read from the bucket, such as one a run snapshotted. */ +export async function parseMirrorPointerObject( + object: Pick, +): Promise { if (object.size > MAX_POINTER_BYTES) throw new Error("Mirror pointer is too large."); return parseMirrorPointer(JSON.parse(await object.text()) as unknown); } diff --git a/workers/release-mirror/src/github.test.ts b/workers/release-mirror/src/github.test.ts index 6d93484..b06f4a9 100644 --- a/workers/release-mirror/src/github.test.ts +++ b/workers/release-mirror/src/github.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { fetchFromGitHub } from "./github"; +import { fetchFromGitHub, retryDelay } from "./github"; afterEach(() => { vi.unstubAllGlobals(); @@ -66,4 +66,38 @@ describe("fetchFromGitHub", () => { expect(response.status).toBe(404); expect(fetchMock).toHaveBeenCalledTimes(1); }); + + it("reads Retry-After as an HTTP date too", () => { + const now = Date.parse("Sat, 21 Nov 2026 20:00:00 GMT"); + const at = (value: string) => + retryDelay(new Response(null, { status: 429, headers: { "Retry-After": value } }), 1, now); + + expect(at("Sat, 21 Nov 2026 20:00:25 GMT")).toBe(25_000); + expect(at("Sat, 21 Nov 2026 20:05:00 GMT")).toBe(30_000); + expect(at("Sat, 21 Nov 2026 19:59:00 GMT")).toBe(0); + expect(at("soon")).toBe(5_000); + }); + + it("retries even when the refused response's body has already failed", async () => { + const broken = new ReadableStream({ + start(controller) { + controller.error(new Error("connection reset")); + }, + }); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(broken, { status: 503 })) + .mockResolvedValueOnce(new Response("ok")); + vi.stubGlobal("fetch", fetchMock); + + const response = await fetchFromGitHub( + "https://github.com/x", + {}, + 1_000, + async () => undefined, + ); + + expect(response.status).toBe(200); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); }); diff --git a/workers/release-mirror/src/github.ts b/workers/release-mirror/src/github.ts index 4105d49..ae95e6f 100644 --- a/workers/release-mirror/src/github.ts +++ b/workers/release-mirror/src/github.ts @@ -7,13 +7,16 @@ const RETRYABLE = new Set([429, 502, 503, 504]); const ATTEMPTS = 3; const MAX_RETRY_AFTER_MS = 30_000; -/** How long GitHub asked us to wait, or a short growing pause when it did not say. */ -function retryDelay(response: Response, attempt: number): number { - const header = response.headers.get("Retry-After"); - const seconds = header === null ? Number.NaN : Number(header); - if (Number.isFinite(seconds) && seconds >= 0) { - return Math.min(seconds * 1000, MAX_RETRY_AFTER_MS); - } +/** + * How long GitHub asked us to wait (`Retry-After` in seconds or as an HTTP date), at most 30 + * seconds, or a short growing pause when it did not say. + */ +export function retryDelay(response: Response, attempt: number, now = Date.now()): number { + const header = response.headers.get("Retry-After")?.trim() ?? ""; + let delay = Number.NaN; + if (/^\d+$/u.test(header)) delay = Number(header) * 1000; + else if (header !== "") delay = Date.parse(header) - now; + if (Number.isFinite(delay)) return Math.min(Math.max(delay, 0), MAX_RETRY_AFTER_MS); return 5_000 * attempt; } @@ -31,7 +34,8 @@ export async function fetchFromGitHub( for (let attempt = 1; ; attempt += 1) { const response = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) }); if (!RETRYABLE.has(response.status) || attempt >= ATTEMPTS) return response; - await response.body?.cancel(); + // Only frees the connection; a body that already failed must not stop the retry. + await response.body?.cancel().catch(() => undefined); await sleep(retryDelay(response, attempt)); } } diff --git a/workers/release-mirror/src/index.ts b/workers/release-mirror/src/index.ts index 1bc5b8a..fced9a4 100644 --- a/workers/release-mirror/src/index.ts +++ b/workers/release-mirror/src/index.ts @@ -26,7 +26,9 @@ export type MirrorReportLine = Readonly> & { readonly me * The Stable update feed runs before the download-page mirror. The feed copies every Stable * installer with its update manifest's SHA-512 as well as the handoff's SHA-256, so the mirror * step then finds each one already verified. In the other order, the feed step had to read every - * installer back from R2 to hash it again, which pushed a release past the CPU limit. + * installer back from R2 to hash it again, which pushed a release past the CPU limit. That still + * happens once when the feed step fails and the mirror step copies the installers alone (GitHub's + * release feed not listing the release yet, say): the next feed run reads them back. */ export async function runMirror( bucket: R2Bucket, diff --git a/workers/release-mirror/src/mirror.test.ts b/workers/release-mirror/src/mirror.test.ts index 687e0eb..d2f5500 100644 --- a/workers/release-mirror/src/mirror.test.ts +++ b/workers/release-mirror/src/mirror.test.ts @@ -359,4 +359,19 @@ describe("desktop release mirror", () => { expect(pointerTag(objects)).toBe("v0.6.28"); vi.useRealTimers(); }); + + it("never moves the pointer back when GitHub still answers with an older release", async () => { + const { bucket, objects } = memoryBucket(); + let latest = handoffFor("0.6.29", "6789"); + vi.stubGlobal( + "fetch", + githubFetch(() => latest), + ); + await syncDesktopMirror(bucket, NOW); + const promoted = objects.get(STABLE)?.body; + + latest = handoff; + await expect(syncDesktopMirror(bucket, NOW)).rejects.toThrow(/back from 0\.6\.29 to 0\.6\.28/); + expect(objects.get(STABLE)?.body).toBe(promoted); + }); }); diff --git a/workers/release-mirror/src/mirror.ts b/workers/release-mirror/src/mirror.ts index 1023422..b8d43ed 100644 --- a/workers/release-mirror/src/mirror.ts +++ b/workers/release-mirror/src/mirror.ts @@ -4,7 +4,10 @@ // Layer: Cloudflare Worker (scheduled) import { readHandoff } from "../../../functions/_lib/latest-release"; -import { readMirroredRelease, writeMirrorPointer } from "../../../functions/_lib/release-mirror"; +import { + parseMirrorPointerObject, + writeMirrorPointer, +} from "../../../functions/_lib/release-mirror"; import { githubReleaseDownloadUrl, parseReleaseHandoff, @@ -186,18 +189,39 @@ export async function mirrorReleaseAsset( return true; } +/** Orders `x.y.z` versions numerically; anything else sorts before every such version. */ +export function compareStableVersions(a: string, b: string): number { + const parse = (version: string) => + /^\d+\.\d+\.\d+$/u.test(version) ? version.split(".").map(Number) : null; + const [x, y] = [parse(a), parse(b)]; + if (x === null || y === null) return x === y ? 0 : x === null ? -1 : 1; + for (let index = 0; index < 3; index += 1) { + if (x[index] !== y[index]) return x[index]! - y[index]!; + } + return 0; +} + /** * Brings the bucket up to GitHub's latest Stable release. Installers are copied (or confirmed) * first; the Stable pointer is written last, so it never names bytes that are not there. */ export async function syncDesktopMirror(bucket: R2Bucket, now: Date): Promise { - // The pointer version this run may replace, captured before anything this run learns from - // GitHub: a run whose GitHub answer predates another run's promotion must lose its swap. - const observed = await bucket.head(DESKTOP_MIRROR_STABLE_KEY); + // The pointer this run may replace, read before anything this run learns from GitHub: a run + // whose GitHub answer predates another run's promotion must lose its swap. + const observed = await bucket.get(DESKTOP_MIRROR_STABLE_KEY); + // An unreadable pointer is replaced below rather than blocking the mirror. + const current = + observed === null ? null : await parseMirrorPointerObject(observed).catch(() => null); const { raw, publishedAt } = await fetchLatestHandoff(); const handoff = parseReleaseHandoff(raw); - // An unreadable pointer is replaced below rather than blocking the mirror. - const current = await readMirroredRelease(bucket).catch(() => null); + // GitHub can still answer with the previous release for a while after a newer one was + // published and mirrored (by the release workflow's trigger, say). The swap alone cannot catch + // that: the pointer has not changed since this run read it. + if (current !== null && compareStableVersions(handoff.version, current.handoff.version) < 0) { + throw new Error( + `Refusing to move the Stable pointer back from ${current.handoff.version} to ${handoff.version}.`, + ); + } const copied: string[] = []; for (const name of requiredInstallerNames(handoff.version)) { diff --git a/workers/release-mirror/src/updateFeed.test.ts b/workers/release-mirror/src/updateFeed.test.ts index c5d5963..d12bdd4 100644 --- a/workers/release-mirror/src/updateFeed.test.ts +++ b/workers/release-mirror/src/updateFeed.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { desktopMirrorUrl, desktopUpdateFeedUrl } from "../../../src/lib/release-source"; +import { runMirror } from "./index"; import { syncUpdateFeed } from "./updateFeed"; // The platform's WebCrypto, which both Node and Workers provide, captured before tests stub it. @@ -587,4 +588,62 @@ describe("desktop update feed", () => { await expect(syncUpdateFeed(bucket, "beta")).rejects.toThrow(/another distribution/); expect(writes).toEqual([]); }); + + describe("a whole Stable run", () => { + const installers = (version: string) => + [ + `Scient-${version}-arm64.dmg`, + `Scient-${version}-x64.dmg`, + `Scient-${version}-x64.exe`, + `Scient-${version}-x86_64.AppImage`, + ].map((name) => `scient-desktop/v${version}/${name}`); + const readBack = (bucket: R2Bucket, version: string) => + (bucket.get as ReturnType).mock.calls + .map(([key]) => String(key)) + .filter((key) => installers(version).includes(key)); + + it("hashes each installer once: the feed copies them, the download pointer reuses them", async () => { + const stable = await release("0.6.30"); + const { bucket, objects } = memoryBucket(); + vi.stubGlobal( + "fetch", + github(() => [stable]), + ); + const report: Array> = []; + + const failures = await runMirror(bucket, new Date(), ["stable"], (line) => report.push(line)); + + expect(failures).toBe(0); + expect(report.map((line) => line.message)).toEqual([ + "Stable update feed synchronized", + "Desktop mirror synchronized", + ]); + expect(report[1]).toMatchObject({ tag: "v0.6.30", copied: [], promoted: true }); + expect(readBack(bucket, "0.6.30")).toEqual([]); + expect(objects.has("scient-desktop/stable.json")).toBe(true); + }); + + it("reads installers back once when the feed could not run first", async () => { + const stable = await release("0.6.30"); + const { bucket } = memoryBucket(); + let atomListed = false; + vi.stubGlobal( + "fetch", + github(() => [stable], { + "https://github.com/ScientFactory/scient-desktop/releases.atom": () => + new Response(atom("ScientFactory/scient-desktop", atomListed ? [stable] : [])), + }), + ); + + // GitHub's feed does not list the release yet: the pointer step copies the installers + // with the handoff's SHA-256 alone. + expect(await runMirror(bucket, new Date(), ["stable"], () => undefined)).toBe(1); + expect(readBack(bucket, "0.6.30")).toEqual([]); + + // The next run's feed step must check each of them against the manifest's SHA-512. + atomListed = true; + expect(await runMirror(bucket, new Date(), ["stable"], () => undefined)).toBe(0); + expect(readBack(bucket, "0.6.30").toSorted()).toEqual(installers("0.6.30").toSorted()); + }); + }); }); From 3d0743a8cedea1bf391be9c7db239ce71f1ca6ba Mon Sep 17 00:00:00 2001 From: Yaacov Date: Sat, 21 Nov 2026 22:03:56 +0200 Subject: [PATCH 3/4] docs(release-mirror): when installers are read back, and the pointer guard Co-Authored-By: Claude Opus 5.5 --- docs/architecture/desktop-downloads.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/architecture/desktop-downloads.md b/docs/architecture/desktop-downloads.md index ae22eae..9c86040 100644 --- a/docs/architecture/desktop-downloads.md +++ b/docs/architecture/desktop-downloads.md @@ -34,7 +34,10 @@ minutes past every ten (off the round minutes, when many Workers reach GitHub at soon as a release is published: the desktop release workflow calls its authenticated `POST /sync?channel=stable|beta` (see [Run on publish](#run-on-publish)). A run publishes the Stable update feed first, then the Stable pointer below, then the Beta feed, so the pointer step -finds every Stable installer already verified against both digests instead of hashing it again. +normally finds every Stable installer already verified against both digests instead of hashing +it again. When the feed step fails and the pointer step copies the installers alone, the next +feed run reads them back once to check their SHA-512. The pointer step also refuses a GitHub +handoff older than the pointer it read, so a stale answer cannot move the pointer back. GitHub answers of 429 or a gateway error are retried up to three attempts, honoring `Retry-After` up to 30 seconds. Each run: From 5974f8950256489fc345e1ef1df031e2acff4584 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Sat, 21 Nov 2026 22:09:47 +0200 Subject: [PATCH 4/4] fix(release-mirror): a pointer that cannot be read stops the run From Codex's recheck of #80: a failed read of the pointer's body was treated like a malformed pointer, so the run kept its ETag and a stale GitHub handoff could still replace a newer pointer. Now only bytes read and found malformed may be replaced; a failed read rejects the run. Also qualifies when installers are read back again (until a feed run publishes the release). Co-Authored-By: Claude Opus 5.5 --- docs/architecture/desktop-downloads.md | 7 ++--- functions/_lib/release-mirror.ts | 20 ++++++++++++-- workers/release-mirror/src/index.ts | 3 ++- workers/release-mirror/src/mirror.test.ts | 32 +++++++++++++++++++++++ workers/release-mirror/src/mirror.ts | 8 +++--- 5 files changed, 60 insertions(+), 10 deletions(-) diff --git a/docs/architecture/desktop-downloads.md b/docs/architecture/desktop-downloads.md index 9c86040..404cc37 100644 --- a/docs/architecture/desktop-downloads.md +++ b/docs/architecture/desktop-downloads.md @@ -35,9 +35,10 @@ soon as a release is published: the desktop release workflow calls its authentic `POST /sync?channel=stable|beta` (see [Run on publish](#run-on-publish)). A run publishes the Stable update feed first, then the Stable pointer below, then the Beta feed, so the pointer step normally finds every Stable installer already verified against both digests instead of hashing -it again. When the feed step fails and the pointer step copies the installers alone, the next -feed run reads them back once to check their SHA-512. The pointer step also refuses a GitHub -handoff older than the pointer it read, so a stale answer cannot move the pointer back. +it again. When the feed step fails and the pointer step copies the installers alone, later +feed runs read them back to check their SHA-512 until one publishes the release (the copies keep +no SHA-512 receipt). The pointer step also refuses a GitHub handoff older than the pointer it +read, and stops if it cannot read the pointer, so a stale answer cannot move the pointer back. GitHub answers of 429 or a gateway error are retried up to three attempts, honoring `Retry-After` up to 30 seconds. Each run: diff --git a/functions/_lib/release-mirror.ts b/functions/_lib/release-mirror.ts index 4fbdef4..9478ab5 100644 --- a/functions/_lib/release-mirror.ts +++ b/functions/_lib/release-mirror.ts @@ -62,14 +62,30 @@ export async function readMirroredRelease(bucket: R2Bucket): Promise, ): Promise { if (object.size > MAX_POINTER_BYTES) throw new Error("Mirror pointer is too large."); return parseMirrorPointer(JSON.parse(await object.text()) as unknown); } +/** + * The pointer in an object a run has just read from the bucket, or null when its bytes are + * malformed (so the run may replace them). A failed read rejects instead: the run must not + * replace a pointer it could not see. + */ +export async function readMirrorPointerSnapshot( + object: Pick, +): Promise { + if (object.size > MAX_POINTER_BYTES) return null; + const text = await object.text(); + try { + return parseMirrorPointer(JSON.parse(text) as unknown); + } catch { + return null; + } +} + /** * Writes the pointer only if it is still the version this run observed (`observedEtag`, or * absent when null). Returns false when another run wrote it first, so a slow run that saw an diff --git a/workers/release-mirror/src/index.ts b/workers/release-mirror/src/index.ts index fced9a4..70c25a3 100644 --- a/workers/release-mirror/src/index.ts +++ b/workers/release-mirror/src/index.ts @@ -28,7 +28,8 @@ export type MirrorReportLine = Readonly> & { readonly me * step then finds each one already verified. In the other order, the feed step had to read every * installer back from R2 to hash it again, which pushed a release past the CPU limit. That still * happens once when the feed step fails and the mirror step copies the installers alone (GitHub's - * release feed not listing the release yet, say): the next feed run reads them back. + * release feed not listing the release yet, say): later feed runs read them back until one + * publishes the release. */ export async function runMirror( bucket: R2Bucket, diff --git a/workers/release-mirror/src/mirror.test.ts b/workers/release-mirror/src/mirror.test.ts index d2f5500..445c91e 100644 --- a/workers/release-mirror/src/mirror.test.ts +++ b/workers/release-mirror/src/mirror.test.ts @@ -374,4 +374,36 @@ describe("desktop release mirror", () => { await expect(syncDesktopMirror(bucket, NOW)).rejects.toThrow(/back from 0\.6\.29 to 0\.6\.28/); expect(objects.get(STABLE)?.body).toBe(promoted); }); + + it("stops, rather than replace the pointer, when its bytes cannot be read", async () => { + const { bucket, objects } = memoryBucket(); + let latest = handoffFor("0.6.29", "6789"); + vi.stubGlobal( + "fetch", + githubFetch(() => latest), + ); + await syncDesktopMirror(bucket, NOW); + const promoted = objects.get(STABLE)?.body; + type Get = (key: string) => Promise<{ text: () => Promise } | null>; + const get = bucket.get as unknown as ReturnType>; + const read = get.getMockImplementation()!; + get.mockImplementation(async (key: string) => { + const object = await read(key); + return key === STABLE && object + ? { ...object, text: async () => Promise.reject(new Error("network lost")) } + : object; + }); + + latest = handoff; + await expect(syncDesktopMirror(bucket, NOW)).rejects.toThrow(/network lost/); + expect(objects.get(STABLE)?.body).toBe(promoted); + }); + + it("replaces a pointer whose bytes are malformed", async () => { + const { bucket, objects } = memoryBucket({ [STABLE]: "not json" }); + vi.stubGlobal("fetch", githubFetch()); + + expect((await syncDesktopMirror(bucket, NOW)).promoted).toBe(true); + expect(pointerTag(objects)).toBe("v0.6.28"); + }); }); diff --git a/workers/release-mirror/src/mirror.ts b/workers/release-mirror/src/mirror.ts index b8d43ed..f9de7aa 100644 --- a/workers/release-mirror/src/mirror.ts +++ b/workers/release-mirror/src/mirror.ts @@ -5,7 +5,7 @@ import { readHandoff } from "../../../functions/_lib/latest-release"; import { - parseMirrorPointerObject, + readMirrorPointerSnapshot, writeMirrorPointer, } from "../../../functions/_lib/release-mirror"; import { @@ -209,9 +209,9 @@ export async function syncDesktopMirror(bucket: R2Bucket, now: Date): Promise null); + // A malformed pointer is replaced below rather than blocking the mirror; one whose bytes could + // not be read stops the run, since its version is unknown. + const current = observed === null ? null : await readMirrorPointerSnapshot(observed); const { raw, publishedAt } = await fetchLatestHandoff(); const handoff = parseReleaseHandoff(raw); // GitHub can still answer with the previous release for a while after a newer one was