From ea92ceab061ba334dfbae0a6805f3540b7dd4a8a Mon Sep 17 00:00:00 2001 From: AlexDeveloperUwU <79850072+AlexDeveloperUwU@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:06:54 +0100 Subject: [PATCH 1/5] fix(captcha): Cancel timed-out worker races Propagate an AbortSignal from the CAPTCHA pool through the solver to worker dispatch, and abort/terminate parallel worker solves when the race deadline expires. Covers warm workers and worker-per-solve workers, with a hung-worker regression test. --- src/proxy/captcha-pool.ts | 38 ++++++++++++++--------- src/proxy/captcha-solver.ts | 4 +-- src/proxy/captcha-worker-dispatch.test.ts | 23 ++++++++++++++ src/proxy/captcha-worker-dispatch.ts | 35 +++++++++++++++++---- 4 files changed, 78 insertions(+), 22 deletions(-) diff --git a/src/proxy/captcha-pool.ts b/src/proxy/captcha-pool.ts index afc67c7..2c9f016 100644 --- a/src/proxy/captcha-pool.ts +++ b/src/proxy/captcha-pool.ts @@ -259,15 +259,21 @@ export class CaptchaTokenPool { // grind through its retry budget (~30-45s) — cap the client-facing wait // and let the background waves finish the job instead. const raceDeadlineMs = Number(process.env.CAPTCHA_SOLVE_RACE_DEADLINE_MS || 35_000); - param = await Promise.race([ - this.solveRaced(cfg), - new Promise((_, rej) => - setTimeout( - () => rej(new Error(`captcha take deadline (${raceDeadlineMs}ms)`)), - Math.max(1_000, raceDeadlineMs), - ), - ), - ]); + const solveAbort = new AbortController(); + let deadlineTimer: ReturnType | undefined; + try { + param = await Promise.race([ + this.solveRaced(cfg, solveAbort.signal), + new Promise((_, rej) => { + deadlineTimer = setTimeout(() => { + solveAbort.abort(new Error(`captcha take deadline (${raceDeadlineMs}ms)`)); + rej(new Error(`captcha take deadline (${raceDeadlineMs}ms)`)); + }, Math.max(1_000, raceDeadlineMs)); + }), + ]); + } finally { + if (deadlineTimer !== undefined) clearTimeout(deadlineTimer); + } } catch (err) { // Mints fail in clusters (pe-stall storms, F008 velocity). Background // refill waves keep retrying — give them a short window to land a @@ -578,7 +584,7 @@ export class CaptchaTokenPool { } } - private async solveFresh(cfg: CaptchaConfig): Promise { + private async solveFresh(cfg: CaptchaConfig, signal?: AbortSignal): Promise { this.activeSolves += 1; this.lastSolveAt = Date.now(); try { @@ -586,7 +592,10 @@ export class CaptchaTokenPool { let sawIpBlock = false; for (let attempt = 1; attempt <= this.opts.solveRetries; attempt += 1) { try { - const param = await runCaptchaSolve(cfg.sceneId, cfg.region, cfg.prefix); + if (signal?.aborted) { + throw signal.reason instanceof Error ? signal.reason : new Error("captcha solve aborted"); + } + const param = await runCaptchaSolve(cfg.sceneId, cfg.region, cfg.prefix, signal); if (!param) { lastErr = "solver returned empty"; continue; @@ -599,6 +608,7 @@ export class CaptchaTokenPool { return param; } catch (err) { lastErr = err instanceof Error ? err.message : String(err); + if (signal?.aborted) throw err; if (isCaptchaDuplicateError(lastErr)) { lastErr = `duplicate certifyId (F008)`; } else if (isCaptchaIpBlockError(lastErr)) { @@ -633,16 +643,16 @@ export class CaptchaTokenPool { * sequential retry chain. Extra successes are banked into the pool instead * of being wasted; the race only rejects when every racer fails. */ - private solveRaced(cfg: CaptchaConfig): Promise { + private solveRaced(cfg: CaptchaConfig, signal?: AbortSignal): Promise { const racers = Math.max(1, Math.min(this.opts.emptyTakeRace, this.opts.solveConcurrency)); - if (racers <= 1) return this.solveFresh(cfg); + if (racers <= 1) return this.solveFresh(cfg, signal); return new Promise((resolve, reject) => { let pending = racers; let served = false; let lastErr: unknown = null; for (let i = 0; i < racers; i += 1) { - this.solveFresh(cfg).then( + this.solveFresh(cfg, signal).then( (param) => { if (served) { // Lost the race but minted a valid token — bank it. diff --git a/src/proxy/captcha-solver.ts b/src/proxy/captcha-solver.ts index 13f8786..3290fa1 100644 --- a/src/proxy/captcha-solver.ts +++ b/src/proxy/captcha-solver.ts @@ -10,11 +10,11 @@ import { solveViaWorkerOrInProcess, stopCaptchaWorkers } from "./captcha-worker- const BACKEND = process.env.ZCODE_CAPTCHA_BACKEND?.trim().toLowerCase() || "happy"; -export async function runCaptchaSolve(scene: string, region: string, prefix: string): Promise { +export async function runCaptchaSolve(scene: string, region: string, prefix: string, signal?: AbortSignal): Promise { if (BACKEND !== "happy") { throw new Error(`captcha backend "${BACKEND}" is not available; use ZCODE_CAPTCHA_BACKEND=happy`); } - return solveViaWorkerOrInProcess({ scene, region, prefix }); + return solveViaWorkerOrInProcess({ scene, region, prefix, signal }); } /** Worker-per-solve needs no concurrency plumbing — kept for the pool API. */ diff --git a/src/proxy/captcha-worker-dispatch.test.ts b/src/proxy/captcha-worker-dispatch.test.ts index 76893a0..2583168 100644 --- a/src/proxy/captcha-worker-dispatch.test.ts +++ b/src/proxy/captcha-worker-dispatch.test.ts @@ -134,6 +134,7 @@ describe("warm worker pool", () => { const warmDir = fs.mkdtempSync(path.join(os.tmpdir(), "cap-warm-")); const markerPath = path.join(warmDir, "spawns.txt"); const warmFixture = path.join(warmDir, "warm-worker.mjs"); + const hangingFixture = path.join(warmDir, "hanging-worker.mjs"); fs.writeFileSync( warmFixture, [ @@ -147,6 +148,11 @@ describe("warm worker pool", () => { ].join("\n"), "utf8", ); + fs.writeFileSync( + hangingFixture, + ['import { parentPort } from "node:worker_threads";', "if (!parentPort) throw new Error('fixture requires parent port');"].join("\n"), + "utf8", + ); const spawnCount = (): number => fs.existsSync(markerPath) ? fs.readFileSync(markerPath, "utf8").split("\n").filter((l) => l === "spawn").length @@ -182,6 +188,23 @@ describe("warm worker pool", () => { // The pool was emptied -> the next solve had to spawn a fresh worker. expect(spawnCount()).toBe(2); }); + + test("aborting a solve terminates a hung worker immediately", async () => { + process.env.ZCODE_CAPTCHA_WORKER = hangingFixture; + delete process.env.ZCODE_CAPTCHA_WARM_WORKERS; + mock.module("./captcha-worker-asset.js", () => ({ default: null })); + __resetCaptchaWorkerDispatchForTest(); + + const controller = new AbortController(); + const pending = solveViaWorkerOrInProcess({ + scene: "abort", + region: "r", + prefix: "p", + signal: controller.signal, + }); + controller.abort(new Error("request deadline")); + await expect(pending).rejects.toThrow("request deadline"); + }); }); // File-level cleanup: runs after every describe above, so shared fixtures diff --git a/src/proxy/captcha-worker-dispatch.ts b/src/proxy/captcha-worker-dispatch.ts index cc784b5..b9e8c81 100644 --- a/src/proxy/captcha-worker-dispatch.ts +++ b/src/proxy/captcha-worker-dispatch.ts @@ -181,8 +181,16 @@ export async function solveViaWorkerOrInProcess(req: { scene: string; region: string; prefix: string; + signal?: AbortSignal; }): Promise { + const throwIfAborted = () => { + if (req.signal?.aborted) { + throw req.signal.reason instanceof Error ? req.signal.reason : new Error("captcha solve aborted"); + } + }; + throwIfAborted(); const entryPath = await getWorkerEntryPath(); + throwIfAborted(); if (entryPath === null) { if (strictWorkerRequired()) { const looked = resolveWorkerEntryCandidates({ @@ -209,7 +217,7 @@ export async function solveViaWorkerOrInProcess(req: { */ async function solveInWorkerDegrading( entryPath: string, - req: { scene: string; region: string; prefix: string }, + req: { scene: string; region: string; prefix: string; signal?: AbortSignal }, ): Promise { try { return await solveInWorker(entryPath, req); @@ -368,9 +376,17 @@ function acquireWarm(entryPath: string): { handle?: WarmHandle; spawnError?: Err return spawnWarm(entryPath); } -function dispatchToWarm(h: WarmHandle, req: { scene: string; region: string; prefix: string }): Promise { +function dispatchToWarm( + h: WarmHandle, + req: { scene: string; region: string; prefix: string; signal?: AbortSignal }, +): Promise { return new Promise((resolve, reject) => { const id = ++nextSolveId; + const onAbort = () => { + killWarm(h, req.signal?.reason instanceof Error ? req.signal.reason.message : "captcha solve aborted"); + dropFromPool(h); + reject(req.signal?.reason instanceof Error ? req.signal.reason : new Error("captcha solve aborted")); + }; const timer = setTimeout(() => { killWarm(h, `captcha worker timeout (${SOLVE_WORKER_TIMEOUT_MS}ms)`); dropFromPool(h); @@ -378,11 +394,12 @@ function dispatchToWarm(h: WarmHandle, req: { scene: string; region: string; pre }, SOLVE_WORKER_TIMEOUT_MS); h.busy = true; h.pending = { id, timer, resolve, reject }; + req.signal?.addEventListener("abort", onAbort, { once: true }); h.worker.postMessage({ id, scene: req.scene, region: req.region, prefix: req.prefix }); }); } -async function warmSolve(entryPath: string, req: { scene: string; region: string; prefix: string }): Promise { +async function warmSolve(entryPath: string, req: { scene: string; region: string; prefix: string; signal?: AbortSignal }): Promise { const acq = acquireWarm(entryPath); if (acq.spawnError) { if (strictWorkerRequired()) { @@ -448,22 +465,28 @@ export function stopCaptchaWorkers(): void { */ function solveInWorker( entryPath: string, - req: { scene: string; region: string; prefix: string }, + req: { scene: string; region: string; prefix: string; signal?: AbortSignal }, ): Promise { const id = ++nextSolveId; return new Promise((resolve, reject) => { let settled = false; let worker: Worker | null = null; + let timer!: ReturnType; const settle = (fn: () => void) => { if (settled) return; settled = true; clearTimeout(timer); + req.signal?.removeEventListener("abort", onAbort); try { worker?.terminate(); } catch {} fn(); }; - const timer = setTimeout(() => { + const onAbort = () => { + settle(() => reject(req.signal?.reason instanceof Error ? req.signal.reason : new Error("captcha solve aborted"))); + }; + timer = setTimeout(() => { settle(() => reject(new Error(`captcha worker timeout (${SOLVE_WORKER_TIMEOUT_MS}ms)`))); }, SOLVE_WORKER_TIMEOUT_MS); + req.signal?.addEventListener("abort", onAbort, { once: true }); try { worker = new Worker(entryPath); @@ -473,7 +496,7 @@ function solveInWorker( settle(() => reject(new WorkerUnavailableError(msg))); return; } - const msg: SolveRequest = { id, ...req }; + const msg: SolveRequest = { id, scene: req.scene, region: req.region, prefix: req.prefix }; worker.on("message", (m: SolveResponse) => { if (!m || m.id !== id) return; if (m.ok) settle(() => resolve(m.param)); From eaeb5c36d3ee224ea09717f810b7d907b42524bb Mon Sep 17 00:00:00 2001 From: AlexDeveloperUwU <79850072+AlexDeveloperUwU@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:39:31 +0100 Subject: [PATCH 2/5] fix(captcha): Guard stale abort listener and pin mid-flight abort in tests Skip the abort handler in dispatchToWarm when the dispatch already ended, so a late abort cannot kill an unrelated take that re-acquired the handle. Reshape the regression test to abort after the worker is dispatched, add a warm-pool variant, and note that in-process solves are not cancellable. --- src/proxy/captcha-worker-dispatch.test.ts | 39 +++++++++++++++++++++-- src/proxy/captcha-worker-dispatch.ts | 6 ++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/proxy/captcha-worker-dispatch.test.ts b/src/proxy/captcha-worker-dispatch.test.ts index 2583168..80fcf5c 100644 --- a/src/proxy/captcha-worker-dispatch.test.ts +++ b/src/proxy/captcha-worker-dispatch.test.ts @@ -150,7 +150,12 @@ describe("warm worker pool", () => { ); fs.writeFileSync( hangingFixture, - ['import { parentPort } from "node:worker_threads";', "if (!parentPort) throw new Error('fixture requires parent port');"].join("\n"), + [ + 'import { parentPort } from "node:worker_threads";', + 'import { appendFileSync } from "node:fs";', + `appendFileSync(${JSON.stringify(markerPath)}, "spawn\\n");`, + "if (!parentPort) throw new Error('fixture requires parent port');", + ].join("\n"), "utf8", ); const spawnCount = (): number => @@ -189,7 +194,9 @@ describe("warm worker pool", () => { expect(spawnCount()).toBe(2); }); - test("aborting a solve terminates a hung worker immediately", async () => { + // Not warm-pool behaviour: this covers the worker-per-solve path and lives + // here only to share the hanging fixture. + test("aborting a worker-per-solve solve mid-flight terminates the hung worker", async () => { process.env.ZCODE_CAPTCHA_WORKER = hangingFixture; delete process.env.ZCODE_CAPTCHA_WARM_WORKERS; mock.module("./captcha-worker-asset.js", () => ({ default: null })); @@ -202,8 +209,36 @@ describe("warm worker pool", () => { prefix: "p", signal: controller.signal, }); + // Let the worker spawn and the job dispatch so the abort is mid-flight. + await new Promise((r) => setTimeout(r, 100)); + const t0 = Date.now(); controller.abort(new Error("request deadline")); await expect(pending).rejects.toThrow("request deadline"); + // Must not ride the 30s worker timeout. + expect(Date.now() - t0).toBeLessThan(5_000); + }); + + test("aborting a warm solve mid-flight drops the handle; the next solve respawns", async () => { + process.env.ZCODE_CAPTCHA_WORKER = hangingFixture; + process.env.ZCODE_CAPTCHA_WARM_WORKERS = "1"; + mock.module("./captcha-worker-asset.js", () => ({ default: null })); + __resetCaptchaWorkerDispatchForTest(); + const baseline = spawnCount(); + + const abortAfterDispatch = async (scene: string): Promise => { + const controller = new AbortController(); + const pending = solveViaWorkerOrInProcess({ scene, region: "r", prefix: "p", signal: controller.signal }); + await new Promise((r) => setTimeout(r, 100)); + const t0 = Date.now(); + controller.abort(new Error("request deadline")); + await expect(pending).rejects.toThrow("request deadline"); + expect(Date.now() - t0).toBeLessThan(5_000); + }; + + await abortAfterDispatch("warm-abort-1"); + await abortAfterDispatch("warm-abort-2"); + // The killed handle was dropped, so the second solve had to spawn anew. + expect(spawnCount() - baseline).toBe(2); }); }); diff --git a/src/proxy/captcha-worker-dispatch.ts b/src/proxy/captcha-worker-dispatch.ts index b9e8c81..7835a25 100644 --- a/src/proxy/captcha-worker-dispatch.ts +++ b/src/proxy/captcha-worker-dispatch.ts @@ -202,6 +202,8 @@ export async function solveViaWorkerOrInProcess(req: { (looked.length ? `; looked for: ${looked.join(", ")}` : ""), ); } + // Cancellation covers the worker faces only: a main-thread in-process + // solve cannot be terminated and is bounded by its own timeouts. return (await happy()).solveTraceless(req); } if (warmWorkerTarget() > 0) { @@ -383,6 +385,10 @@ function dispatchToWarm( return new Promise((resolve, reject) => { const id = ++nextSolveId; const onAbort = () => { + // Stale-listener guard: if this dispatch already ended (ok:false reply, + // timeout), the handle may have been re-acquired by ANOTHER take, and a + // late abort from the original signal must not kill that solve. + if (h.pending?.id !== id) return; killWarm(h, req.signal?.reason instanceof Error ? req.signal.reason.message : "captcha solve aborted"); dropFromPool(h); reject(req.signal?.reason instanceof Error ? req.signal.reason : new Error("captcha solve aborted")); From bff1dfa5f694a7dd815a617ea7d859b2365911b5 Mon Sep 17 00:00:00 2001 From: AlexDeveloperUwU <79850072+AlexDeveloperUwU@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:40:36 +0100 Subject: [PATCH 3/5] refactor(captcha): Remove added code comments --- src/proxy/captcha-worker-dispatch.test.ts | 5 ----- src/proxy/captcha-worker-dispatch.ts | 5 ----- 2 files changed, 10 deletions(-) diff --git a/src/proxy/captcha-worker-dispatch.test.ts b/src/proxy/captcha-worker-dispatch.test.ts index 80fcf5c..24c702c 100644 --- a/src/proxy/captcha-worker-dispatch.test.ts +++ b/src/proxy/captcha-worker-dispatch.test.ts @@ -194,8 +194,6 @@ describe("warm worker pool", () => { expect(spawnCount()).toBe(2); }); - // Not warm-pool behaviour: this covers the worker-per-solve path and lives - // here only to share the hanging fixture. test("aborting a worker-per-solve solve mid-flight terminates the hung worker", async () => { process.env.ZCODE_CAPTCHA_WORKER = hangingFixture; delete process.env.ZCODE_CAPTCHA_WARM_WORKERS; @@ -209,12 +207,10 @@ describe("warm worker pool", () => { prefix: "p", signal: controller.signal, }); - // Let the worker spawn and the job dispatch so the abort is mid-flight. await new Promise((r) => setTimeout(r, 100)); const t0 = Date.now(); controller.abort(new Error("request deadline")); await expect(pending).rejects.toThrow("request deadline"); - // Must not ride the 30s worker timeout. expect(Date.now() - t0).toBeLessThan(5_000); }); @@ -237,7 +233,6 @@ describe("warm worker pool", () => { await abortAfterDispatch("warm-abort-1"); await abortAfterDispatch("warm-abort-2"); - // The killed handle was dropped, so the second solve had to spawn anew. expect(spawnCount() - baseline).toBe(2); }); }); diff --git a/src/proxy/captcha-worker-dispatch.ts b/src/proxy/captcha-worker-dispatch.ts index 7835a25..953a3c3 100644 --- a/src/proxy/captcha-worker-dispatch.ts +++ b/src/proxy/captcha-worker-dispatch.ts @@ -202,8 +202,6 @@ export async function solveViaWorkerOrInProcess(req: { (looked.length ? `; looked for: ${looked.join(", ")}` : ""), ); } - // Cancellation covers the worker faces only: a main-thread in-process - // solve cannot be terminated and is bounded by its own timeouts. return (await happy()).solveTraceless(req); } if (warmWorkerTarget() > 0) { @@ -385,9 +383,6 @@ function dispatchToWarm( return new Promise((resolve, reject) => { const id = ++nextSolveId; const onAbort = () => { - // Stale-listener guard: if this dispatch already ended (ok:false reply, - // timeout), the handle may have been re-acquired by ANOTHER take, and a - // late abort from the original signal must not kill that solve. if (h.pending?.id !== id) return; killWarm(h, req.signal?.reason instanceof Error ? req.signal.reason.message : "captcha solve aborted"); dropFromPool(h); From 1ecb6b300e69a448b2b099dc2d5a0d9ae42d4975 Mon Sep 17 00:00:00 2001 From: AlexDeveloperUwU <79850072+AlexDeveloperUwU@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:44:18 +0100 Subject: [PATCH 4/5] fix(captcha): Keep hanging test worker alive with a message listener --- src/proxy/captcha-worker-dispatch.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/proxy/captcha-worker-dispatch.test.ts b/src/proxy/captcha-worker-dispatch.test.ts index 24c702c..2a02eba 100644 --- a/src/proxy/captcha-worker-dispatch.test.ts +++ b/src/proxy/captcha-worker-dispatch.test.ts @@ -155,6 +155,7 @@ describe("warm worker pool", () => { 'import { appendFileSync } from "node:fs";', `appendFileSync(${JSON.stringify(markerPath)}, "spawn\\n");`, "if (!parentPort) throw new Error('fixture requires parent port');", + "parentPort.on('message', () => {});", ].join("\n"), "utf8", ); From 4079a4561626b667ed991bd14fbc98d4f58d734f Mon Sep 17 00:00:00 2001 From: AlexDeveloperUwU <79850072+AlexDeveloperUwU@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:11:27 +0100 Subject: [PATCH 5/5] test(captcha): Cover late abort from a finished warm solve --- src/proxy/captcha-worker-dispatch.test.ts | 46 +++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/proxy/captcha-worker-dispatch.test.ts b/src/proxy/captcha-worker-dispatch.test.ts index 2a02eba..741353a 100644 --- a/src/proxy/captcha-worker-dispatch.test.ts +++ b/src/proxy/captcha-worker-dispatch.test.ts @@ -159,6 +159,21 @@ describe("warm worker pool", () => { ].join("\n"), "utf8", ); + const failThenHangFixture = path.join(warmDir, "fail-then-hang-worker.mjs"); + fs.writeFileSync( + failThenHangFixture, + [ + 'import { parentPort } from "node:worker_threads";', + 'import { appendFileSync } from "node:fs";', + `appendFileSync(${JSON.stringify(markerPath)}, "spawn\\n");`, + "if (!parentPort) throw new Error('fixture requires parent port');", + "let n = 0;", + "parentPort.on('message', (m) => {", + " if (n++ === 0) parentPort.postMessage({ id: m.id, ok: false, error: 'boom' });", + "});", + ].join("\n"), + "utf8", + ); const spawnCount = (): number => fs.existsSync(markerPath) ? fs.readFileSync(markerPath, "utf8").split("\n").filter((l) => l === "spawn").length @@ -236,6 +251,37 @@ describe("warm worker pool", () => { await abortAfterDispatch("warm-abort-2"); expect(spawnCount() - baseline).toBe(2); }); + + test("a late abort from a finished warm solve does not kill the next take on the same handle", async () => { + process.env.ZCODE_CAPTCHA_WORKER = failThenHangFixture; + process.env.ZCODE_CAPTCHA_WARM_WORKERS = "1"; + mock.module("./captcha-worker-asset.js", () => ({ default: null })); + __resetCaptchaWorkerDispatchForTest(); + const baseline = spawnCount(); + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + + const controllerA = new AbortController(); + await expect( + solveViaWorkerOrInProcess({ scene: "a", region: "r", prefix: "p", signal: controllerA.signal }), + ).rejects.toThrow("boom"); + + const controllerB = new AbortController(); + let settledB = false; + const pendingB = solveViaWorkerOrInProcess({ scene: "b", region: "r", prefix: "p", signal: controllerB.signal }); + pendingB.then( + () => { settledB = true; }, + () => { settledB = true; }, + ); + await sleep(100); + + controllerA.abort(new Error("late abort A")); + await sleep(100); + expect(settledB).toBe(false); + + controllerB.abort(new Error("B deadline")); + await expect(pendingB).rejects.toThrow("B deadline"); + expect(spawnCount() - baseline).toBe(1); + }); }); // File-level cleanup: runs after every describe above, so shared fixtures