(
+ "scient/telemetry/ScientRecordedAnalyticsIngest",
+ { defaultValue: () => neverSent },
+) {}
+
export interface ScientAnalytics {
readonly analytics: AnalyticsService["Service"];
readonly available: boolean;
+ readonly recorded: RecordedAnalyticsIngest;
}
function parseBuildChannel(
@@ -104,7 +125,11 @@ export const disabledAnalyticsService: AnalyticsService["Service"] = {
flush: Effect.void,
};
-const disabled: ScientAnalytics = { analytics: disabledAnalyticsService, available: false };
+const disabled: ScientAnalytics = {
+ analytics: disabledAnalyticsService,
+ available: false,
+ recorded: neverSent,
+};
export const make = Effect.gen(function* () {
const analyticsConfig = yield* AnalyticsEnvConfig;
@@ -196,17 +221,27 @@ export const make = Effect.gen(function* () {
Effect.asVoid,
);
+ const recorded: RecordedAnalyticsIngest = {
+ ingest: (events) =>
+ Effect.tryPromise({
+ try: () => runtime.ingest(events),
+ catch: () => "analytics-ingest-failed" as const,
+ }).pipe(Effect.orElseSucceed(() => false)),
+ };
+
return {
analytics: AnalyticsService.of({ record, recordOwners, flush }),
available: true,
+ recorded,
} satisfies ScientAnalytics;
});
/** Provides AnalyticsService and its availability in place of T3's PostHog layer. */
export const layer = Layer.effectContext(
- Effect.map(make, ({ analytics, available }) =>
+ Effect.map(make, ({ analytics, available, recorded }) =>
Context.make(AnalyticsService, analytics).pipe(
Context.add(ScientAnalyticsAvailability, { available }),
+ Context.add(ScientRecordedAnalyticsIngest, recorded),
),
),
);
diff --git a/apps/web/src/scient/providerConnection/ProviderRuntimeSection.browser.test.tsx b/apps/web/src/scient/providerConnection/ProviderRuntimeSection.browser.test.tsx
index cde7c6ef52..afac648c95 100644
--- a/apps/web/src/scient/providerConnection/ProviderRuntimeSection.browser.test.tsx
+++ b/apps/web/src/scient/providerConnection/ProviderRuntimeSection.browser.test.tsx
@@ -1836,6 +1836,7 @@ describe("ProviderRuntimeSection", () => {
const cancelledProvider: ServerProvider = {
...provider,
+ driver: ProviderDriverKind.make("claudeAgent"),
installed: true,
version: "2.1.170",
status: "ready",
@@ -1877,6 +1878,51 @@ describe("ProviderRuntimeSection", () => {
expect(markup).toContain('aria-label="Use Scient-managed Claude"');
expect(markup).not.toContain("Provider runtime setup cancelled");
expect(markup).not.toContain("previous working runtime");
+
+ const restartedProvider: ServerProvider = {
+ ...cancelledProvider,
+ connection: {
+ ...cancelledProvider.connection!,
+ runtime: {
+ ...cancelledProvider.connection!.runtime!,
+ operation: null,
+ pendingRecovery: {
+ requestId: "requested-before-restart",
+ action: "install",
+ nextRetryAt: "2026-11-16T12:05:00.000Z",
+ },
+ },
+ },
+ };
+ const clearedProvider: ServerProvider = {
+ ...restartedProvider,
+ connection: {
+ ...restartedProvider.connection!,
+ runtime: { ...restartedProvider.connection!.runtime!, pendingRecovery: null },
+ },
+ };
+ commands.cancel.mockResolvedValue({ _tag: "Success", value: { providers: [clearedProvider] } });
+ const host = await renderSection(_fixture, {
+ compact: true,
+ environmentId,
+ provider: restartedProvider,
+ displayName: "Claude",
+ });
+ expect(host.textContent).toContain("Scient will retry this requested install after");
+ expect(host.querySelector('[role="progressbar"]')).toBeNull();
+ expect(host.querySelector('[aria-label="Cancel Claude install retry"]')).not.toBeNull();
+ const cancelRetry = findActionButton(host, "Cancel retry");
+ expect(cancelRetry).toBeDefined();
+ await act(() => cancelRetry!.click());
+ await vi.waitFor(() => {
+ expect(commands.cancel).toHaveBeenCalledWith({
+ environmentId,
+ input: { instanceId, requestId: "requested-before-restart" },
+ });
+ expect(host.textContent).not.toContain("Cancel retry");
+ });
+ expect(host.textContent).toContain("System installation");
+ expect(commands.start).not.toHaveBeenCalled();
});
it("shows the current missing-runtime state after removal instead of a stale success row", async () => {
diff --git a/apps/web/src/scient/providerConnection/ProviderRuntimeSection.tsx b/apps/web/src/scient/providerConnection/ProviderRuntimeSection.tsx
index e166329380..c17158ab3b 100644
--- a/apps/web/src/scient/providerConnection/ProviderRuntimeSection.tsx
+++ b/apps/web/src/scient/providerConnection/ProviderRuntimeSection.tsx
@@ -75,6 +75,11 @@ type LocalRuntimeFailure =
readonly kind: "operation";
readonly operationId: ProviderRuntimeOperation["operationId"];
readonly message: string;
+ }
+ | {
+ readonly kind: "recovery";
+ readonly requestId: string;
+ readonly message: string;
};
function runtimeFromResult(
@@ -101,6 +106,9 @@ function localRuntimeFailureMessage(
? failure.message
: null;
}
+ if (failure.kind === "recovery") {
+ return runtime.pendingRecovery?.requestId === failure.requestId ? failure.message : null;
+ }
const operation = runtime.operation;
return operation?.operationId === failure.operationId &&
isActiveProviderRuntimeOperation(operation)
@@ -114,6 +122,12 @@ export function resolveProviderRuntimeForPresentation(
): ProviderRuntimeSummary | null | undefined {
if (!serverRuntime) return localRuntime;
if (!localRuntime) return serverRuntime;
+ if (
+ serverRuntime.pendingRecovery &&
+ localRuntime.pendingRecovery === null &&
+ !isActiveProviderRuntimeOperation(localRuntime.operation)
+ )
+ return localRuntime;
if (serverRuntime.operation?.operationId === localRuntime.operation?.operationId) {
return serverRuntime;
}
@@ -258,6 +272,7 @@ export function ProviderRuntimeSection(props: {
const operation = runtime?.operation ?? null;
const runtimeActions = runtime?.actions;
const activeOperation = isActiveProviderRuntimeOperation(operation) ? operation : null;
+ const pendingRecovery = runtime?.pendingRecovery;
const plan =
!activeOperation && preparedPlan?.action === "remove" && runtimeActions?.includes("remove")
? preparedPlan
@@ -444,27 +459,30 @@ export function ProviderRuntimeSection(props: {
};
const cancel = async () => {
- if (!activeOperation) return;
+ if (!activeOperation && !pendingRecovery) return;
setLocalFailure(null);
setPendingAction("cancel");
const result = await cancelRuntime({
environmentId,
input: {
instanceId: provider.instanceId,
- operationId: activeOperation.operationId,
+ ...(activeOperation
+ ? { operationId: activeOperation.operationId }
+ : { requestId: pendingRecovery!.requestId }),
},
});
setPendingAction(null);
if (result._tag === "Failure") {
if (!isAtomCommandInterrupted(result)) {
- setLocalFailure({
- kind: "operation",
- operationId: activeOperation.operationId,
- message: providerLifecycleFailureMessage(
- squashAtomCommandFailure(result),
- `Scient could not cancel the ${runtimeDisplayName} runtime operation.`,
- ),
- });
+ const message = providerLifecycleFailureMessage(
+ squashAtomCommandFailure(result),
+ `Scient could not cancel the ${runtimeDisplayName} runtime operation.`,
+ );
+ setLocalFailure(
+ activeOperation
+ ? { kind: "operation", operationId: activeOperation.operationId, message }
+ : { kind: "recovery", requestId: pendingRecovery!.requestId, message },
+ );
}
return;
}
@@ -650,6 +668,28 @@ export function ProviderRuntimeSection(props: {
: "space-y-3 rounded-lg border p-3"
}
>
+ {pendingRecovery ? (
+
+
+ Scient will retry this requested {pendingRecovery.action} after{" "}
+
+ .
+
+
+
+ ) : null}
>> = {
+ check: ["started", "completed", "failed"],
+ download: ["started", "completed", "failed"],
+ install: ["handoff-attempted", "failed"],
+ apply: ["completed", "not-applied", "superseded"],
+};
+const UPDATE_FAILURE_CLASSES = [
+ "network",
+ "timeout",
+ "certificate",
+ "http_4xx",
+ "http_5xx",
+ "feed",
+ "checksum",
+ "signature",
+ "disk_full",
+ "permission",
+ "configuration",
+ "cancelled",
+ "internal",
+ "unknown",
+];
+const UPDATE_STATUSES = [
+ "disabled",
+ "idle",
+ "checking",
+ "up-to-date",
+ "available",
+ "downloading",
+ "downloaded",
+ "error",
+];
+const UPDATE_DISABLED_CLASSES = [
+ "policy",
+ "channel-policy",
+ "safety",
+ "no-feed",
+ "development",
+ "environment",
+ "linux-package",
+ "other",
+];
+const UPDATE_ATTEMPT_REF_PATTERN = /^[0-9a-f]{16}$/u;
const OPERATION_KINDS = new Set([
"file-preview",
"pdf-open",
@@ -842,6 +901,17 @@ function byteBucket(value: unknown): string {
return "over-100mb";
}
+/** Bytes per second, bucketed in megabits per second as connections are quoted. */
+function throughputBucket(value: unknown): string {
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return "unknown";
+ const megabits = (value * 8) / 1_000_000;
+ if (megabits < 1) return "under-1mbps";
+ if (megabits < 5) return "1-5mbps";
+ if (megabits < 20) return "5-20mbps";
+ if (megabits < 100) return "20-100mbps";
+ return "over-100mbps";
+}
+
function percentBucket(value: unknown): string {
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return "unknown";
if (value < 25) return "0-24";
@@ -861,6 +931,98 @@ export function durationBucket(value: unknown): string {
return "over-10m";
}
+/** How long an update stage or offer has lasted: minutes while transferring, days while waiting. */
+function updateAgeBucket(value: unknown): string {
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return "unknown";
+ if (value < 60_000) return "under-1m";
+ if (value < 600_000) return "1-10m";
+ if (value < 3_600_000) return "10-60m";
+ if (value < 21_600_000) return "1-6h";
+ if (value < 86_400_000) return "6-24h";
+ if (value < 3 * 86_400_000) return "1-3d";
+ if (value < 7 * 86_400_000) return "3-7d";
+ return "over-7d";
+}
+
+/** Copies a field only when it is one of `allowed`; any other value stays absent. */
+function optionalClosed(
+ input: Readonly>,
+ key: string,
+ allowed: ReadonlyArray,
+): Record {
+ const value = property(input, key);
+ return typeof value === "string" && allowed.includes(value) ? { [key]: value } : {};
+}
+
+/** Marks every field read, so an event sends only what its normalizer sets. */
+function closedEvent(input: Readonly>): void {
+ for (const key of Object.keys(input)) property(input, key);
+}
+
+function updateOutcomeProperties(input: Readonly>) {
+ closedEvent(input);
+ const step = closedCategory(property(input, "step"), Object.keys(UPDATE_STEP_OUTCOMES));
+ const outcome = closedCategory(property(input, "outcome"), UPDATE_STEP_OUTCOMES[step] ?? []);
+ const failureClass = closedCategory(property(input, "failureClass"), UPDATE_FAILURE_CLASSES);
+ const trigger = property(input, "trigger");
+ const targetVersion = property(input, "targetVersion");
+ const attemptRef = property(input, "attemptRef");
+ const durationMs = property(input, "durationMs");
+ return {
+ step,
+ outcome,
+ ...(outcome === "failed" ? { failureClass } : {}),
+ ...(step !== "apply" && (trigger === "user" || trigger === "background") ? { trigger } : {}),
+ ...(typeof targetVersion === "string" && analyticsAppVersion(targetVersion) !== "unknown"
+ ? { targetVersion }
+ : {}),
+ ...optionalClosed(input, "sourceKind", ["github", "scient_mirror", "administrator"]),
+ ...optionalClosed(input, "packageKind", ["zip", "nsis", "appimage", "deb"]),
+ ...(typeof attemptRef === "string" && UPDATE_ATTEMPT_REF_PATTERN.test(attemptRef)
+ ? { attemptRef }
+ : {}),
+ ...(durationMs === undefined ? {} : { durationBucket: updateAgeBucket(durationMs) }),
+ };
+}
+
+function updateSummaryProperties(input: Readonly>) {
+ closedEvent(input);
+ const status = closedCategory(property(input, "status"), UPDATE_STATUSES);
+ const lastCheckAgeMs = property(input, "lastCheckAgeMs");
+ return {
+ reason: closedCategory(property(input, "reason"), ["launch", "change", "daily"]),
+ status,
+ channel: closedCategory(property(input, "channel"), ["latest", "beta", "nightly"]),
+ behind: closedCategory(property(input, "behind"), ["yes", "no", "unknown"]),
+ stageAgeBucket: updateAgeBucket(property(input, "stageAgeMs")),
+ lastCheckAgeBucket: lastCheckAgeMs === undefined ? "never" : updateAgeBucket(lastCheckAgeMs),
+ droppedFactsBucket: countBucket(property(input, "droppedFactCount") ?? 0),
+ ...(status === "error"
+ ? {
+ errorContext: closedCategory(property(input, "errorContext"), [
+ "check",
+ "download",
+ "install",
+ ]),
+ }
+ : {}),
+ ...(status === "downloading"
+ ? {
+ progressBucket: percentBucket(property(input, "downloadPercent")),
+ sinceProgressBucket: updateAgeBucket(property(input, "sinceProgressMs")),
+ }
+ : {}),
+ ...(status === "downloaded"
+ ? { readyAgeBucket: updateAgeBucket(property(input, "readyAgeMs")) }
+ : {}),
+ ...(status === "disabled"
+ ? {
+ disabledClass: closedCategory(property(input, "disabledClass"), UPDATE_DISABLED_CLASSES),
+ }
+ : {}),
+ };
+}
+
function turnTerminalStatus(value: unknown): string {
if (value === "cancelled" || value === "interrupted") return "stopped";
return normalizedEnum(value, new Set(["completed", "failed", "stopped"]), "other");
@@ -1037,29 +1199,43 @@ function normalizeEvent(
case "provider.lifecycle.started":
case "provider.lifecycle.completed":
case "provider.lifecycle.failed":
- case "provider.lifecycle.cancelled":
+ case "provider.lifecycle.cancelled": {
+ const failed = name === "provider.lifecycle.failed";
+ // Read on every lifecycle event so a stray class never passes through on the others.
+ const failureClass = normalizedEnum(property(input, "failureClass"), FAILURE_CLASSES);
return {
name,
- privacyLevel: name === "provider.lifecycle.failed" ? "essential" : "product",
- priority: name === "provider.lifecycle.failed" ? "critical" : "core",
+ privacyLevel: failed ? "essential" : "product",
+ priority: failed ? "critical" : "core",
properties: {
provider,
action: normalizedEnum(property(input, "action"), LIFECYCLE_ACTIONS),
runtimeSource: normalizedEnum(property(input, "source"), RUNTIME_SOURCES),
stage: normalizedEnum(property(input, "stage"), LIFECYCLE_STAGES),
- failureClass: normalizedEnum(property(input, "failureClass"), FAILURE_CLASSES),
+ ...(failed ? { failureClass } : {}),
durationBucket: durationBucket(property(input, "durationMs")),
trigger: normalizedEnum(property(input, "trigger"), LIFECYCLE_TRIGGERS, "user"),
...optionalProperties(input, {
attempt: "count",
downloadSource: DOWNLOAD_SOURCES,
- compressed: "boolean",
+ sourceKind: DOWNLOAD_SOURCE_KINDS,
+ downloadVariant: DOWNLOAD_VARIANTS,
+ downloadAttempts: "count",
+ networkKind: DOWNLOAD_NETWORK_KINDS,
+ compressedFailure: FAILURE_CLASSES,
}),
...(property(input, "resumedBytes") === undefined
? {}
: { resumedBucket: byteBucket(property(input, "resumedBytes")) }),
+ ...(property(input, "progressPercent") === undefined
+ ? {}
+ : { progressBucket: percentBucket(property(input, "progressPercent")) }),
+ ...(property(input, "bytesPerSecond") === undefined
+ ? {}
+ : { throughputBucket: throughputBucket(property(input, "bytesPerSecond")) }),
},
};
+ }
case "provider.runtime.first_pick":
return {
name,
@@ -1112,6 +1288,23 @@ function normalizeEvent(
...optionalProperties(input, { stoppedCount: "count" }),
},
};
+ case "app.update.outcome": {
+ const properties = updateOutcomeProperties(input);
+ return {
+ name,
+ privacyLevel: "essential",
+ priority:
+ properties.step === "install" || properties.step === "apply" ? "critical" : "core",
+ properties,
+ };
+ }
+ case "app.update.summary":
+ return {
+ name,
+ privacyLevel: "essential",
+ priority: "summary",
+ properties: updateSummaryProperties(input),
+ };
case "app.session.started":
return {
name,
diff --git a/packages/scient-analytics/src/index.ts b/packages/scient-analytics/src/index.ts
index 70590c5e11..09fe6701a5 100644
--- a/packages/scient-analytics/src/index.ts
+++ b/packages/scient-analytics/src/index.ts
@@ -14,5 +14,6 @@ export {
createAnalyticsRuntime,
type AnalyticsRuntime,
type AnalyticsRuntimeOptions,
+ type RecordedAnalyticsEvent,
} from "./runtime.ts";
export type { AnalyticsWorkerInput } from "./workerProtocol.ts";
diff --git a/packages/scient-analytics/src/runtime.test.ts b/packages/scient-analytics/src/runtime.test.ts
index cb44280f23..bf020a7dc0 100644
--- a/packages/scient-analytics/src/runtime.test.ts
+++ b/packages/scient-analytics/src/runtime.test.ts
@@ -280,4 +280,81 @@ describe("Scient analytics background runtime", () => {
expect(deadLetters.count).toBe(1);
expect(server.bodies).toHaveLength(0);
});
+
+ it("ingests a recorded event once, keeping its id, time and release", async () => {
+ const outboxPath = fixturePath();
+ const runtime = createAnalyticsRuntime(
+ options({ outboxPath, now: () => new Date("2026-11-16T12:00:00.000Z") }),
+ );
+ const recorded = {
+ id: "6f1c2a9e-4b7d-4e21-9a3f-0c5d8e7b1a24",
+ name: "app.update.outcome",
+ occurredAt: "2026-11-16T11:00:00.000Z",
+ appVersion: "0.6.29",
+ buildChannel: "stable",
+ properties: { step: "install", outcome: "handoff-attempted", targetVersion: "0.6.30" },
+ } as const;
+ try {
+ expect(await runtime.ingest([recorded])).toBe(true);
+ // A replay after a lost acknowledgement finds the same row.
+ expect(await runtime.ingest([recorded])).toBe(true);
+ expect(await runtime.pendingCount()).toBe(1);
+ } finally {
+ await runtime.close();
+ }
+ const database = new NodeSqlite.DatabaseSync(outboxPath);
+ const row = database
+ .prepare("SELECT id, occurred_at, privacy_level, properties_json FROM analytics_outbox")
+ .get() as {
+ readonly id: string;
+ readonly occurred_at: string;
+ readonly privacy_level: string;
+ readonly properties_json: string;
+ };
+ database.close();
+ expect(row).toMatchObject({
+ id: recorded.id,
+ occurred_at: recorded.occurredAt,
+ privacy_level: "essential",
+ });
+ expect(JSON.parse(row.properties_json)).toMatchObject({
+ appVersion: "0.6.29",
+ step: "install",
+ targetVersion: "0.6.30",
+ });
+ });
+
+ it("settles recorded events it must not send without queueing them", async () => {
+ const runtime = createAnalyticsRuntime(
+ options({ now: () => new Date("2026-11-16T12:00:00.000Z") }),
+ );
+ const summary = {
+ name: "app.update.summary",
+ appVersion: "0.6.29",
+ buildChannel: "stable",
+ properties: { reason: "launch", status: "idle" },
+ } as const;
+ try {
+ // Too old, a malformed id, and an unknown name are never sent.
+ expect(
+ await runtime.ingest([
+ {
+ ...summary,
+ id: "6f1c2a9e-4b7d-4e21-9a3f-0c5d8e7b1a25",
+ occurredAt: "2026-10-01T00:00:00.000Z",
+ },
+ { ...summary, id: "not-a-uuid", occurredAt: "2026-11-16T11:00:00.000Z" },
+ {
+ ...summary,
+ id: "6f1c2a9e-4b7d-4e21-9a3f-0c5d8e7b1a26",
+ name: "Not An Event",
+ occurredAt: "2026-11-16T11:00:00.000Z",
+ },
+ ]),
+ ).toBe(true);
+ expect(await runtime.pendingCount()).toBe(0);
+ } finally {
+ await runtime.close();
+ }
+ });
});
diff --git a/packages/scient-analytics/src/runtime.ts b/packages/scient-analytics/src/runtime.ts
index 93b8e46234..49fb56b91d 100644
--- a/packages/scient-analytics/src/runtime.ts
+++ b/packages/scient-analytics/src/runtime.ts
@@ -22,6 +22,9 @@ const MAX_IN_FLIGHT_BATCHES = 2;
const WRITE_COALESCE_MS = 100;
const CONTROL_TIMEOUT_MS = 3_500;
const LOCAL_SHUTDOWN_BUDGET_MS = 300;
+const RECORDED_EVENT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u;
+const RECORDED_EVENT_MAX_AGE_MS = 30 * 86_400_000;
+const RECORDED_EVENT_MAX_SKEW_MS = 86_400_000;
const PRIORITY_RANK: Readonly> = {
critical: 0,
@@ -40,9 +43,22 @@ export interface AnalyticsRuntimeOptions extends NormalizationContext {
readonly controlTimeoutMs?: number;
}
+/** An event another Scient process recorded; it keeps its own id, time and release. */
+export interface RecordedAnalyticsEvent extends NormalizationContext {
+ readonly id: string;
+ readonly name: string;
+ readonly occurredAt: string;
+ readonly properties: Readonly>;
+}
+
export interface AnalyticsRuntime {
readonly enabled: boolean;
readonly record: (name: string, properties?: Readonly>) => boolean;
+ /**
+ * True once the outbox holds these events or none of them may be sent (shape, age);
+ * false when that is unconfirmed, so the caller keeps them to offer again.
+ */
+ readonly ingest: (events: ReadonlyArray) => Promise;
readonly flush: () => Promise;
readonly pendingCount: () => Promise;
readonly close: () => Promise;
@@ -52,6 +68,7 @@ function disabledRuntime(): AnalyticsRuntime {
return {
enabled: false,
record: () => false,
+ ingest: async () => true,
flush: async () => 0,
pendingCount: async () => 0,
close: async () => undefined,
@@ -98,6 +115,7 @@ export function createAnalyticsRuntime(options: AnalyticsRuntimeOptions): Analyt
let nextRequestId = 1;
const queue: AnalyticsEventDraft[] = [];
const inFlight = new Map>();
+ const ingestions = new Map void>();
const requests = new Map<
number,
{
@@ -127,6 +145,8 @@ export function createAnalyticsRuntime(options: AnalyticsRuntimeOptions): Analyt
}
queue.length = 0;
inFlight.clear();
+ for (const settle of ingestions.values()) settle(false);
+ ingestions.clear();
settleRequests();
};
@@ -203,6 +223,8 @@ export function createAnalyticsRuntime(options: AnalyticsRuntimeOptions): Analyt
return;
case "persisted":
inFlight.delete(response.batchId);
+ ingestions.get(response.batchId)?.(true);
+ ingestions.delete(response.batchId);
drain();
return;
case "result": {
@@ -262,11 +284,68 @@ export function createAnalyticsRuntime(options: AnalyticsRuntimeOptions): Analyt
return accepted;
};
+ const recordedDraft = (event: RecordedAnalyticsEvent): AnalyticsEventDraft | null => {
+ const occurredAt = Date.parse(event.occurredAt);
+ const nowMs = now().valueOf();
+ if (
+ !RECORDED_EVENT_ID.test(event.id) ||
+ !Number.isFinite(occurredAt) ||
+ occurredAt > nowMs + RECORDED_EVENT_MAX_SKEW_MS ||
+ occurredAt < nowMs - RECORDED_EVENT_MAX_AGE_MS
+ ) {
+ return null;
+ }
+ const normalized = normalizeInheritedEvent(event.name, event.properties, {
+ appVersion: event.appVersion,
+ buildChannel: event.buildChannel,
+ });
+ if (!normalized) return null;
+ return {
+ id: event.id,
+ name: normalized.name,
+ session_id: sessionId,
+ occurred_at: new Date(occurredAt).toISOString(),
+ privacy_level: normalized.privacyLevel,
+ // Same constant `record` sends: the gateway still reads a consent level.
+ consent_level: "diagnostic",
+ properties: normalized.properties,
+ priority: normalized.priority,
+ };
+ };
+
+ const ingest: AnalyticsRuntime["ingest"] = async (events) => {
+ if (closed || !available) return false;
+ const drafts = events.flatMap((event) => recordedDraft(event) ?? []);
+ if (drafts.length === 0) return true;
+ await waitForCondition(() => ready || !available, Date.now() + controlTimeoutMs);
+ if (!ready || !available || closed) return false;
+ const batchId = nextBatchId++;
+ return new Promise((resolve) => {
+ const timer = setTimeout(() => {
+ ingestions.delete(batchId);
+ resolve(false);
+ }, controlTimeoutMs);
+ timer.unref();
+ ingestions.set(batchId, (persisted) => {
+ clearTimeout(timer);
+ resolve(persisted);
+ });
+ inFlight.set(batchId, drafts);
+ if (!post({ type: "enqueue", batchId, events: drafts })) {
+ inFlight.delete(batchId);
+ ingestions.delete(batchId);
+ clearTimeout(timer);
+ resolve(false);
+ }
+ });
+ };
+
return {
get enabled() {
return available && !closed;
},
record,
+ ingest,
flush: async () => {
if (closed || !available) return 0;
await waitForCondition(() => ready || !available, Date.now() + controlTimeoutMs);
diff --git a/packages/scient-provider-runtime/src/managedProviderRuntime.ts b/packages/scient-provider-runtime/src/managedProviderRuntime.ts
index b6829a4f89..ef5f3ee95a 100644
--- a/packages/scient-provider-runtime/src/managedProviderRuntime.ts
+++ b/packages/scient-provider-runtime/src/managedProviderRuntime.ts
@@ -15,6 +15,11 @@ import {
downloadManagedRuntime,
type ManagedRuntimeDownloadResult,
type ManagedRuntimeDownloadRetry,
+ type ManagedRuntimeFailureClass,
+ managedRuntimeFailureClass,
+ managedRuntimeFailureRetryable,
+ managedRuntimeRetryAfterMs,
+ ManagedRuntimeFileError,
materializeManagedRuntimeArtifact,
resolveManagedRuntimeArtifactPath,
unpackManagedRuntimeZstd,
@@ -45,14 +50,26 @@ export interface ManagedProviderRuntimeProgress {
readonly totalBytes?: number | undefined;
/** Set while the download waits to try again after a temporary failure. */
readonly retry?: ManagedRuntimeDownloadRetry | undefined;
- /** Set once the download is complete, on the `verifying` stage. */
+ /** How the download went: after each failed try, and once complete on the `verifying` stage. */
readonly download?: ManagedRuntimeDownloadFacts | undefined;
}
-/** How a finished download went, for analytics. */
+/** Who served a download: Scient's own copy, GitHub, or the provider's own host. */
+export type ManagedRuntimeSourceKind = "scient_mirror" | "github" | "vendor";
+
+/** How a download went, for analytics. */
export interface ManagedRuntimeDownloadFacts extends ManagedRuntimeDownloadResult {
/** The zstd copy, not the raw binary, was downloaded. */
readonly compressed: boolean;
+ /** Who served the latest try. */
+ readonly sourceKind: ManagedRuntimeSourceKind;
+ /** Why the zstd copy failed, when the raw binary was downloaded after it. */
+ readonly compressedFailure?: ManagedRuntimeFailureClass | undefined;
+}
+
+function hostSourceKind(url: string): ManagedRuntimeSourceKind {
+ const host = new URL(url).hostname.toLowerCase();
+ return host === "github.com" || host.endsWith(".githubusercontent.com") ? "github" : "vendor";
}
export interface ManagedProviderRuntimeStateV1 {
@@ -116,8 +133,6 @@ export interface ManagedProviderRuntimeInstallInput {
readonly beforeActivate?: (signal: AbortSignal) => Promise;
/** Told why the compressed copy failed, before the raw binary is downloaded; a throw stops the install. */
readonly onCompressedFallback?: (cause: unknown) => void;
- /** Per download copy; `null` waits as long as data keeps arriving. Ten minutes by default. */
- readonly downloadTimeLimitMs?: number | null | undefined;
}
interface ManagedProviderRuntimeActivation {
@@ -683,11 +698,13 @@ export class ManagedProviderRuntime {
readonly signal: AbortSignal;
readonly onProgress: ManagedProviderRuntimeInstallInput["onProgress"];
readonly onCompressedFallback: ManagedProviderRuntimeInstallInput["onCompressedFallback"];
- readonly timeLimitMs: ManagedProviderRuntimeInstallInput["downloadTimeLimitMs"];
}): Promise {
const { artifact, archivePath, signal, onProgress } = input;
const compressed = artifact.compressed;
const unpack = this.#dependencies.unpackZstd;
+ let compressedFailure: ManagedRuntimeFailureClass | undefined;
+ let compressedCause: unknown;
+ const sourceCooldowns = new Map();
const fetchVerified = async (copy: {
readonly url: string;
readonly mirrorUrl?: string | undefined;
@@ -696,7 +713,18 @@ export class ManagedProviderRuntime {
readonly destination: string;
readonly compressed: boolean;
}) => {
- onProgress?.({ stage: "downloading", downloadedBytes: 0, totalBytes: copy.size });
+ const describe = (download: ManagedRuntimeDownloadResult): ManagedRuntimeDownloadFacts => ({
+ ...download,
+ compressed: copy.compressed,
+ sourceKind:
+ download.source === "fallback" || !copy.mirrorUrl
+ ? hostSourceKind(copy.url)
+ : "scient_mirror",
+ ...(compressedFailure ? { compressedFailure } : {}),
+ });
+ let downloadedBytes = 0;
+ // The downloader reports the bytes an earlier try kept as soon as it has checked them.
+ onProgress?.({ stage: "downloading", totalBytes: copy.size });
const result = await this.#dependencies.download({
...(copy.mirrorUrl
? {
@@ -705,21 +733,27 @@ export class ManagedProviderRuntime {
allowedHosts: [...artifact.allowedHosts, new URL(copy.mirrorUrl).hostname],
}
: { url: copy.url, allowedHosts: artifact.allowedHosts }),
+ sourceCooldowns,
checksum: copy.checksum,
destination: copy.destination,
partialPath: this.#partialDownloadPath(copy.checksum),
- ...(input.timeLimitMs === undefined ? {} : { timeLimitMs: input.timeLimitMs }),
expectedSize: copy.size,
signal,
- onProgress: (downloadedBytes, totalBytes) =>
- onProgress?.({ stage: "downloading", downloadedBytes, totalBytes }),
+ onProgress: (bytes, totalBytes) => {
+ downloadedBytes = bytes;
+ onProgress?.({ stage: "downloading", downloadedBytes: bytes, totalBytes });
+ },
onRetry: (retry) =>
- onProgress?.({ stage: "downloading", downloadedBytes: 0, totalBytes: copy.size, retry }),
- });
- onProgress?.({
- stage: "verifying",
- ...(result ? { download: { ...result, compressed: copy.compressed } } : {}),
+ onProgress?.({ stage: "downloading", downloadedBytes, totalBytes: copy.size, retry }),
+ onFailedTry: (download) =>
+ onProgress?.({
+ stage: "downloading",
+ downloadedBytes,
+ totalBytes: copy.size,
+ download: describe(download),
+ }),
});
+ onProgress?.({ stage: "verifying", ...(result ? { download: describe(result) } : {}) });
};
if (compressed && unpack) {
const compressedPath = `${archivePath}.zst`;
@@ -737,34 +771,55 @@ export class ManagedProviderRuntime {
} catch (cause) {
if (signal.aborted) throw cause;
await NodeFSP.rm(archivePath, { force: true });
+ compressedCause = cause;
+ compressedFailure = managedRuntimeFailureClass(cause);
input.onCompressedFallback?.(cause);
} finally {
await NodeFSP.rm(compressedPath, { force: true });
}
}
- await fetchVerified({
- url: artifact.url,
- mirrorUrl: managedRuntimeMirrorUrl(artifact),
- checksum: artifact.checksum,
- size: artifact.size,
- destination: archivePath,
- compressed: false,
- });
- await this.#dependencies.verify(archivePath, artifact.checksum);
+ try {
+ await fetchVerified({
+ url: artifact.url,
+ mirrorUrl: managedRuntimeMirrorUrl(artifact),
+ checksum: artifact.checksum,
+ size: artifact.size,
+ destination: archivePath,
+ compressed: false,
+ });
+ await this.#dependencies.verify(archivePath, artifact.checksum);
+ } catch (cause) {
+ if (signal.aborted || compressedCause === undefined) throw cause;
+ const failureClass = managedRuntimeFailureClass(cause);
+ const sourceFailure =
+ failureClass === "http_4xx" ||
+ failureClass === "http_5xx" ||
+ failureClass === "network" ||
+ failureClass === "timeout" ||
+ failureClass === "checksum" ||
+ failureClass === "size_mismatch" ||
+ failureClass === "certificate";
+ throw new ManagedRuntimeFileError(
+ cause instanceof Error ? cause.message : "Managed runtime download failed.",
+ {
+ cause,
+ failureClass,
+ retryable:
+ managedRuntimeFailureRetryable(cause) ||
+ (sourceFailure && managedRuntimeFailureRetryable(compressedCause)),
+ retryAfterMs: Math.max(
+ managedRuntimeRetryAfterMs(cause) ?? 0,
+ managedRuntimeRetryAfterMs(compressedCause) ?? 0,
+ ),
+ },
+ );
+ }
}
async #installOwned(
input: ManagedProviderRuntimeInstallInput,
): Promise {
- const {
- artifact,
- signal,
- onProgress,
- qualify,
- beforeActivate,
- onCompressedFallback,
- downloadTimeLimitMs,
- } = input;
+ const { artifact, signal, onProgress, qualify, beforeActivate, onCompressedFallback } = input;
await NodeFSP.mkdir(this.#stagingDir, { recursive: true, mode: 0o700 });
// This process launches what it installs. Pruning needs the mutation lock held here.
await this.#lease(artifact.version);
@@ -789,7 +844,6 @@ export class ManagedProviderRuntime {
signal,
onProgress,
onCompressedFallback,
- timeLimitMs: downloadTimeLimitMs,
});
onProgress?.({ stage: "installing" });
const stagedExecutable = await this.#dependencies.materialize({
diff --git a/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts b/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts
index 0e8c18aa26..d5a4174320 100644
--- a/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts
+++ b/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts
@@ -8,9 +8,11 @@ import * as NodeZlib from "node:zlib";
import { afterEach, describe, expect, it, vi } from "vite-plus/test";
+import type { ManagedProviderRuntimeProgress } from "./managedProviderRuntime.ts";
import { hydrateManagedRuntimeArtifact } from "./managedRuntimeArtifact.ts";
import { ManagedScientAgentRuntime } from "./managedScientAgentRuntime.ts";
import { resolveScientAgentArtifactPolicy } from "./scientAgentManifest.ts";
+import { managedRuntimeFailureRetryable, managedRuntimeRetryAfterMs } from "./runtimeFiles.ts";
vi.mock("node:timers/promises", { spy: true });
// The transient-response backoff resolves at once but still honours cancellation.
@@ -62,6 +64,7 @@ async function installedFixture() {
}
afterEach(async () => {
+ vi.useRealTimers();
vi.unstubAllGlobals();
await Promise.all(
roots.splice(0).map((root) => NodeFSP.rm(root, { recursive: true, force: true })),
@@ -110,6 +113,41 @@ it("recovers the Scient download and still runs smoke, admission, and qualificat
expect(await NodeFSP.readFile(otherState, "utf8")).toBe("keep");
});
+// regression: progress fell back to zero while a resumed download waited or started again.
+it("reports progress from the bytes kept by an earlier try", async () => {
+ const { root } = await installedFixture();
+ // Cut off after 5 bytes on Scient's copy and after 10 on GitHub, then whole on Scient's.
+ const ends = [5, 10, bytes.length];
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async (_url: URL, init: RequestInit) => {
+ const range = (init.headers as Record | undefined)?.range;
+ const from = Number(/bytes=(\d+)-/u.exec(range ?? "")?.[1] ?? 0);
+ const chunk = new Uint8Array(bytes.subarray(from, ends.shift()));
+ return from === 0
+ ? new Response(chunk, { headers: { "content-length": String(bytes.length) } })
+ : new Response(chunk, {
+ status: 206,
+ headers: { "content-range": `bytes ${from}-${bytes.length - 1}/${bytes.length}` },
+ });
+ }),
+ );
+ const downloaded: number[] = [];
+ const runtime = new ManagedScientAgentRuntime(root, { smoke: async () => undefined });
+ await runtime.install({
+ artifact: artifact("0.2.0"),
+ signal: new AbortController().signal,
+ onProgress: (report) => {
+ if (report.stage === "downloading" && report.downloadedBytes !== undefined)
+ downloaded.push(report.downloadedBytes);
+ },
+ });
+ expect(downloaded.at(-1)).toBe(bytes.length);
+ expect(downloaded.slice(downloaded.indexOf(5))).toEqual(
+ downloaded.slice(downloaded.indexOf(5)).toSorted((left, right) => left - right),
+ );
+});
+
it.each(["checksum", "smoke", "admission", "qualification", "cancellation"] as const)(
"preserves the installed release when %s refuses an update after a transient response",
async (failure) => {
@@ -258,28 +296,88 @@ describe("installing from the compressed copy", () => {
});
it.each([
- { failure: "a damaged copy on both servers", catalogCopy: packed, served: Buffer.from("junk") },
+ {
+ failure: "a damaged copy on both servers",
+ catalogCopy: packed,
+ served: Buffer.from("junk"),
+ compressedFailure: "size_mismatch",
+ },
{
failure: "a copy that unpacks to other bytes",
catalogCopy: NodeZlib.zstdCompressSync(Buffer.from("other bytes")),
served: NodeZlib.zstdCompressSync(Buffer.from("other bytes")),
+ compressedFailure: "other",
},
- ])("falls back to the binary after $failure", async ({ catalogCopy, served }) => {
- const { root } = await installedFixture();
+ ])(
+ "falls back to the binary after $failure",
+ async ({ catalogCopy, served, compressedFailure }) => {
+ const { root } = await installedFixture();
+ const urls: string[] = [];
+ vi.stubGlobal("fetch", serve(served, urls));
+ const onCompressedFallback = vi.fn();
+ const progress: ManagedProviderRuntimeProgress[] = [];
+ const runtime = new ManagedScientAgentRuntime(root, { smoke: async () => undefined });
+ await runtime.install({
+ artifact: compressedArtifact("0.2.0", catalogCopy),
+ signal: new AbortController().signal,
+ onCompressedFallback,
+ onProgress: (report) => progress.push(report),
+ });
+ expect(onCompressedFallback).toHaveBeenCalledOnce();
+ // The binary's download still says why the compressed copy was given up.
+ expect(progress.findLast((report) => report.stage === "verifying")?.download).toMatchObject({
+ compressed: false,
+ sourceKind: "scient_mirror",
+ compressedFailure,
+ });
+ expect(urls.at(-1)).toBe(
+ "https://downloads.scientfactory.com/scient-agent/v0.2.0/scient-agent-darwin-arm64",
+ );
+ await expectInstalled(runtime, root, "0.2.0");
+ },
+ );
+
+ it("carries a compressed source cooldown through raw fallback and the terminal failure", async () => {
+ const { root, previousState } = await installedFixture();
+ vi.useFakeTimers({ toFake: ["Date"] });
+ vi.setSystemTime(0);
+ using wait = vi.spyOn(NodeTimersPromises, "setTimeout");
+ wait.mockClear();
const urls: string[] = [];
- vi.stubGlobal("fetch", serve(served, urls));
- const onCompressedFallback = vi.fn();
- const runtime = new ManagedScientAgentRuntime(root, { smoke: async () => undefined });
- await runtime.install({
- artifact: compressedArtifact("0.2.0", catalogCopy),
- signal: new AbortController().signal,
- onCompressedFallback,
+ const fetch = vi.fn(async (url: URL) => {
+ urls.push(String(url));
+ return url.hostname === "downloads.scientfactory.com"
+ ? new Response("busy", { status: 429, headers: { "retry-after": "3600" } })
+ : new Response("missing", { status: 404 });
});
- expect(onCompressedFallback).toHaveBeenCalledOnce();
- expect(urls.at(-1)).toBe(
- "https://downloads.scientfactory.com/scient-agent/v0.2.0/scient-agent-darwin-arm64",
- );
- await expectInstalled(runtime, root, "0.2.0");
+ vi.stubGlobal("fetch", fetch);
+ const runtime = new ManagedScientAgentRuntime(root, { smoke: async () => undefined });
+ const compressedFailures: unknown[] = [];
+ const failure = await runtime
+ .install({
+ artifact: compressedArtifact("0.2.0"),
+ signal: new AbortController().signal,
+ onCompressedFallback: (cause) => {
+ compressedFailures.push(cause);
+ vi.advanceTimersByTime(10_000);
+ },
+ })
+ .catch((cause: unknown) => cause);
+ expect(compressedFailures).toHaveLength(1);
+ expect(managedRuntimeRetryAfterMs(compressedFailures[0])).toBe(3_590_000);
+ expect(managedRuntimeRetryAfterMs(new Error("wrapped", { cause: failure }))).toBe(3_590_000);
+ expect(managedRuntimeFailureRetryable(failure)).toBe(true);
+ expect(String(failure)).toContain("github.com Managed runtime download failed with HTTP 404");
+ expect(urls).toEqual([
+ "https://downloads.scientfactory.com/scient-agent/v0.2.0/scient-agent-darwin-arm64.zst",
+ "https://github.com/ScientFactory/scient-agent/releases/download/v0.2.0/scient-agent-darwin-arm64.zst",
+ "https://github.com/ScientFactory/scient-agent/releases/download/v0.2.0/scient-agent-darwin-arm64",
+ ]);
+ expect(wait).not.toHaveBeenCalled();
+ expect(await runtime.readState()).toEqual(previousState);
+ expect(
+ await NodeFSP.readdir(NodePath.join(root, "provider-runtimes", "scient-agent", "staging")),
+ ).toEqual([]);
});
it("stops without the binary when told not to fall back, or when cancelled", async () => {
diff --git a/packages/scient-provider-runtime/src/runtimeDownload.test.ts b/packages/scient-provider-runtime/src/runtimeDownload.test.ts
index 4e59c46c06..6ec7522253 100644
--- a/packages/scient-provider-runtime/src/runtimeDownload.test.ts
+++ b/packages/scient-provider-runtime/src/runtimeDownload.test.ts
@@ -1,4 +1,4 @@
-// @effect-diagnostics nodeBuiltinImport:off -- Synthetic responses exercise the private download boundary without network or provider credentials.
+// @effect-diagnostics nodeBuiltinImport:off globalTimers:off -- Synthetic responses exercise the private download boundary without network or provider credentials; the slow-download case paces its body on the test clock.
import * as NodeCrypto from "node:crypto";
import * as NodeFSP from "node:fs/promises";
import * as NodeOS from "node:os";
@@ -11,6 +11,8 @@ import {
downloadManagedRuntime,
ManagedRuntimeFileError,
managedRuntimeFailureClass,
+ managedRuntimeFailureRetryable,
+ managedRuntimeRetryAfterMs,
verifyManagedRuntimeChecksum,
} from "./runtimeFiles.ts";
@@ -18,12 +20,14 @@ vi.mock("node:timers/promises", { spy: true });
// The retry backoff resolves at once but still honours cancellation; the
// "cancels the retry delay promptly" test drives the real timer.
beforeEach(() => {
+ vi.useFakeTimers({ toFake: ["Date"] });
vi.mocked(NodeTimersPromises.setTimeout).mockImplementation((async (
- _delay: number,
+ delay: number,
value: unknown,
options?: { signal?: AbortSignal },
) => {
options?.signal?.throwIfAborted();
+ vi.advanceTimersByTime(delay);
return value;
}) as never);
});
@@ -42,6 +46,7 @@ async function input(signal = new AbortController().signal) {
}
afterEach(async () => {
+ vi.useRealTimers();
vi.unstubAllGlobals();
await Promise.all(
roots.splice(0).map((root) => NodeFSP.rm(root, { recursive: true, force: true })),
@@ -109,41 +114,60 @@ describe("transient managed runtime downloads", () => {
await expect(NodeFSP.stat(request.destination)).rejects.toMatchObject({ code: "ENOENT" });
});
- it("waits as long as the server asks, up to a minute", async () => {
+ it("honours a short server cooldown inside the local retry budget", async () => {
const request = await input();
const fetch = vi
.fn()
.mockResolvedValueOnce(new Response("busy", { status: 503, headers: { "retry-after": "7" } }))
- .mockResolvedValueOnce(
- new Response("busy", { status: 429, headers: { "retry-after": "3600" } }),
- )
.mockResolvedValueOnce(new Response("ok", { headers: { "content-length": "2" } }));
vi.stubGlobal("fetch", fetch);
vi.mocked(NodeTimersPromises.setTimeout).mockClear();
await downloadManagedRuntime(request);
expect(vi.mocked(NodeTimersPromises.setTimeout).mock.calls.map(([delay]) => delay)).toEqual([
- 7_000, 60_000,
+ 7_000,
]);
+ expect(fetch).toHaveBeenCalledTimes(2);
});
- it("cancels recovery without sending another request", async () => {
- const controller = new AbortController();
- const request = await input(controller.signal);
- const fetch = vi.fn().mockImplementation(() =>
- Promise.resolve(
- new Response(
- new ReadableStream({
- cancel: () => controller.abort(),
- }),
- { status: 503 },
+ it.each(["3600", "99999999999999999999999999999999999999999999999999999999999999999999999"])(
+ "leaves long Retry-After %s to durable recovery without waiting locally",
+ async (header) => {
+ const request = await input();
+ const fetch = vi.fn(
+ async () => new Response("busy", { status: 429, headers: { "retry-after": header } }),
+ );
+ vi.stubGlobal("fetch", fetch);
+ vi.mocked(NodeTimersPromises.setTimeout).mockClear();
+ const failure = await downloadManagedRuntime(request).catch((cause: unknown) => cause);
+ expect(failure).toBeInstanceOf(ManagedRuntimeFileError);
+ expect(managedRuntimeFailureRetryable(failure)).toBe(true);
+ expect(managedRuntimeRetryAfterMs(failure)).toBe(header === "3600" ? 3_600_000 : 14_400_000);
+ expect(fetch).toHaveBeenCalledOnce();
+ expect(NodeTimersPromises.setTimeout).not.toHaveBeenCalled();
+ },
+ );
+
+ it.each([undefined, "3600"])(
+ "cancels recovery without another request (Retry-After %s)",
+ async (header) => {
+ const controller = new AbortController();
+ const request = await input(controller.signal);
+ const fetch = vi.fn().mockImplementation(() =>
+ Promise.resolve(
+ new Response(
+ new ReadableStream({
+ cancel: () => controller.abort(),
+ }),
+ { status: 503, ...(header ? { headers: { "retry-after": header } } : {}) },
+ ),
),
- ),
- );
- vi.stubGlobal("fetch", fetch);
- await expect(downloadManagedRuntime(request)).rejects.toMatchObject({ name: "AbortError" });
- expect(fetch).toHaveBeenCalledOnce();
- await expect(NodeFSP.stat(request.destination)).rejects.toMatchObject({ code: "ENOENT" });
- });
+ );
+ vi.stubGlobal("fetch", fetch);
+ await expect(downloadManagedRuntime(request)).rejects.toMatchObject({ name: "AbortError" });
+ expect(fetch).toHaveBeenCalledOnce();
+ await expect(NodeFSP.stat(request.destination)).rejects.toMatchObject({ code: "ENOENT" });
+ },
+ );
it("retains host admission when a retried response redirects", async () => {
const request = await input();
@@ -207,37 +231,46 @@ describe("transient managed runtime downloads", () => {
await expect(NodeFSP.stat(request.destination)).rejects.toMatchObject({ code: "ENOENT" });
});
- it("cancels the retry delay promptly", async () => {
- const controller = new AbortController();
- const request = await input(controller.signal);
- const bodyReleased = Promise.withResolvers();
- const fetch = vi.fn().mockImplementation(
- async () =>
- new Response(new ReadableStream({ cancel: () => bodyReleased.resolve() }), {
- status: 503,
- }),
- );
- vi.stubGlobal("fetch", fetch);
- const retryStarted = Promise.withResolvers();
- const { setTimeout: waitForRetry } =
- await vi.importActual("node:timers/promises");
- vi.mocked(NodeTimersPromises.setTimeout).mockClear();
- using retryTimer = vi.spyOn(NodeTimersPromises, "setTimeout").mockImplementation((...args) => {
- const waiting = waitForRetry(...args);
- retryStarted.resolve();
- return waiting;
- });
- const download = downloadManagedRuntime(request);
- await bodyReleased.promise;
- await retryStarted.promise;
- const [delay] = retryTimer.mock.calls[0]!;
- // The first wait is about two seconds, spread by up to a fifth either way.
- expect(delay).toBeGreaterThanOrEqual(1_600);
- expect(delay).toBeLessThanOrEqual(2_400);
- controller.abort();
- await expect(download).rejects.toMatchObject({ name: "AbortError" });
- expect(fetch).toHaveBeenCalledOnce();
- });
+ it.each([undefined, "60"])(
+ "cancels the retry delay promptly (Retry-After %s)",
+ async (header) => {
+ const controller = new AbortController();
+ const request = await input(controller.signal);
+ const bodyReleased = Promise.withResolvers();
+ const fetch = vi.fn().mockImplementation(
+ async () =>
+ new Response(new ReadableStream({ cancel: () => bodyReleased.resolve() }), {
+ status: 503,
+ ...(header ? { headers: { "retry-after": header } } : {}),
+ }),
+ );
+ vi.stubGlobal("fetch", fetch);
+ const retryStarted = Promise.withResolvers();
+ const { setTimeout: waitForRetry } =
+ await vi.importActual("node:timers/promises");
+ vi.mocked(NodeTimersPromises.setTimeout).mockClear();
+ using retryTimer = vi
+ .spyOn(NodeTimersPromises, "setTimeout")
+ .mockImplementation((...args) => {
+ const waiting = waitForRetry(...args);
+ retryStarted.resolve();
+ return waiting;
+ });
+ const download = downloadManagedRuntime(request);
+ await bodyReleased.promise;
+ await retryStarted.promise;
+ const [delay] = retryTimer.mock.calls[0]!;
+ if (header) expect(delay).toBe(60_000);
+ else {
+ // The first wait is about two seconds, spread by up to a fifth either way.
+ expect(delay).toBeGreaterThanOrEqual(1_600);
+ expect(delay).toBeLessThanOrEqual(2_400);
+ }
+ controller.abort();
+ await expect(download).rejects.toMatchObject({ name: "AbortError" });
+ expect(fetch).toHaveBeenCalledOnce();
+ },
+ );
it("retries a failed connection and names it when every try fails", async () => {
const request = await input();
@@ -348,29 +381,88 @@ describe("downloads with Scient's own copy first", () => {
expect(urls(fetch)).toEqual([mirror, request.fallbackUrl, request.fallbackUrl]);
});
- it("gives GitHub its own time limit when Scient's copy runs out of time", async () => {
+ it.each([
+ { mirrorFirst: true, fallbackElapsedMs: 0 },
+ { mirrorFirst: false, fallbackElapsedMs: 0 },
+ { mirrorFirst: true, fallbackElapsedMs: 10_000 },
+ ])(
+ "preserves the mirror cooldown after GitHub fails ($mirrorFirst, $fallbackElapsedMs)",
+ async ({ mirrorFirst, fallbackElapsedMs }) => {
+ vi.setSystemTime(0);
+ using _random = vi.spyOn(Math, "random").mockReturnValue(0.5);
+ const base = await twoSources();
+ const request = mirrorFirst
+ ? base
+ : { ...base, url: base.fallbackUrl, fallbackUrl: base.url };
+ let fallbackTries = 0;
+ const fetch = vi.fn(async (url: URL) => {
+ if (url.hostname === "downloads.scientfactory.com")
+ return new Response("busy", { status: 429, headers: { "retry-after": "3600" } });
+ if (fallbackTries++ === 0) vi.advanceTimersByTime(fallbackElapsedMs);
+ return new Response("busy", { status: 503 });
+ });
+ vi.stubGlobal("fetch", fetch);
+ const failure = await downloadManagedRuntime(request).catch((cause: unknown) => cause);
+ expect(failure).toBeInstanceOf(ManagedRuntimeFileError);
+ expect(managedRuntimeFailureRetryable(failure)).toBe(true);
+ expect(failure).toMatchObject({ retryAfterMs: 3_548_000 - fallbackElapsedMs });
+ expect(managedRuntimeRetryAfterMs(new Error("wrapped", { cause: failure }))).toBe(
+ 3_548_000 - fallbackElapsedMs,
+ );
+ expect(urls(fetch).filter((url) => url === mirror)).toHaveLength(1);
+ expect(fallbackTries).toBe(5);
+ expect(fetch).toHaveBeenCalledTimes(6);
+ expect(String(failure)).toContain("downloads.scientfactory.com returned HTTP 429");
+ expect(String(failure)).toContain("github.com returned HTTP 503");
+ vi.advanceTimersByTime(3_600_000);
+ expect(managedRuntimeRetryAfterMs(failure)).toBeUndefined();
+ },
+ );
+
+ it.each([false, true])(
+ "exposes the longest outstanding source cooldown (reverse %s)",
+ async (reverse) => {
+ vi.setSystemTime(0);
+ const base = await twoSources();
+ const request = reverse ? { ...base, url: base.fallbackUrl, fallbackUrl: base.url } : base;
+ const fetch = vi.fn(
+ async (url: URL) =>
+ new Response("busy", {
+ status: 429,
+ headers: {
+ "retry-after": url.hostname === "downloads.scientfactory.com" ? "7200" : "3600",
+ },
+ }),
+ );
+ vi.stubGlobal("fetch", fetch);
+ vi.mocked(NodeTimersPromises.setTimeout).mockClear();
+ const failure = await downloadManagedRuntime(request).catch((cause: unknown) => cause);
+ expect(managedRuntimeRetryAfterMs(failure)).toBe(7_200_000);
+ expect(fetch).toHaveBeenCalledTimes(2);
+ expect(NodeTimersPromises.setTimeout).not.toHaveBeenCalled();
+ },
+ );
+
+ it("revisits a cooled source only once its deadline has elapsed", async () => {
+ vi.setSystemTime(0);
+ using _random = vi.spyOn(Math, "random").mockReturnValue(0.5);
const request = await twoSources();
- const limits: AbortController[] = [];
- using _timeout = vi.spyOn(AbortSignal, "timeout").mockImplementation(() => {
- const limit = new AbortController();
- limits.push(limit);
- return limit.signal;
- });
const fetch = vi
.fn()
- .mockImplementationOnce(
- (_url: URL, init: RequestInit) =>
- new Promise((_resolve, reject) => {
- init.signal!.addEventListener("abort", () => reject(init.signal!.reason));
- // Scient's copy is still busy when its own ten minutes end.
- limits[0]!.abort(new DOMException("time limit", "TimeoutError"));
- }),
- )
+ .mockResolvedValueOnce(new Response("busy", { status: 429, headers: { "retry-after": "7" } }))
+ .mockResolvedValueOnce(new Response("busy", { status: 503 }))
+ .mockResolvedValueOnce(new Response("busy", { status: 503 }))
.mockResolvedValueOnce(ok());
vi.stubGlobal("fetch", fetch);
- await downloadManagedRuntime(request);
- expect(urls(fetch)).toEqual([mirror, request.fallbackUrl]);
- expect(limits).toHaveLength(2);
+ vi.mocked(NodeTimersPromises.setTimeout).mockClear();
+ await expect(downloadManagedRuntime(request)).resolves.toMatchObject({
+ tries: 4,
+ source: "primary",
+ });
+ expect(urls(fetch)).toEqual([mirror, request.fallbackUrl, request.fallbackUrl, mirror]);
+ expect(vi.mocked(NodeTimersPromises.setTimeout).mock.calls.map(([delay]) => delay)).toEqual([
+ 2_000, 5_000,
+ ]);
});
it("fails only when both copies fail, and names both", async () => {
@@ -427,9 +519,11 @@ describe("resumable downloads", () => {
return partialContent(2, "llo");
});
vi.stubGlobal("fetch", fetch);
- await expect(downloadManagedRuntime(request)).resolves.toEqual({
+ await expect(downloadManagedRuntime(request)).resolves.toMatchObject({
resumedBytes: 2,
source: "primary",
+ tries: 2,
+ progressPercent: 100,
});
expect(await NodeFSP.readFile(request.destination, "utf8")).toBe(body);
expect(await NodeFSP.readdir(NodePath.dirname(request.partialPath))).toEqual([]);
@@ -511,36 +605,36 @@ describe("resumable downloads", () => {
return partialContent(2, "llo");
});
vi.stubGlobal("fetch", fetch);
- await expect(downloadManagedRuntime(request)).resolves.toEqual({
+ await expect(downloadManagedRuntime(request)).resolves.toMatchObject({
resumedBytes: 2,
source: "fallback",
});
});
- it("keeps the bytes when the time limit ends and says trying again continues", async () => {
+ it("says how far a failed download got, and that a later try may finish it", async () => {
const request = await resumable();
- const limit = new AbortController();
- using _timeout = vi.spyOn(AbortSignal, "timeout").mockReturnValue(limit.signal);
- await NodeFSP.mkdir(NodePath.dirname(request.partialPath), { recursive: true });
- await NodeFSP.writeFile(request.partialPath, "he");
- const fetch = vi.fn().mockImplementationOnce(
- (_url: URL, init: RequestInit) =>
- new Promise((_resolve, reject) => {
- init.signal!.addEventListener("abort", () => reject(init.signal!.reason));
- limit.abort(new DOMException("time limit", "TimeoutError"));
- }),
- );
+ const reset = Object.assign(new Error("private detail"), { code: "ECONNRESET" });
+ const fetch = vi
+ .fn()
+ .mockResolvedValueOnce(new Response("he"))
+ .mockRejectedValue(new TypeError("fetch failed", { cause: reset }));
vi.stubGlobal("fetch", fetch);
- const failure = await downloadManagedRuntime(request).then(
- () => expect.unreachable("the download should stop"),
+ const onFailedTry = vi.fn();
+ const failure = await downloadManagedRuntime({ ...request, onFailedTry }).then(
+ () => expect.unreachable("the download should fail"),
(cause: unknown) => cause,
);
- expect(failure).toBeInstanceOf(ManagedRuntimeFileError);
- expect((failure as Error).message).toBe(
- "The download took longer than 10 minutes. Scient kept what it downloaded; trying again continues from there.",
- );
- expect(managedRuntimeFailureClass(failure)).toBe("timeout");
- expect(await NodeFSP.readFile(request.partialPath, "utf8")).toBe("he");
+ expect(onFailedTry).toHaveBeenCalledTimes(5);
+ expect(onFailedTry.mock.calls[0]![0]).toMatchObject({ tries: 1, networkKind: "cut_off" });
+ expect(onFailedTry.mock.calls.at(-1)![0]).toMatchObject({
+ resumedBytes: 2,
+ source: "primary",
+ tries: 5,
+ progressPercent: 40,
+ networkKind: "reset",
+ });
+ expect(managedRuntimeFailureClass(failure)).toBe("network");
+ expect(managedRuntimeFailureRetryable(failure)).toBe(true);
});
it("starts the retries afresh while a slow line keeps bringing data", async () => {
@@ -571,6 +665,73 @@ describe("resumable downloads", () => {
// Nine tries, more than the five a failing server gets.
expect(fetch).toHaveBeenCalledTimes(9);
});
+
+ it("bounds retries when a server repeatedly replaces the same retained prefix", async () => {
+ const request = await resumable();
+ const content = Buffer.alloc(2 * 1024 * 1024, 7);
+ const fetch = vi.fn(async () => new Response(content.subarray(0, 1024 * 1024)));
+ vi.stubGlobal("fetch", fetch);
+ await expect(
+ downloadManagedRuntime({
+ ...request,
+ expectedSize: content.length,
+ checksum: {
+ algorithm: "sha256",
+ digest: NodeCrypto.createHash("sha256").update(content).digest("hex"),
+ },
+ }),
+ ).rejects.toThrow("cut off on all 5 tries");
+ expect(fetch).toHaveBeenCalledTimes(5);
+ expect((await NodeFSP.stat(request.partialPath)).size).toBe(1024 * 1024);
+ });
+});
+
+describe("downloads without an overall time limit", () => {
+ // regression: slow downloads failed after ten minutes while data still arrived.
+ it("keeps a slow download going past ten minutes while data arrives", async () => {
+ const request = { ...(await input()), expectedSize: 36 * 1024 };
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"] });
+ // A limit built on AbortSignal.timeout would otherwise ignore the test clock.
+ using _timeout = vi.spyOn(AbortSignal, "timeout").mockImplementation((ms) => {
+ const limit = new AbortController();
+ setTimeout(() => limit.abort(new DOMException("time limit", "TimeoutError")), ms);
+ return limit.signal;
+ });
+ let body!: ReadableStreamDefaultController;
+ const response = new Response(
+ new ReadableStream({ start: (controller) => (body = controller) }),
+ {
+ headers: { "content-length": String(request.expectedSize) },
+ },
+ );
+ vi.stubGlobal("fetch", vi.fn().mockResolvedValue(response));
+ let awaited: { readonly bytes: number; readonly resolve: () => void } | undefined;
+ const download = downloadManagedRuntime({
+ ...request,
+ onProgress: (bytes) => {
+ if (awaited && bytes >= awaited.bytes) awaited.resolve();
+ },
+ });
+ const ended = download.then(
+ () => undefined,
+ () => undefined,
+ );
+ // Opening the file takes real time, so the clock moves on only once each KiB has arrived.
+ const written = (bytes: number) =>
+ Promise.race([new Promise((resolve) => (awaited = { bytes, resolve })), ended]);
+ // 1 KiB every 20 seconds, inside the 30-second stall bound: twelve minutes in all.
+ for (let chunk = 1; chunk <= 36; chunk += 1) {
+ await vi.advanceTimersByTimeAsync(20_000);
+ body.enqueue(new Uint8Array(1024).fill(7));
+ await written(chunk * 1024);
+ }
+ body.close();
+ await expect(download).resolves.toMatchObject({
+ tries: 1,
+ progressPercent: 100,
+ bytesPerSecond: Math.round(request.expectedSize / 720),
+ });
+ });
});
describe("managed runtime failure classes", () => {
@@ -607,6 +768,35 @@ describe("managed runtime failure classes", () => {
);
});
+ it.each([
+ ["HTTP 429 on every try", () => new Response("slow down", { status: 429 }), true],
+ ["HTTP 501", () => new Response("never", { status: 501 }), false],
+ ["a missing file", () => new Response("missing", { status: 404 }), false],
+ [
+ "an intercepted certificate",
+ () => Promise.reject(fetchFailure("SELF_SIGNED_CERT_IN_CHAIN")),
+ false,
+ ],
+ ])("says whether a later try may fix %s", async (_case, response, retryable) => {
+ vi.stubGlobal("fetch", vi.fn().mockImplementation(response));
+ const failure = await failureOf(downloadManagedRuntime(await input()));
+ expect(managedRuntimeFailureRetryable(failure)).toBe(retryable);
+ });
+
+ it("reads the remaining bounded cooldown through wrappers without hiding a longer inner one", () => {
+ vi.setSystemTime(0);
+ const inner = new ManagedRuntimeFileError("busy", { retryAfterMs: 99_999_999 });
+ const wrapped = new ManagedRuntimeFileError("outer", { cause: inner, retryAfterMs: 1_000 });
+ expect(managedRuntimeRetryAfterMs(new Error("wrapped", { cause: wrapped }))).toBe(14_400_000);
+ vi.advanceTimersByTime(60_000);
+ expect(inner.retryAfterMs).toBe(14_340_000);
+ expect(wrapped.retryAfterMs).toBeUndefined();
+ expect(managedRuntimeRetryAfterMs(wrapped)).toBe(14_340_000);
+ vi.advanceTimersByTime(14_340_000);
+ expect(managedRuntimeRetryAfterMs(wrapped)).toBeUndefined();
+ expect(managedRuntimeRetryAfterMs(new Error("Retry-After: 3600"))).toBeUndefined();
+ });
+
it("classifies a damaged download, cancellation and local disk errors without reading messages", async () => {
const request = await input();
await NodeFSP.writeFile(request.destination, "ko");
@@ -634,6 +824,8 @@ describe("managed runtime failure classes", () => {
new Error("Scient could not install it.", { cause: Object.assign(new Error("x"), { code }) });
expect(managedRuntimeFailureClass(wrapped("ENOSPC"))).toBe("disk_full");
expect(managedRuntimeFailureClass(wrapped("EACCES"))).toBe("permission");
+ for (const local of [damaged, wrapped("ENOSPC"), wrapped("EACCES")])
+ expect(managedRuntimeFailureRetryable(local)).toBe(false);
expect(
managedRuntimeFailureClass(new ManagedRuntimeFileError("Unsafe archive path"), "installing"),
).toBe("extract");
diff --git a/packages/scient-provider-runtime/src/runtimeFiles.ts b/packages/scient-provider-runtime/src/runtimeFiles.ts
index 432622ce19..bfb3a7018a 100644
--- a/packages/scient-provider-runtime/src/runtimeFiles.ts
+++ b/packages/scient-provider-runtime/src/runtimeFiles.ts
@@ -27,7 +27,9 @@ function isTransientDownloadStatus(status: number): boolean {
* minute in all, spread so many installs don't retry at the same moment.
*/
const RETRY_DELAYS_MS = [2_000, 5_000, 15_000, 30_000] as const;
-const MAX_RETRY_AFTER_MS = 60_000;
+const MAX_RETRY_AFTER_MS = 4 * 60 * 60 * 1_000;
+// Covers all four jittered backoffs; longer server waits belong to durable recovery.
+const LOCAL_RETRY_WAIT_BUDGET_MS = 65_000;
/** Largest artifact the installer downloads; release discovery rejects larger ones. */
export const MANAGED_RUNTIME_MAX_DOWNLOAD_BYTES = 512 * 1024 * 1024;
const MAX_DOWNLOAD_BYTES = MANAGED_RUNTIME_MAX_DOWNLOAD_BYTES;
@@ -38,7 +40,6 @@ const DEFAULT_EXTRACTION_LIMITS: ManagedRuntimeExtractionLimits = {
maxExpandedBytes: 512 * 1024 * 1024,
maxEntries: 32,
};
-const DOWNLOAD_TIMEOUT_MS = 10 * 60 * 1_000;
const DOWNLOAD_IDLE_TIMEOUT_MS = 30 * 1_000;
/** A failed try that still brought this much of a resumable download starts the retries afresh. */
const RESUME_PROGRESS_BYTES = 1024 * 1024;
@@ -62,16 +63,60 @@ export type ManagedRuntimeFailureClass =
// @effect-diagnostics-next-line extendsNativeError:off - @scientfactory/provider-runtime has no Effect runtime; callers catch it as a native error.
export class ManagedRuntimeFileError extends Error {
readonly failureClass: ManagedRuntimeFailureClass | undefined;
+ /** Whether trying the same change again later may succeed, when the class alone can't say. */
+ readonly retryable: boolean | undefined;
+ readonly #retryNotBefore: number | undefined;
+ /** Remaining server cooldown, bounded to four hours; absent once it has elapsed. */
+ get retryAfterMs(): number | undefined {
+ if (this.#retryNotBefore === undefined) return undefined;
+ const remaining = Math.min(MAX_RETRY_AFTER_MS, this.#retryNotBefore - Date.now());
+ return remaining > 0 ? remaining : undefined;
+ }
constructor(
message: string,
- options?: ErrorOptions & { readonly failureClass?: ManagedRuntimeFailureClass },
+ options?: ErrorOptions & {
+ readonly failureClass?: ManagedRuntimeFailureClass;
+ readonly retryable?: boolean;
+ readonly retryAfterMs?: number;
+ },
) {
super(message, options);
this.name = "ManagedRuntimeFileError";
this.failureClass = options?.failureClass;
+ this.retryable = options?.retryable;
+ const delay = options?.retryAfterMs;
+ this.#retryNotBefore =
+ delay !== undefined && Number.isFinite(delay) && delay > 0
+ ? Date.now() + Math.min(delay, MAX_RETRY_AFTER_MS)
+ : undefined;
}
}
+/** How a try's connection failed, from its error code; never its message. */
+export type ManagedRuntimeNetworkKind =
+ | "dns"
+ | "refused"
+ | "reset"
+ | "unreachable"
+ | "connect_timeout"
+ | "stalled"
+ | "cut_off"
+ | "other";
+
+const NETWORK_KINDS: Readonly> = {
+ ENOTFOUND: "dns",
+ EAI_AGAIN: "dns",
+ ECONNREFUSED: "refused",
+ ECONNRESET: "reset",
+ ECONNABORTED: "reset",
+ EPIPE: "reset",
+ UND_ERR_SOCKET: "reset",
+ ENETUNREACH: "unreachable",
+ EHOSTUNREACH: "unreachable",
+ ETIMEDOUT: "connect_timeout",
+ UND_ERR_CONNECT_TIMEOUT: "connect_timeout",
+};
+
function assertAllowedHttpsUrl(url: URL, allowedHosts: ReadonlySet): void {
if (url.protocol !== "https:") {
throw new ManagedRuntimeFileError("Managed runtime downloads must use HTTPS.");
@@ -99,11 +144,12 @@ class TransientDownloadFailure extends Error {
readonly reason: string;
readonly retryAfterMs: number | null;
readonly status: number | null;
+ readonly networkKind: ManagedRuntimeNetworkKind | undefined;
constructor(
kind: TransientDownloadFailure["kind"],
reason: string,
retryAfterMs: number | null = null,
- options?: ErrorOptions & { readonly status?: number },
+ options?: ErrorOptions & { readonly status?: number; readonly code?: string | undefined },
) {
super(reason, options);
this.name = "TransientDownloadFailure";
@@ -111,6 +157,14 @@ class TransientDownloadFailure extends Error {
this.reason = reason;
this.retryAfterMs = retryAfterMs;
this.status = options?.status ?? null;
+ this.networkKind =
+ kind === "stalled"
+ ? "stalled"
+ : kind === "cut-off"
+ ? "cut_off"
+ : kind === "connection"
+ ? ((options?.code === undefined ? undefined : NETWORK_KINDS[options.code]) ?? "other")
+ : undefined;
}
}
@@ -160,12 +214,7 @@ export function managedRuntimeFailureClass(
cause: unknown,
stage?: string,
): ManagedRuntimeFailureClass {
- const chain: unknown[] = [];
- for (let current = cause; current !== undefined && chain.length < 8;) {
- chain.push(current);
- current = current instanceof Error ? current.cause : undefined;
- }
- const errors = chain.filter((entry): entry is Error => entry instanceof Error);
+ const errors = causeChain(cause);
for (const error of errors) {
if (error instanceof ManagedRuntimeFileError && error.failureClass !== undefined)
return error.failureClass;
@@ -184,10 +233,49 @@ export function managedRuntimeFailureClass(
return "other";
}
+function causeChain(cause: unknown): ReadonlyArray {
+ const errors: Error[] = [];
+ for (let current = cause; current instanceof Error && errors.length < 8; current = current.cause)
+ errors.push(current);
+ return errors;
+}
+
+const RETRYABLE_FAILURE_CLASSES = new Set([
+ "network",
+ "timeout",
+ "busy",
+]);
+
+/**
+ * Whether the same change may succeed if tried again later, from error types: a server that stayed
+ * busy (HTTP 408, 429, most 5xx) or a lost connection may; a 501, certificate, checksum, disk or
+ * permission failure will not.
+ */
+export function managedRuntimeFailureRetryable(cause: unknown, stage?: string): boolean {
+ for (const error of causeChain(cause)) {
+ if (error instanceof ManagedRuntimeFileError && error.retryable !== undefined)
+ return error.retryable;
+ if (error instanceof TransientDownloadFailure) return true;
+ }
+ return RETRYABLE_FAILURE_CLASSES.has(managedRuntimeFailureClass(cause, stage));
+}
+
+/** Remaining server cooldown through wrapped failures; contains no transport details. */
+export function managedRuntimeRetryAfterMs(cause: unknown): number | undefined {
+ let remaining = 0;
+ for (const error of causeChain(cause)) {
+ if (error instanceof ManagedRuntimeFileError)
+ remaining = Math.max(remaining, error.retryAfterMs ?? 0);
+ }
+ return remaining > 0 ? remaining : undefined;
+}
+
function retryAfterMs(header: string | null): number | null {
if (!header) return null;
const seconds = Number(header);
- const ms = Number.isFinite(seconds) ? seconds * 1_000 : Date.parse(header) - Date.now();
+ const ms = Number.isFinite(seconds)
+ ? Math.min(seconds, MAX_RETRY_AFTER_MS / 1_000) * 1_000
+ : Date.parse(header) - Date.now();
return Number.isFinite(ms) ? Math.min(Math.max(ms, 0), MAX_RETRY_AFTER_MS) : null;
}
@@ -209,7 +297,7 @@ function connectionFailure(cause: unknown): TransientDownloadFailure | ManagedRu
"connection",
code ? `connection failed, ${code}` : "connection failed",
null,
- { cause },
+ { cause, code },
);
}
@@ -283,6 +371,8 @@ interface ManagedRuntimeDownloadInput {
* before going back to a source that already failed.
*/
readonly fallbackUrl?: string | undefined;
+ /** Absolute not-before times by HTTPS origin, shared across compressed and raw copies. */
+ readonly sourceCooldowns?: Map | undefined;
/** Checked after every try, so a damaged copy counts as that source failing. */
readonly checksum?: ManagedRuntimeChecksum | undefined;
readonly destination: string;
@@ -291,21 +381,38 @@ interface ManagedRuntimeDownloadInput {
* Name it by the release digest: both copies serve the same bytes. Needs `checksum`.
*/
readonly partialPath?: string | undefined;
- /** Per copy; `null` waits as long as data keeps arriving. Ten minutes by default. */
- readonly timeLimitMs?: number | null | undefined;
readonly allowedHosts: ReadonlyArray;
readonly expectedSize: number;
readonly signal: AbortSignal;
+ /** From the bytes kept by an earlier try, when a try starts, then as bytes arrive. */
readonly onProgress?: (downloaded: number, total: number) => void;
readonly onRetry?: (retry: ManagedRuntimeDownloadRetry) => void;
+ /** After each failed try, before Scient waits or gives up, so a failed download says how far it got. */
+ readonly onFailedTry?: (download: ManagedRuntimeDownloadResult) => void;
}
-/** What a finished download continued from, for analytics. */
+/** How a download went up to its latest try, for analytics. */
export interface ManagedRuntimeDownloadResult {
- /** Bytes already on disk when the successful try started; 0 for a fresh download. */
+ /** Bytes already on disk when the latest try started; 0 for a fresh download. */
readonly resumedBytes: number;
- /** `fallback` when the second copy served the file. */
+ /** The copy the latest try used; `fallback` is the second. */
readonly source: "primary" | "fallback";
+ /** Tries made, the first included. */
+ readonly tries: number;
+ /** How much of the file the latest try had when it ended, 0 to 100. */
+ readonly progressPercent: number;
+ /** Received bytes per second of request time, kept bytes and waits excluded; absent with none. */
+ readonly bytesPerSecond?: number;
+ /** How the latest try's connection failed. */
+ readonly networkKind?: ManagedRuntimeNetworkKind;
+}
+
+/** Shared by the tries of one download. */
+interface DownloadTransfer {
+ resumedBytes: number;
+ downloadedBytes: number;
+ receivedBytes: number;
+ activeMs: number;
}
function describeSourceFailure(failure: Error): string {
@@ -323,10 +430,10 @@ function describeSourceFailure(failure: Error): string {
}
/**
- * Downloads one reviewed artifact, retrying temporary failures for about a
- * minute. With a fallback copy, any failure of one copy moves to the other.
- * Cancellation, the time limit and local disk errors always stay final.
- * With `partialPath`, a failed try keeps its bytes and the next try asks only for the rest.
+ * Downloads one reviewed artifact, retrying temporary failures (30 seconds without data among them)
+ * for about a minute; a fallback copy takes over from a failing one. No overall time limit: a slow
+ * download goes on while data arrives. Cancellation and local disk errors stay final. With
+ * `partialPath`, a failed try keeps its bytes and the next asks only for the rest.
*/
export async function downloadManagedRuntime(
input: ManagedRuntimeDownloadInput,
@@ -335,116 +442,128 @@ export async function downloadManagedRuntime(
const allowedHosts = new Set(input.allowedHosts.map((host) => host.toLowerCase()));
const expected = input.checksum ? expectedDigest(input.checksum) : undefined;
const partialPath = input.checksum ? input.partialPath : undefined;
- const timeLimitMs = input.timeLimitMs === undefined ? DOWNLOAD_TIMEOUT_MS : input.timeLimitMs;
- // Each copy gets the whole time limit, so a slow copy never uses up the other's.
- const deadlines = new Map();
- const deadline = (source: string): AbortSignal => {
- const existing = deadlines.get(source);
- if (existing) return existing;
- const created =
- timeLimitMs === null ? new AbortController().signal : AbortSignal.timeout(timeLimitMs);
- deadlines.set(source, created);
- return created;
+ const transfer: DownloadTransfer = {
+ resumedBytes: 0,
+ downloadedBytes: 0,
+ receivedBytes: 0,
+ activeMs: 0,
};
- const waitSignal =
- sources.length === 1 ? AbortSignal.any([input.signal, deadline(input.url)]) : input.signal;
const lastFailure = new Map();
let usable = [...sources];
- let round = [...usable];
+ const sourceCooldowns = input.sourceCooldowns ?? new Map();
+ const notBefore = (source: string) => sourceCooldowns.get(new URL(source).origin) ?? 0;
+ const remainingCooldown = () =>
+ Math.max(0, ...sources.map((source) => notBefore(source) - Date.now()));
+ let waitBudgetMs = LOCAL_RETRY_WAIT_BUDGET_MS;
+ let retryProgressBytes = 0;
+ let cause: unknown;
let tries = 0;
+ const facts = (source: string, cause?: unknown): ManagedRuntimeDownloadResult => {
+ const networkKind = cause instanceof TransientDownloadFailure ? cause.networkKind : undefined;
+ return {
+ resumedBytes: transfer.resumedBytes,
+ source: source === input.url ? "primary" : "fallback",
+ tries,
+ progressPercent: Math.min(
+ 100,
+ Math.floor((transfer.downloadedBytes / input.expectedSize) * 100),
+ ),
+ ...(transfer.receivedBytes > 0 && transfer.activeMs > 0
+ ? { bytesPerSecond: Math.round((transfer.receivedBytes * 1_000) / transfer.activeMs) }
+ : {}),
+ ...(networkKind ? { networkKind } : {}),
+ };
+ };
+ const exhausted = async (): Promise => {
+ input.signal.throwIfAborted();
+ const kept =
+ partialPath && (await partialDownloadSize(partialPath)) > 0
+ ? " Scient kept what it downloaded; trying again continues from there."
+ : "";
+ const retryAfterMs = remainingCooldown();
+ const retryable =
+ retryAfterMs > 0 ||
+ [...lastFailure.values()].some((failure) => failure instanceof TransientDownloadFailure);
+ if (sources.length === 1 && cause instanceof TransientDownloadFailure) {
+ throw new ManagedRuntimeFileError(`${exhaustedMessage(cause, tries)}${kept}`, {
+ cause,
+ failureClass: transientFailureClass(cause),
+ retryable,
+ retryAfterMs,
+ });
+ }
+ const details = sources
+ .flatMap((candidate) => {
+ const failure = lastFailure.get(candidate);
+ return failure ? [`${new URL(candidate).hostname} ${describeSourceFailure(failure)}`] : [];
+ })
+ .join(" ");
+ throw new ManagedRuntimeFileError(
+ `Scient couldn't download it from ${sources.length > 1 ? "either source" : "the source"} after ${tries} tries. ${details}${kept}`,
+ {
+ cause,
+ failureClass: managedRuntimeFailureClass(cause),
+ retryable,
+ retryAfterMs,
+ },
+ );
+ };
for (let retry = 0; ;) {
- const source = round.shift()!;
- tries += 1;
- const received = { bytes: 0 };
- let cause: unknown;
- try {
- const attempt = await downloadOnce(
- input,
- source,
- allowedHosts,
- deadline(source),
- partialPath,
- received,
- );
- let resumedBytes = attempt.resumedBytes;
- if (expected !== undefined && attempt.digest !== expected) {
- await removePartialDownload(partialPath ?? input.destination);
- // Bytes kept from an earlier try may be what is wrong, so fetch the whole file once.
- if (resumedBytes === 0) throw checksumMismatch();
- const fresh = await downloadOnce(
- input,
- source,
- allowedHosts,
- deadline(source),
- partialPath,
- received,
- );
- if (fresh.digest !== expected) {
+ for (const source of usable) {
+ input.signal.throwIfAborted();
+ if (notBefore(source) > Date.now()) continue;
+ tries += 1;
+ try {
+ const attempt = await downloadOnce(input, source, allowedHosts, partialPath, transfer);
+ if (expected !== undefined && attempt.digest !== expected) {
await removePartialDownload(partialPath ?? input.destination);
- throw checksumMismatch();
+ // Bytes kept from an earlier try may be what is wrong, so fetch the whole file once.
+ if (attempt.resumedBytes === 0) throw checksumMismatch();
+ const fresh = await downloadOnce(input, source, allowedHosts, partialPath, transfer);
+ if (fresh.digest !== expected) {
+ await removePartialDownload(partialPath ?? input.destination);
+ throw checksumMismatch();
+ }
}
- resumedBytes = 0;
- }
- if (partialPath) {
- await NodeFSP.rename(partialPath, input.destination);
- await NodeFSP.rm(partialValidatorPath(partialPath), { force: true }).catch(() => undefined);
+ if (partialPath) {
+ await NodeFSP.rename(partialPath, input.destination);
+ await NodeFSP.rm(partialValidatorPath(partialPath), { force: true }).catch(
+ () => undefined,
+ );
+ }
+ return facts(source);
+ } catch (caught) {
+ cause = caught;
}
- return { resumedBytes, source: source === input.url ? "primary" : "fallback" };
- } catch (caught) {
- cause = caught;
- }
- // With a second copy, a problem with this one, running out of time
- // included, only rules this copy out. Cancellation still ends everything.
- const outOfTime = deadline(source).aborted && !input.signal.aborted;
- const ruledOut = sources.length > 1 && (cause instanceof ManagedRuntimeFileError || outOfTime);
- if (!(cause instanceof TransientDownloadFailure) && !ruledOut) {
- if (outOfTime && partialPath && (await partialDownloadSize(partialPath)) > 0) {
- throw new ManagedRuntimeFileError(
- `The download took longer than ${describeTimeLimit(timeLimitMs)}. Scient kept what it downloaded; trying again continues from there.`,
- { cause, failureClass: "timeout" },
+ input.signal.throwIfAborted();
+ if (cause instanceof TransientDownloadFailure && cause.retryAfterMs !== null) {
+ sourceCooldowns.set(
+ new URL(source).origin,
+ Math.max(notBefore(source), Date.now() + cause.retryAfterMs),
);
}
- throw cause;
+ input.onFailedTry?.(facts(source, cause));
+ // With a second copy, a problem with this one only rules this copy out.
+ const ruledOut = sources.length > 1 && cause instanceof ManagedRuntimeFileError;
+ if (!(cause instanceof TransientDownloadFailure) && !ruledOut) throw cause;
+ lastFailure.set(source, cause as Error);
+ if (ruledOut) usable = usable.filter((candidate) => candidate !== source);
}
- lastFailure.set(
- source,
- outOfTime
- ? new Error(`took longer than ${describeTimeLimit(timeLimitMs)}.`)
- : (cause as Error),
- );
- if (ruledOut) {
- usable = usable.filter((candidate) => candidate !== source);
- round = round.filter((candidate) => candidate !== source);
+ // Only a growing retained prefix replenishes retries; a server may ignore Range.
+ if (partialPath && transfer.downloadedBytes - retryProgressBytes >= RESUME_PROGRESS_BYTES) {
+ retry = 0;
+ retryProgressBytes = transfer.downloadedBytes;
+ waitBudgetMs = LOCAL_RETRY_WAIT_BUDGET_MS;
}
- if (usable.length > 0 && round.length > 0) continue;
- // Steady progress on a slow line is not a failing server; keep going while it lasts.
- if (received.bytes >= RESUME_PROGRESS_BYTES) retry = 0;
const baseDelay = RETRY_DELAYS_MS[retry];
- if (usable.length === 0 || baseDelay === undefined) {
- const kept =
- partialPath && (await partialDownloadSize(partialPath)) > 0
- ? " Scient kept what it downloaded; trying again continues from there."
- : "";
- if (sources.length === 1) {
- throw new ManagedRuntimeFileError(
- `${exhaustedMessage(cause as TransientDownloadFailure, tries)}${kept}`,
- { cause, failureClass: transientFailureClass(cause as TransientDownloadFailure) },
- );
- }
- const details = sources
- .map(
- (candidate) =>
- `${new URL(candidate).hostname} ${describeSourceFailure(lastFailure.get(candidate)!)}`,
- )
- .join(" ");
- throw new ManagedRuntimeFileError(
- `Scient couldn't download it from either source after ${tries} tries. ${details}${kept}`,
- { cause, failureClass: outOfTime ? "timeout" : managedRuntimeFailureClass(cause) },
- );
- }
- const delayMs =
- (cause instanceof TransientDownloadFailure ? cause.retryAfterMs : null) ??
- Math.round(baseDelay * (0.8 + Math.random() * 0.4));
+ if (usable.length === 0 || baseDelay === undefined) return exhausted();
+ const delayMs = Math.max(
+ Math.round(baseDelay * (0.8 + Math.random() * 0.4)),
+ Math.min(...usable.map((source) => Math.max(0, notBefore(source) - Date.now()))),
+ );
+ // Long server cooldowns belong to durable recovery, not an open install dialog.
+ if (delayMs > waitBudgetMs) return exhausted();
+ waitBudgetMs -= delayMs;
retry += 1;
input.onRetry?.({
retry,
@@ -452,16 +571,10 @@ export async function downloadManagedRuntime(
reason: cause instanceof TransientDownloadFailure ? cause.reason : "download failed",
delayMs,
});
- await NodeTimersPromises.setTimeout(delayMs, undefined, { signal: waitSignal });
- round = [...usable];
+ await NodeTimersPromises.setTimeout(delayMs, undefined, { signal: input.signal });
}
}
-function describeTimeLimit(timeLimitMs: number | null): string {
- const minutes = Math.round((timeLimitMs ?? 0) / 60_000);
- return minutes === 1 ? "1 minute" : `${minutes} minutes`;
-}
-
function checksumMismatch(): ManagedRuntimeFileError {
return new ManagedRuntimeFileError("Managed runtime verification failed: checksum mismatch.", {
failureClass: "checksum",
@@ -535,9 +648,8 @@ async function downloadOnce(
input: ManagedRuntimeDownloadInput,
url: string,
allowedHosts: ReadonlySet,
- timeoutSignal: AbortSignal,
partialPath: string | undefined,
- received: { bytes: number },
+ transfer: DownloadTransfer,
): Promise<{ readonly resumedBytes: number; readonly digest: string | undefined }> {
const target = partialPath ?? input.destination;
const newHash = () =>
@@ -549,10 +661,10 @@ async function downloadOnce(
have = 0;
}
if (have > 0 && hash) await hashPrefix(target, have, hash);
- if (have === input.expectedSize) {
- input.onProgress?.(have, input.expectedSize);
- return { resumedBytes: have, digest: hash?.digest("hex") };
- }
+ transfer.resumedBytes = have;
+ transfer.downloadedBytes = have;
+ input.onProgress?.(have, input.expectedSize);
+ if (have === input.expectedSize) return { resumedBytes: have, digest: hash?.digest("hex") };
const idleController = new AbortController();
let idleTimer: ReturnType | undefined;
@@ -560,12 +672,12 @@ async function downloadOnce(
if (idleTimer) clearTimeout(idleTimer);
idleTimer = setTimeout(() => idleController.abort(), DOWNLOAD_IDLE_TIMEOUT_MS);
};
- const signal = AbortSignal.any([input.signal, timeoutSignal, idleController.signal]);
- // A stall is worth another try; cancellation and the overall limit are not.
+ const signal = AbortSignal.any([input.signal, idleController.signal]);
+ // A stall is worth another try; cancellation is not.
const classify = (cause: unknown, fromNetwork: boolean): unknown => {
if (cause instanceof ManagedRuntimeFileError || cause instanceof TransientDownloadFailure)
return cause;
- if (input.signal.aborted || timeoutSignal.aborted) return cause;
+ if (input.signal.aborted) return cause;
if (idleController.signal.aborted)
return new TransientDownloadFailure("stalled", "download stalled", null, { cause });
return fromNetwork ? connectionFailure(cause) : cause;
@@ -585,6 +697,7 @@ async function downloadOnce(
});
};
resetIdleTimer();
+ const startedAt = Date.now();
try {
let response = await request(have);
@@ -595,6 +708,8 @@ async function downloadOnce(
response = await request(0);
}
have = 0;
+ transfer.resumedBytes = 0;
+ transfer.downloadedBytes = 0;
hash = newHash();
}
if (!response.ok || !response.body) {
@@ -628,7 +743,8 @@ async function downloadOnce(
const source = NodeStream.Readable.fromWeb(response.body as never);
source.on("data", (chunk: Buffer) => {
downloaded += chunk.byteLength;
- received.bytes += chunk.byteLength;
+ transfer.downloadedBytes = downloaded;
+ transfer.receivedBytes += chunk.byteLength;
hash?.update(chunk);
resetIdleTimer();
if (downloaded > MAX_DOWNLOAD_BYTES || downloaded > input.expectedSize) {
@@ -665,6 +781,7 @@ async function downloadOnce(
return { resumedBytes: have, digest: hash?.digest("hex") };
} finally {
if (idleTimer) clearTimeout(idleTimer);
+ transfer.activeMs += Date.now() - startedAt;
}
}
diff --git a/scripts/create-scient-release-handoff.ts b/scripts/create-scient-release-handoff.ts
index db009b7822..28e6e1aaad 100644
--- a/scripts/create-scient-release-handoff.ts
+++ b/scripts/create-scient-release-handoff.ts
@@ -10,6 +10,8 @@ import {
scientReleaseRepository,
} from "@t3tools/shared/scientRelease";
+import { readWindowsSigningAttestation } from "./windows-publication-signing.ts";
+
interface ReleaseAsset {
readonly name: string;
readonly size: number;
@@ -45,6 +47,12 @@ export function createScientReleaseHandoff(args: ReadonlyArray): string
const sourceSha = value(args, "--source-sha");
const sourceTree = value(args, "--source-tree");
const output = NodePath.resolve(value(args, "--output"));
+ // Later publications read this to learn which publisher installed Windows clients pin.
+ const windows = args.includes("--windows-signing")
+ ? readWindowsSigningAttestation(
+ JSON.parse(NodeFS.readFileSync(value(args, "--windows-signing"), "utf8")) as unknown,
+ )
+ : null;
const excluded = new Set([NodePath.basename(output), "SHA256SUMS.txt"]);
const assets: ReleaseAsset[] = NodeFS.readdirSync(assetsDir)
.filter((name) => !excluded.has(name))
@@ -70,6 +78,13 @@ export function createScientReleaseHandoff(args: ReadonlyArray): string
version,
tag: `v${version}`,
distribution: { repository: distributionRepository, channel },
+ ...(windows === null
+ ? {}
+ : {
+ windows: windows.signed
+ ? { signed: true, publisherName: windows.publisherName }
+ : { signed: false },
+ }),
source: {
repository: sourceRepository,
commit: sourceSha,
diff --git a/scripts/scient-beta-publication.test.ts b/scripts/scient-beta-publication.test.ts
index f4a1e0270c..bb2cabb076 100644
--- a/scripts/scient-beta-publication.test.ts
+++ b/scripts/scient-beta-publication.test.ts
@@ -99,6 +99,7 @@ else if (args.includes('repos/ScientFactory/scient-desktop-beta/releases?per_pag
{ channel: "beta", failure: "none" },
{ channel: "beta", failure: "corrupt-upload" },
{ channel: "beta", failure: "stable-advanced" },
+ { channel: "beta", failure: "publisher-pinned" },
])(
"publishes $channel only into its owning repository, failure=$failure",
({ channel, failure }) => {
@@ -110,6 +111,21 @@ else if (args.includes('repos/ScientFactory/scient-desktop-beta/releases?per_pag
try {
NodeFS.mkdirSync(bin);
NodeFS.mkdirSync(NodePath.join(root, "release-assets"));
+ NodeFS.mkdirSync(NodePath.join(root, "scripts"));
+ NodeFS.copyFileSync(
+ NodePath.join(import.meta.dirname, "windows-publication-signing.ts"),
+ NodePath.join(root, "scripts/windows-publication-signing.ts"),
+ );
+ NodeFS.writeFileSync(
+ NodePath.join(root, "release-assets/scient-release-handoff.json"),
+ JSON.stringify({
+ version,
+ windows:
+ channel === "beta"
+ ? { signed: false }
+ : { signed: true, publisherName: "ScientFactory Ltd" },
+ }),
+ );
NodeFS.writeFileSync(
NodePath.join(root, "release-assets/fixture.txt"),
"immutable fixture",
@@ -122,17 +138,32 @@ const path = require('node:path');
const args = process.argv.slice(2);
fs.appendFileSync(process.env.CALLS, JSON.stringify(args) + '\\n');
if (args[0] === 'api') {
- if (args[1].endsWith('/releases/latest')) process.stdout.write(process.env.RELEASE_CHANNEL === 'beta' ? (process.env.FAILURE === 'stable-advanced' ? 'v0.6.23' : 'v0.6.22') : process.env.RELEASE_TAG);
- else if (args[1] === 'repos/ScientFactory/scient-desktop-beta') process.stdout.write('main');
+ const endpoint = args.find((arg) => arg.startsWith('repos/'));
+ if (endpoint?.endsWith('/releases/latest')) {
+ const tag = process.env.RELEASE_CHANNEL === 'beta'
+ ? (process.env.FAILURE === 'stable-advanced' ? 'v0.6.23' : 'v0.6.22')
+ : process.env.RELEASE_TAG;
+ process.stdout.write(args.includes('--jq') ? tag : JSON.stringify({ tag_name: 'v0.6.22' }));
+ } else if (endpoint === 'repos/ScientFactory/scient-desktop-beta/releases?per_page=100') {
+ process.stdout.write(JSON.stringify([[]]));
+ } else if (endpoint === 'repos/ScientFactory/scient-desktop-beta') process.stdout.write('main');
else process.exit(2);
} else if (args[0] === 'release' && ['create', 'edit'].includes(args[1])) {
if (args[1] === 'create') fs.writeFileSync(process.env.NOTES_RECEIPT, fs.readFileSync(args[args.indexOf('--notes-file') + 1]));
} else if (args[0] === 'release' && args[1] === 'view') {
process.stdout.write(JSON.stringify({ isDraft: false, isPrerelease: true }));
} else if (args[0] === 'release' && args[1] === 'download') {
- const destination = args[args.indexOf('--dir') + 1];
- for (const name of fs.readdirSync('release-assets')) fs.copyFileSync(path.join('release-assets', name), path.join(destination, name));
- if (process.env.FAILURE === 'corrupt-upload') fs.writeFileSync(path.join(destination, 'fixture.txt'), 'modified upload');
+ if (args.includes('--output')) {
+ process.stdout.write(JSON.stringify({
+ windows: process.env.FAILURE === 'publisher-pinned'
+ ? { signed: true, publisherName: 'ScientFactory Ltd' }
+ : { signed: false },
+ }));
+ } else {
+ const destination = args[args.indexOf('--dir') + 1];
+ for (const name of fs.readdirSync('release-assets')) fs.copyFileSync(path.join('release-assets', name), path.join(destination, name));
+ if (process.env.FAILURE === 'corrupt-upload') fs.writeFileSync(path.join(destination, 'fixture.txt'), 'modified upload');
+ }
} else process.exit(2);
`,
{ mode: 0o700 },
@@ -155,6 +186,7 @@ process.stdout.write(process.env.SOURCE_SHA + '\\trefs/tags/' + process.env.RELE
...process.env,
PATH: `${bin}:${process.env.PATH}`,
GH_TOKEN: "synthetic-unused-token",
+ PIN_READ_TOKEN: "synthetic-read-token",
CALLS: calls,
FAILURE: failure,
NOTES_RECEIPT: NodePath.join(root, "published-notes.md"),
@@ -184,11 +216,11 @@ process.stdout.write(process.env.SOURCE_SHA + '\\trefs/tags/' + process.env.RELE
);
if (failure !== "none") {
expect(result.status).not.toBe(0);
- expect(mutations).toHaveLength(failure === "corrupt-upload" ? 1 : 0);
+ expect(mutations).toHaveLength(failure === "stable-advanced" ? 0 : 1);
expect(mutations.some((args) => args[1] === "edit")).toBe(false);
return;
}
- expect(result.status).toBe(0);
+ expect(result.status, result.stderr.toString()).toBe(0);
expect(mutations).toHaveLength(2);
for (const args of mutations) expect(args[args.indexOf("--repo") + 1]).toBe(repository);
if (channel === "beta") {
diff --git a/scripts/scient-release-machinery.test.ts b/scripts/scient-release-machinery.test.ts
index 2ddea191ab..11dde615c1 100644
--- a/scripts/scient-release-machinery.test.ts
+++ b/scripts/scient-release-machinery.test.ts
@@ -143,23 +143,6 @@ describe("Scient release machinery", () => {
}
});
- it("keeps unsigned Windows publication an explicit exception", () => {
- const workflow = NodeFS.readFileSync(
- NodePath.join(import.meta.dirname, "../.github/workflows/release.yml"),
- "utf8",
- );
- assert.include(workflow, "allow_unsigned_windows:");
- assert.include(
- workflow,
- "allow_unsigned_windows is only valid for an explicitly publishing run.",
- );
- assert.include(
- workflow,
- "Windows publication is unsigned; rerun only with explicit allow_unsigned_windows=true.",
- );
- assert.include(workflow, "Publication requires signed macOS artifacts:");
- });
-
// regression: 20117b09c1 fix(release): use Bash for version alignment on all platforms (#44)
it("pins Bash for every package-version alignment step", () => {
const workflow = NodeFS.readFileSync(
diff --git a/scripts/windows-publication-signing.test.ts b/scripts/windows-publication-signing.test.ts
new file mode 100644
index 0000000000..88a63f269e
--- /dev/null
+++ b/scripts/windows-publication-signing.test.ts
@@ -0,0 +1,349 @@
+// @effect-diagnostics nodeBuiltinImport:off - Rehearse the publication gate with temporary files and a local gh fixture.
+import * as NodeChildProcess from "node:child_process";
+import * as NodeFS from "node:fs";
+import * as NodeOS from "node:os";
+import * as NodePath from "node:path";
+
+import { assert, describe, it } from "@effect/vitest";
+
+import { createScientReleaseHandoff } from "./create-scient-release-handoff.ts";
+import {
+ decideWindowsPublication,
+ pinnedWindowsPublishers,
+ readWindowsSigningAttestation,
+} from "./windows-publication-signing.ts";
+
+const PUBLISHER = "ScientFactory Ltd";
+const signed = { signed: true, publisherName: PUBLISHER } as const;
+const unsigned = { signed: false, publisherName: null } as const;
+
+describe("Windows publication signing gate", () => {
+ it("keeps today's unsigned exception while no installed client pins a publisher", () => {
+ // Handoffs published before the signing record pin nothing.
+ const pinned = pinnedWindowsPublishers([{ version: "0.6.29" }, { windows: { signed: false } }]);
+ assert.deepStrictEqual(pinned, []);
+ assert.throws(
+ () => decideWindowsPublication({ candidate: unsigned, pinned, allowUnsigned: false }),
+ /allow_unsigned_windows=true/,
+ );
+ assert.match(
+ decideWindowsPublication({ candidate: unsigned, pinned, allowUnsigned: true }).warning ?? "",
+ /unsigned Windows installer/,
+ );
+ assert.isNull(
+ decideWindowsPublication({ candidate: signed, pinned, allowUnsigned: false }).warning,
+ );
+ });
+
+ it("refuses an unsigned or differently signed installer once any channel pins a publisher", () => {
+ const pinned = pinnedWindowsPublishers([
+ { windows: { signed: false } },
+ { windows: { signed: true, publisherName: PUBLISHER } },
+ ]);
+ assert.deepStrictEqual(pinned, [PUBLISHER]);
+ assert.throws(
+ () => decideWindowsPublication({ candidate: unsigned, pinned, allowUnsigned: true }),
+ /cannot override/,
+ );
+ assert.throws(
+ () =>
+ decideWindowsPublication({
+ candidate: { signed: true, publisherName: "Someone Else" },
+ pinned,
+ allowUnsigned: true,
+ }),
+ /signed by 'Someone Else'/,
+ );
+ assert.isNull(
+ decideWindowsPublication({ candidate: signed, pinned, allowUnsigned: false }).warning,
+ );
+ const conflictingPins = pinnedWindowsPublishers([
+ { windows: { signed: true, publisherName: PUBLISHER } },
+ { windows: { signed: true, publisherName: "Different Channel Publisher" } },
+ ]);
+ for (const publisherName of conflictingPins) {
+ assert.throws(
+ () =>
+ decideWindowsPublication({
+ candidate: { signed: true, publisherName },
+ pinned: conflictingPins,
+ allowUnsigned: true,
+ }),
+ /would fail their update signature check/,
+ );
+ }
+ });
+
+ it("fails closed on attestations and records it cannot read", () => {
+ assert.throws(
+ () => readWindowsSigningAttestation({ platform: "win", arch: "x64", signed: true }),
+ /must name its publisher/,
+ );
+ assert.throws(
+ () =>
+ readWindowsSigningAttestation({
+ platform: "win",
+ arch: "x64",
+ signed: false,
+ publisherName: PUBLISHER,
+ }),
+ /cannot name a publisher/,
+ );
+ assert.throws(() => pinnedWindowsPublishers([{ windows: { signed: "yes" } }]));
+ });
+
+ it("records the signing attestation in the handoff the next publication reads", () => {
+ const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "scient-windows-handoff-"));
+ try {
+ const assets = NodePath.join(root, "assets");
+ NodeFS.mkdirSync(assets);
+ NodeFS.writeFileSync(NodePath.join(assets, "Scient-0.6.30-x64.exe"), "installer");
+ const attestation = NodePath.join(root, "signing-win-x64.json");
+ const output = NodePath.join(assets, "scient-release-handoff.json");
+ const create = () =>
+ createScientReleaseHandoff([
+ "--assets-dir",
+ assets,
+ "--version",
+ "0.6.30",
+ "--channel",
+ "stable",
+ "--repository",
+ "ScientFactory/scient-desktop",
+ "--source-sha",
+ "a".repeat(40),
+ "--source-tree",
+ "b".repeat(40),
+ "--windows-signing",
+ attestation,
+ "--output",
+ output,
+ ]);
+ const pinnedAfter = (record: object) => {
+ NodeFS.writeFileSync(
+ attestation,
+ JSON.stringify({ platform: "win", arch: "x64", ...record }),
+ );
+ create();
+ return pinnedWindowsPublishers([JSON.parse(NodeFS.readFileSync(output, "utf8"))]);
+ };
+
+ assert.deepStrictEqual(pinnedAfter({ signed: true, publisherName: PUBLISHER }), [PUBLISHER]);
+ assert.deepStrictEqual(pinnedAfter({ signed: false }), []);
+ } finally {
+ NodeFS.rmSync(root, { recursive: true, force: true });
+ }
+ });
+});
+
+const stableRepository = "ScientFactory/scient-desktop";
+const betaRepository = "ScientFactory/scient-desktop-beta";
+const stableTag = "v0.6.30";
+const betaTag = "v0.6.31-beta.20261116.1";
+const betaRelease = { tag_name: betaTag, draft: false };
+
+function publicationFixture() {
+ const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "scient-windows-gate-"));
+ const bin = NodePath.join(root, "bin");
+ const assets = NodePath.join(root, "release-assets");
+ const attestation = NodePath.join(assets, "signing-win-x64.json");
+ const candidateHandoff = NodePath.join(assets, "scient-release-handoff.json");
+ const statePath = NodePath.join(root, "published.json");
+ NodeFS.mkdirSync(bin);
+ NodeFS.mkdirSync(assets);
+ NodeFS.writeFileSync(NodePath.join(assets, "Scient-0.6.31-x64.exe"), "fixture installer");
+ const setCandidate = (
+ windows: { signed: boolean; publisherName?: string },
+ channel: "stable" | "beta" = "stable",
+ ) => {
+ NodeFS.writeFileSync(attestation, JSON.stringify({ platform: "win", arch: "x64", ...windows }));
+ createScientReleaseHandoff([
+ "--assets-dir",
+ assets,
+ "--version",
+ channel === "beta" ? betaTag.slice(1) : "0.6.31",
+ "--channel",
+ channel,
+ "--repository",
+ stableRepository,
+ "--source-sha",
+ "a".repeat(40),
+ "--source-tree",
+ "b".repeat(40),
+ "--windows-signing",
+ attestation,
+ "--output",
+ candidateHandoff,
+ ]);
+ };
+ const setPublished = (state: {
+ betaPages?: unknown;
+ stableHandoff?: unknown;
+ betaHandoff?: unknown;
+ failure?: { key: string; status: number };
+ }) => NodeFS.writeFileSync(statePath, JSON.stringify(state));
+ NodeFS.writeFileSync(
+ NodePath.join(bin, "gh"),
+ `#!/usr/bin/env node
+const fs = require('node:fs');
+const args = process.argv.slice(2);
+const state = JSON.parse(fs.readFileSync(process.env.PUBLISHED_STATE, 'utf8'));
+const endpoint = args.find(arg => arg.startsWith('repos/'));
+let key;
+let result;
+if (args[0] === 'api' && endpoint === 'repos/${stableRepository}/releases/latest') {
+ key = 'stable';
+ result = { tag_name: '${stableTag}' };
+} else if (args[0] === 'api' && args.includes('--paginate') && args.includes('--slurp') && endpoint === 'repos/${betaRepository}/releases?per_page=100') {
+ key = 'beta';
+ result = state.betaPages ?? [[]];
+} else if (args[0] === 'release' && args[1] === 'download' && args[args.indexOf('--output') + 1] === '-') {
+ const repo = args[args.indexOf('--repo') + 1];
+ if (repo === '${stableRepository}' && args[2] === '${stableTag}') {
+ key = 'stable-handoff';
+ result = state.stableHandoff ?? { version: '0.6.30' };
+ } else if (repo === '${betaRepository}' && args[2] === '${betaTag}') {
+ key = 'beta-handoff';
+ result = state.betaHandoff ?? { windows: { signed: true, publisherName: '${PUBLISHER}' } };
+ }
+}
+if (!key) process.exit(2);
+if (state.failure?.key === key) {
+ process.stderr.write('HTTP ' + state.failure.status + '\\n');
+ process.exit(1);
+}
+process.stdout.write(JSON.stringify(result));
+`,
+ { mode: 0o700 },
+ );
+ setCandidate({ signed: false });
+ setPublished({});
+ return {
+ setCandidate,
+ setPublished,
+ candidateHandoff,
+ attestation,
+ run(phase: "assembly" | "publication", allowUnsigned = true) {
+ return NodeChildProcess.spawnSync(
+ process.execPath,
+ [
+ NodePath.join(import.meta.dirname, "windows-publication-signing.ts"),
+ phase === "assembly" ? "--attestation" : "--candidate-handoff",
+ phase === "assembly" ? attestation : candidateHandoff,
+ "--allow-unsigned",
+ String(allowUnsigned),
+ "--stable-repository",
+ stableRepository,
+ "--beta-repository",
+ betaRepository,
+ ],
+ {
+ cwd: root,
+ env: {
+ ...process.env,
+ PATH: `${bin}:${process.env.PATH}`,
+ GH_TOKEN: "local-fixture",
+ PUBLISHED_STATE: statePath,
+ },
+ encoding: "utf8",
+ },
+ );
+ },
+ [Symbol.dispose]() {
+ NodeFS.rmSync(root, { recursive: true, force: true });
+ },
+ };
+}
+
+// oxlint-disable-next-line t3code/no-global-process-runtime -- The gh fixture uses a POSIX shebang, like the Linux publication job.
+describe.skipIf(process.platform === "win32")("Windows publication gate CLI", () => {
+ it("reads the newest Stable and Beta handoffs before deciding", () => {
+ using fixture = publicationFixture();
+ // A full first page of drafts must not hide the newest published Beta on the next page.
+ fixture.setPublished({
+ betaPages: [
+ Array.from({ length: 100 }, () => ({ ...betaRelease, draft: true })),
+ [betaRelease],
+ ],
+ });
+ const pinned = fixture.run("assembly");
+ assert.notStrictEqual(pinned.status, 0);
+ assert.match(pinned.stderr, /pin publisher 'ScientFactory Ltd'/);
+ fixture.setPublished({});
+ const none = fixture.run("assembly");
+ assert.strictEqual(none.status, 0, none.stderr);
+ assert.match(none.stdout, /WARNING: Publishing an unsigned Windows installer/);
+ const unapproved = fixture.run("assembly", false);
+ assert.notStrictEqual(unapproved.status, 0);
+ assert.match(unapproved.stderr, /allow_unsigned_windows=true/);
+ });
+
+ it("rejects the same unsigned candidate when Beta gains a pin while Stable awaits approval", () => {
+ using fixture = publicationFixture();
+ const candidate = NodeFS.readFileSync(fixture.candidateHandoff, "utf8");
+ const assembled = fixture.run("assembly");
+ assert.strictEqual(assembled.status, 0, assembled.stderr);
+ NodeFS.unlinkSync(fixture.attestation);
+ const beforePin = fixture.run("publication");
+ assert.strictEqual(beforePin.status, 0, beforePin.stderr);
+
+ fixture.setPublished({ betaPages: [[betaRelease]] });
+ const afterPin = fixture.run("publication");
+ assert.notStrictEqual(afterPin.status, 0);
+ assert.match(afterPin.stderr, /cannot override/);
+ assert.strictEqual(NodeFS.readFileSync(fixture.candidateHandoff, "utf8"), candidate);
+
+ fixture.setCandidate({ signed: true, publisherName: PUBLISHER });
+ const compatible = fixture.run("publication", false);
+ assert.strictEqual(compatible.status, 0, compatible.stderr);
+ fixture.setCandidate({ signed: true, publisherName: "Someone Else" });
+ const incompatible = fixture.run("publication");
+ assert.notStrictEqual(incompatible.status, 0);
+ assert.match(incompatible.stderr, /signed by 'Someone Else'/);
+ });
+
+ it("also enforces a Stable pin when publishing Beta and refuses missing candidate signing", () => {
+ using fixture = publicationFixture();
+ fixture.setCandidate({ signed: false }, "beta");
+ fixture.setPublished({ stableHandoff: { windows: signed } });
+ const pinned = fixture.run("publication");
+ assert.notStrictEqual(pinned.status, 0);
+ assert.match(pinned.stderr, /pin publisher 'ScientFactory Ltd'/);
+
+ fixture.setPublished({});
+ const handoff = JSON.parse(NodeFS.readFileSync(fixture.candidateHandoff, "utf8"));
+ delete handoff.windows;
+ NodeFS.writeFileSync(fixture.candidateHandoff, JSON.stringify(handoff));
+ const missing = fixture.run("publication");
+ assert.notStrictEqual(missing.status, 0);
+ assert.match(missing.stderr, /candidate handoff must record Windows signing/);
+ });
+
+ it("fails closed on public lookup and download errors, including 404 and 5xx", () => {
+ using fixture = publicationFixture();
+ for (const key of ["stable", "beta", "stable-handoff", "beta-handoff"]) {
+ for (const status of [404, 503]) {
+ fixture.setPublished({ betaPages: [[betaRelease]], failure: { key, status } });
+ const failed = fixture.run("publication");
+ assert.notStrictEqual(failed.status, 0, `${key}: HTTP ${status}`);
+ assert.match(failed.stderr, new RegExp(`HTTP ${status}`));
+ }
+ }
+ });
+
+ it("fails closed on malformed public inventories and Windows signing records", () => {
+ using fixture = publicationFixture();
+ for (const state of [
+ { betaPages: {} },
+ { betaPages: [[{ tag_name: betaTag }]] },
+ { stableHandoff: [] },
+ { stableHandoff: { windows: null } },
+ { betaPages: [[betaRelease]], betaHandoff: { windows: { signed: true } } },
+ ]) {
+ fixture.setPublished(state);
+ const failed = fixture.run("publication");
+ assert.notStrictEqual(failed.status, 0, JSON.stringify(state));
+ assert.match(failed.stderr, /Cannot read|Invalid|must name its publisher/);
+ }
+ });
+});
diff --git a/scripts/windows-publication-signing.ts b/scripts/windows-publication-signing.ts
new file mode 100644
index 0000000000..2fb165ed18
--- /dev/null
+++ b/scripts/windows-publication-signing.ts
@@ -0,0 +1,200 @@
+#!/usr/bin/env node
+// @effect-diagnostics nodeBuiltinImport:off - Standalone release gate reads build attestations and published handoffs before publication.
+
+import * as NodeChildProcess from "node:child_process";
+import * as NodeFS from "node:fs";
+import * as NodeURL from "node:url";
+
+/** What a release records about its Windows installer's Authenticode signature. */
+export interface WindowsSigningRecord {
+ readonly signed: boolean;
+ /** The certificate CN the installed app pins in app-update.yml; null when unsigned. */
+ readonly publisherName: string | null;
+}
+
+function isRecord(value: unknown): value is Record {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/** The Windows build's signing attestation, as the release workflow writes it. */
+export function readWindowsSigningAttestation(value: unknown): WindowsSigningRecord {
+ if (
+ !isRecord(value) ||
+ value.platform !== "win" ||
+ value.arch !== "x64" ||
+ typeof value.signed !== "boolean"
+ ) {
+ throw new Error("Invalid Windows signing attestation.");
+ }
+ const publisherName = value.publisherName;
+ if (value.signed) {
+ if (typeof publisherName !== "string" || publisherName.trim() === "") {
+ throw new Error("A signed Windows attestation must name its publisher.");
+ }
+ return { signed: true, publisherName };
+ }
+ if (publisherName !== undefined) {
+ throw new Error("An unsigned Windows attestation cannot name a publisher.");
+ }
+ return { signed: false, publisherName: null };
+}
+
+/**
+ * Publishers that installed Windows clients already pin, from each channel's newest published
+ * handoff. Once a signed build ships, every later build is signed by the same publisher, so the
+ * newest release of each channel carries the pin. Handoffs older than this record name none.
+ */
+export function pinnedWindowsPublishers(handoffs: ReadonlyArray): ReadonlyArray {
+ const pinned = new Set();
+ for (const handoff of handoffs) {
+ const record = readHandoffWindowsSigning(handoff);
+ if (record === null) continue;
+ if (record.publisherName !== null) pinned.add(record.publisherName);
+ }
+ return [...pinned];
+}
+
+/**
+ * Clients switch between Stable and Beta, so a pin on either binds both, and electron-updater
+ * rejects an installer whose signer misses the pin: no exception may override it. Before any
+ * pin, an unsigned build still needs the explicit unsigned-Windows exception.
+ */
+export function decideWindowsPublication(input: {
+ readonly candidate: WindowsSigningRecord;
+ readonly pinned: ReadonlyArray;
+ readonly allowUnsigned: boolean;
+}): { readonly warning: string | null } {
+ const { candidate, pinned, allowUnsigned } = input;
+ if (pinned.length > 0) {
+ const names = pinned.map((name) => `'${name}'`).join(", ");
+ if (!candidate.signed) {
+ throw new Error(
+ `Installed Windows clients pin publisher ${names}; an unsigned installer would fail their update signature check on Stable and Beta. allow_unsigned_windows cannot override this.`,
+ );
+ }
+ if (pinned.some((publisher) => publisher !== candidate.publisherName)) {
+ throw new Error(
+ `Installed Windows clients pin publisher ${names}; this installer is signed by '${candidate.publisherName}' and would fail their update signature check.`,
+ );
+ }
+ return { warning: null };
+ }
+ if (candidate.signed) return { warning: null };
+ if (!allowUnsigned) {
+ throw new Error(
+ "Windows publication is unsigned; rerun only with explicit allow_unsigned_windows=true.",
+ );
+ }
+ return { warning: "Publishing an unsigned Windows installer by explicit release approval." };
+}
+
+function values(args: ReadonlyArray, flag: string): ReadonlyArray {
+ return args.flatMap((arg, index) => (arg === flag && args[index + 1] ? [args[index + 1]!] : []));
+}
+
+function readJson(path: string): unknown {
+ return JSON.parse(NodeFS.readFileSync(path, "utf8")) as unknown;
+}
+
+function readHandoffWindowsSigning(handoff: unknown): WindowsSigningRecord | null {
+ if (!isRecord(handoff)) throw new Error("Invalid release handoff.");
+ if (handoff.windows === undefined) return null;
+ if (!isRecord(handoff.windows)) throw new Error("Invalid Windows signing record in handoff.");
+ return readWindowsSigningAttestation({ platform: "win", arch: "x64", ...handoff.windows });
+}
+
+function githubJson(args: ReadonlyArray): unknown {
+ return JSON.parse(
+ NodeChildProcess.execFileSync("gh", args, {
+ encoding: "utf8",
+ stdio: ["ignore", "pipe", "pipe"],
+ maxBuffer: 16 * 1024 * 1024,
+ }),
+ ) as unknown;
+}
+
+function newestPublishedHandoffs(stableRepository: string, betaRepository: string): unknown[] {
+ const stable = githubJson(["api", `repos/${stableRepository}/releases/latest`]);
+ if (!isRecord(stable) || typeof stable.tag_name !== "string" || !stable.tag_name) {
+ throw new Error("Cannot identify the newest published Stable release.");
+ }
+ const pages = githubJson([
+ "api",
+ "--paginate",
+ "--slurp",
+ `repos/${betaRepository}/releases?per_page=100`,
+ ]);
+ if (!Array.isArray(pages) || !pages.every(Array.isArray)) {
+ throw new Error("Cannot read published Beta releases.");
+ }
+ const releases: unknown[] = pages.flat();
+ let betaTag: string | null = null;
+ for (const release of releases) {
+ if (
+ !isRecord(release) ||
+ typeof release.draft !== "boolean" ||
+ typeof release.tag_name !== "string"
+ ) {
+ throw new Error("Invalid published Beta release record.");
+ }
+ if (!release.draft && release.tag_name.includes("-beta.")) {
+ betaTag = release.tag_name;
+ break;
+ }
+ }
+ const download = (repository: string, tag: string) =>
+ githubJson([
+ "release",
+ "download",
+ tag,
+ "--repo",
+ repository,
+ "--pattern",
+ "scient-release-handoff.json",
+ "--output",
+ "-",
+ ]);
+ return [
+ download(stableRepository, stable.tag_name),
+ ...(betaTag === null ? [] : [download(betaRepository, betaTag)]),
+ ];
+}
+
+export function runWindowsPublicationGate(args: ReadonlyArray): string | null {
+ const [attestation] = values(args, "--attestation");
+ const [candidateHandoff] = values(args, "--candidate-handoff");
+ if (Boolean(attestation) === Boolean(candidateHandoff)) {
+ throw new Error("Exactly one of --attestation or --candidate-handoff is required.");
+ }
+ const candidate = attestation
+ ? readWindowsSigningAttestation(readJson(attestation))
+ : readHandoffWindowsSigning(readJson(candidateHandoff!));
+ if (candidate === null) throw new Error("The candidate handoff must record Windows signing.");
+ const [stableRepository] = values(args, "--stable-repository");
+ const [betaRepository] = values(args, "--beta-repository");
+ if (!stableRepository || !betaRepository) {
+ throw new Error("--stable-repository and --beta-repository are required.");
+ }
+ const allowUnsigned = values(args, "--allow-unsigned")[0] === "true";
+ const { warning } = decideWindowsPublication({
+ candidate,
+ pinned: pinnedWindowsPublishers(newestPublishedHandoffs(stableRepository, betaRepository)),
+ allowUnsigned,
+ });
+ return warning;
+}
+
+// Real paths, so an indirect invocation still runs the gate rather than silently passing it.
+const isMain = process.argv[1]
+ ? NodeFS.realpathSync(process.argv[1]) ===
+ NodeFS.realpathSync(NodeURL.fileURLToPath(import.meta.url))
+ : false;
+if (isMain) {
+ try {
+ const warning = runWindowsPublicationGate(process.argv.slice(2));
+ if (warning !== null) process.stdout.write(`WARNING: ${warning}\n`);
+ } catch (error) {
+ process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+ process.exitCode = 1;
+ }
+}