From 58091da30e54b7a8a88d4b1cb5eb89924cad0d35 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Mon, 16 Nov 2026 23:08:49 +0200 Subject: [PATCH 01/12] fix(provider-runtime): finish slow runtime downloads and recover requested runtime changes Managed runtime downloads failed after ten minutes while bytes still arrived, showed zero progress while resuming, and a user's install, update or repair that failed transiently or was cut off by a restart stayed failed until clicked again. Failed lifecycle events carried no download diagnostics, and every lifecycle event carried failureClass. - No overall transfer deadline for any managed runtime download; the 30-second stall bound, cancellation, size limits, digest checks and resume stay. Applies to provider runtimes and the compute uv/Ark downloads that share the downloader. - Progress starts from the kept prefix on every try and retry wait; the progress filter (now shared with Codex) reports a reset to zero. - Durable recovery of user-requested runtime changes: the runtime manager records the reviewed plan per runtime and settles it on the outcome. A serialized loop retries it in the background at a jittered 5 min, 30 min, 2 h, then every 4 h, after revalidating that the instance is enabled and still offers the same plan. Success, cancel, remove, disable or another user change clears it. Retryability is typed: exhausted 408/429/5xx and lost connections retry; 501, certificate, checksum, disk and permission failures do not. Scient Agent's own provisioning is unchanged. - Download facts are captured after every failed try: source kind (scient_mirror/github/vendor), variant, inner tries, progress, active-transfer throughput, network kind (stall split from transport timeouts) and why a compressed copy was given up. The operation contract only gains optional fields, so older clients still decode it; a stall keeps failureClass "timeout" on the wire and analytics reports "stalled". - failureClass only on provider.lifecycle.failed, in both the mapper and the contract normalizer; conformance fixture regenerated. Co-Authored-By: Claude Opus 5.5 --- .../provider/ScientProviderInstanceSeams.ts | 6 - apps/server/src/scient/ScientServerLayers.ts | 8 +- .../CodexManagedRuntimeActions.ts | 16 +- .../ManagedProviderRuntimeActions.ts | 49 ++-- .../ProviderRuntimeManager.test.ts | 84 ++++-- .../ProviderRuntimeManager.ts | 35 ++- .../ProviderRuntimeRecovery.test.ts | 168 ++++++++++++ .../ProviderRuntimeRecovery.ts | 92 +++++++ .../ProviderRuntimeRequests.ts | 181 +++++++++++++ .../ProviderLifecycleAnalytics.test.ts | 32 ++- .../telemetry/ProviderLifecycleAnalytics.ts | 47 +++- docs/internals/product-analytics.md | 15 +- docs/user/providers.md | 8 + packages/contracts/src/providerLifecycle.ts | 25 +- .../fixtures/contract-v11.json | 74 ++++-- packages/scient-analytics/src/conformance.ts | 8 +- packages/scient-analytics/src/contract.ts | 48 +++- .../src/managedProviderRuntime.ts | 68 +++-- .../src/managedScientAgentDownload.test.ts | 89 +++++-- .../src/runtimeDownload.test.ts | 140 ++++++---- .../src/runtimeFiles.ts | 247 +++++++++++------- 21 files changed, 1157 insertions(+), 283 deletions(-) create mode 100644 apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts create mode 100644 apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts create mode 100644 apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts diff --git a/apps/server/src/provider/ScientProviderInstanceSeams.ts b/apps/server/src/provider/ScientProviderInstanceSeams.ts index 56124b9bd7..fa523994a9 100644 --- a/apps/server/src/provider/ScientProviderInstanceSeams.ts +++ b/apps/server/src/provider/ScientProviderInstanceSeams.ts @@ -330,11 +330,6 @@ export interface ProviderManagedRuntimeProgress { readonly download?: ProviderRuntimeOperation["download"] | undefined; } -export interface ProviderManagedRuntimeRunOptions { - /** Scient started it on its own: the download waits as long as data keeps arriving. */ - readonly background?: boolean | undefined; -} - export interface ProviderManagedRuntimeActions { readonly getSummary: Effect.Effect; readonly plan: ( @@ -350,7 +345,6 @@ export interface ProviderManagedRuntimeActions { catalogRevision: string, report: (progress: ProviderManagedRuntimeProgress) => Effect.Effect, awaitActivationWindow?: Effect.Effect, - options?: ProviderManagedRuntimeRunOptions, ) => Effect.Effect; /** * Whether a fresh check would select a different runtime than the one this diff --git a/apps/server/src/scient/ScientServerLayers.ts b/apps/server/src/scient/ScientServerLayers.ts index 75864d94c7..e6018b550d 100644 --- a/apps/server/src/scient/ScientServerLayers.ts +++ b/apps/server/src/scient/ScientServerLayers.ts @@ -15,6 +15,8 @@ import * as ProviderConnectionManager from "./providerLifecycle/ProviderConnecti import * as ProviderLifecycleCoordinator from "./providerLifecycle/ProviderLifecycleCoordinator.ts"; import * as ProviderActivity from "./providerLifecycle/ProviderActivity.ts"; import * as ProviderRuntimeManager from "./providerLifecycle/ProviderRuntimeManager.ts"; +import * as ProviderRuntimeRecovery from "./providerLifecycle/ProviderRuntimeRecovery.ts"; +import * as ProviderRuntimeRequests from "./providerLifecycle/ProviderRuntimeRequests.ts"; import * as ManagedRuntimeCatalogReconciler from "./providerLifecycle/ManagedRuntimeCatalogReconciler.ts"; import * as ScientAgentBackgroundRuntime from "./providerLifecycle/ScientAgentBackgroundRuntime.ts"; import * as ToolDiscovery from "./compute/ToolDiscovery.ts"; @@ -89,7 +91,10 @@ export const WorkspaceBindingResolverLayerLive = WorkspaceBindingResolver.layer. ); /** Connection, runtime and catalog managers over one lifecycle coordinator. */ -export const ScientProviderLifecycleLive = ScientAgentBackgroundRuntime.layer.pipe( +export const ScientProviderLifecycleLive = Layer.mergeAll( + ScientAgentBackgroundRuntime.layer, + ProviderRuntimeRecovery.layer, +).pipe( Layer.provideMerge( Layer.mergeAll( ProviderConnectionManager.layer, @@ -97,6 +102,7 @@ export const ScientProviderLifecycleLive = ScientAgentBackgroundRuntime.layer.pi ManagedRuntimeCatalogReconciler.layer, ), ), + Layer.provideMerge(ProviderRuntimeRequests.layer), Layer.provideMerge(ProviderLifecycleCoordinator.layer), Layer.provideMerge(ProviderActivity.layer), ); diff --git a/apps/server/src/scient/providerLifecycle/CodexManagedRuntimeActions.ts b/apps/server/src/scient/providerLifecycle/CodexManagedRuntimeActions.ts index 3cfc4d5021..a44c448a1a 100644 --- a/apps/server/src/scient/providerLifecycle/CodexManagedRuntimeActions.ts +++ b/apps/server/src/scient/providerLifecycle/CodexManagedRuntimeActions.ts @@ -46,6 +46,7 @@ import { import { detectTargetSafely, isManagedRuntimeDowngrade, + makeManagedRuntimeProgressFilter, managedRuntimeDowngradeMessage, managedRuntimeInstallationFailureMessage, managedRuntimeRetryMessage, @@ -105,6 +106,7 @@ function mapProgress(progress: ManagedCodexRuntimeProgress): ProviderManagedRunt ? {} : { downloadedBytes: progress.downloadedBytes }), ...(progress.totalBytes === undefined ? {} : { totalBytes: progress.totalBytes }), + ...(progress.download ? { download: progress.download } : {}), }; } @@ -773,8 +775,7 @@ export const makeCodexManagedRuntimeResolution = Effect.fn("CodexManagedRuntime. if (!actionArtifact) { return yield* runtimeError("No qualified Codex artifact is available."); } - let lastStatus: ManagedCodexRuntimeProgress["stage"] | undefined; - let lastReportedBytes = 0; + const worthReporting = makeManagedRuntimeProgressFilter(); yield* Effect.tryPromise({ try: (signal) => runtime.install({ @@ -806,16 +807,7 @@ export const makeCodexManagedRuntimeResolution = Effect.fn("CodexManagedRuntime. } }, onProgress: (progress) => { - const stageChanged = progress.stage !== lastStatus; - const downloadedBytes = progress.downloadedBytes ?? 0; - const downloadAdvanced = downloadedBytes - lastReportedBytes >= 1024 * 1024; - const downloadFinished = - progress.totalBytes !== undefined && downloadedBytes === progress.totalBytes; - if (!stageChanged && !downloadAdvanced && !downloadFinished && !progress.retry) - return; - lastStatus = progress.stage; - lastReportedBytes = downloadedBytes; - runFork(report(mapProgress(progress))); + if (worthReporting(progress)) runFork(report(mapProgress(progress))); }, }), catch: (cause) => diff --git a/apps/server/src/scient/providerLifecycle/ManagedProviderRuntimeActions.ts b/apps/server/src/scient/providerLifecycle/ManagedProviderRuntimeActions.ts index 8680bc5919..04ea30ca3d 100644 --- a/apps/server/src/scient/providerLifecycle/ManagedProviderRuntimeActions.ts +++ b/apps/server/src/scient/providerLifecycle/ManagedProviderRuntimeActions.ts @@ -86,6 +86,31 @@ export function managedRuntimeRetryMessage( return `Couldn't download ${providerName} (${retry.reason}). Trying again, ${retry.retry} of ${retry.retries}.`; } +/** + * Passes a progress report on when it says something new: another stage, a retry, download facts, + * or a move of at least 1 MiB either way (a server that ignores a range starts over from zero). + */ +export function makeManagedRuntimeProgressFilter(): ( + progress: ManagedProviderRuntimeProgress, +) => boolean { + let lastStage: ManagedProviderRuntimeProgress["stage"] | undefined; + let lastBytes = 0; + return (progress) => { + const bytes = progress.downloadedBytes ?? 0; + const report = + progress.stage !== lastStage || + Math.abs(bytes - lastBytes) >= 1024 * 1024 || + (progress.totalBytes !== undefined && bytes === progress.totalBytes) || + progress.retry !== undefined || + progress.download !== undefined; + if (report) { + lastStage = progress.stage; + lastBytes = bytes; + } + return report; + }; +} + /** What `binary --version` prints when it exits 0 within five seconds. */ export const readConfiguredRuntimeVersion = Effect.fn( "ManagedProviderRuntimeActions.readConfiguredRuntimeVersion", @@ -626,7 +651,6 @@ export const makeManagedProviderRuntimeResolution = Effect.fn( catalogRevision: Parameters[1], report: Parameters[2], awaitActivationWindow: NonNullable[3]>, - options: Parameters[4], ) { const prepared = yield* prepareAction(action); const planned = prepared.plan; @@ -655,34 +679,18 @@ export const makeManagedProviderRuntimeResolution = Effect.fn( if (!actionArtifact) { return yield* runtimeError(`No qualified ${providerName} artifact is available.`); } - let lastStatus: ManagedProviderRuntimeProgress["stage"] | undefined; - let lastReportedBytes = 0; + const worthReporting = makeManagedRuntimeProgressFilter(); yield* Effect.tryPromise({ try: (signal) => runtime.install({ artifact: actionArtifact, signal, - ...(options?.background ? { downloadTimeLimitMs: null } : {}), ...(input.qualify ? { qualify: input.qualify } : {}), beforeActivate: (activationSignal) => runPromise(awaitActivationWindow, { signal: activationSignal }), onProgress: (progress) => { - const stageChanged = progress.stage !== lastStatus; - const downloadedBytes = progress.downloadedBytes ?? 0; - const downloadAdvanced = downloadedBytes - lastReportedBytes >= 1024 * 1024; - const downloadFinished = - progress.totalBytes !== undefined && downloadedBytes === progress.totalBytes; + if (!worthReporting(progress)) return; const retry = progress.retry; - if ( - !stageChanged && - !downloadAdvanced && - !downloadFinished && - !retry && - !progress.download - ) - return; - lastStatus = progress.stage; - lastReportedBytes = downloadedBytes; runFork( report({ status: progress.stage, @@ -711,8 +719,7 @@ export const makeManagedProviderRuntimeResolution = Effect.fn( catalogRevision, report, awaitActivationWindow = Effect.void, - options, - ) => runEffect(action, catalogRevision, report, awaitActivationWindow, options); + ) => runEffect(action, catalogRevision, report, awaitActivationWindow); const summary = yield* getSummary.pipe( Effect.orElseSucceed(() => ({ diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts index 2da3574de7..38e1bfdb7b 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts @@ -48,6 +48,7 @@ import { make, ProviderRuntimeManager, } from "./ProviderRuntimeManager.ts"; +import { ProviderRuntimeRequests } from "./ProviderRuntimeRequests.ts"; const CODEX = ProviderDriverKind.make("codex"); const INSTANCE = ProviderInstanceId.make("codex"); @@ -224,6 +225,13 @@ function makeHarness( }); const managerScope = yield* Scope.make(); yield* Effect.addFinalizer(() => Scope.close(managerScope, Exit.void)); + const requestLogRef = yield* SubscriptionRef.make>([]); + const requestsLayer = Layer.mock(ProviderRuntimeRequests)({ + requested: (input) => + SubscriptionRef.update(requestLogRef, (log) => [...log, `requested:${input.action}`]), + settled: (input) => + SubscriptionRef.update(requestLogRef, (log) => [...log, `settled:${input.outcome}`]), + }); const providerSessionsLayer = Layer.mock(ProviderSessionManagerV2)({ closeInstance: (instanceId) => Ref.update(stopCountRef, (count) => count + 1).pipe( @@ -240,6 +248,7 @@ function makeHarness( Layer.succeed(ProviderLifecycleCoordinator, trackedCoordinator), Layer.succeed(ProviderActivity, activity), providerSessionsLayer, + requestsLayer, NodeServices.layer, NodeCrypto.layer, ), @@ -254,7 +263,12 @@ function makeHarness( Effect.provideService(ProviderLifecycleCoordinator, trackedCoordinator), Effect.provideService(ProviderActivity, activity), Effect.provide( - Layer.mergeAll(providerSessionsLayer, NodeServices.layer, NodeCrypto.layer), + Layer.mergeAll( + providerSessionsLayer, + requestsLayer, + NodeServices.layer, + NodeCrypto.layer, + ), ), Scope.provide(managerScope), ); @@ -268,6 +282,7 @@ function makeHarness( stopCountRef, coordinator: trackedCoordinator, lifecycleReleaseCountRef, + requestLogRef, closeManager: Scope.close(managerScope, Exit.void), }; }); @@ -988,6 +1003,42 @@ describe("ProviderRuntimeManager", () => { }), ); + // A later try fixes a server that stayed busy, not one that does not have the file. + it.effect.each([ + { + caseTitle: "keeps the user's request after HTTP 429 on every try", + cause: new ManagedRuntimeFileError("HTTP 429 on all 5 tries.", { + failureClass: "http_4xx", + retryable: true, + }), + outcome: "retryable", + }, + { + caseTitle: "drops the user's request after HTTP 404", + cause: new ManagedRuntimeFileError("HTTP 404.", { failureClass: "http_4xx" }), + outcome: "failed", + }, + ])("$caseTitle", ({ cause, outcome }) => + Effect.gen(function* () { + const actions: ProviderManagedRuntimeActions = { + getSummary: Effect.succeed(systemRuntime), + plan: () => Effect.succeed(installPlan()), + run: () => + Effect.fail(new ProviderConnectionActionError({ message: "Install failed.", cause })), + }; + const { manager, requestLogRef } = yield* makeHarness(actions, [systemProvider]); + yield* manager.start({ + instanceId: INSTANCE, + action: "install", + catalogRevision: "reviewed:1", + }); + assert.deepStrictEqual(yield* awaitState(requestLogRef, (log) => log.length === 2), [ + "requested:install", + `settled:${outcome}`, + ]); + }), + ); + it.effect("does not mutate a shared runtime when active sessions cannot stop", () => Effect.gen(function* () { const runCount = yield* Ref.make(0); @@ -1054,24 +1105,19 @@ describe("ProviderRuntimeManager", () => { }), ); - it.effect("runs a background install without a download limit and says how it ended", () => + it.effect("runs a background install, says how it ended, and records no user request", () => Effect.gen(function* () { - const backgroundRuns = yield* Ref.make>([]); const actions: ProviderManagedRuntimeActions = { getSummary: Effect.succeed(missingRuntime), plan: () => Effect.succeed(installPlan()), - run: (_action, _revision, report, _await, options) => - Ref.update(backgroundRuns, (runs) => [...runs, options?.background]).pipe( - Effect.andThen( - report({ - status: "verifying", - message: "Verifying.", - download: { resumedBytes: 40, source: "fallback", compressed: true }, - }), - ), - ), + run: (_action, _revision, report) => + report({ + status: "verifying", + message: "Verifying.", + download: { resumedBytes: 40, source: "fallback", compressed: true }, + }), }; - const { manager, providersRef } = yield* makeHarness(actions); + const { manager, providersRef, requestLogRef } = yield* makeHarness(actions); const outcome = yield* manager.startInBackground({ instanceId: INSTANCE, action: "install", @@ -1079,7 +1125,7 @@ describe("ProviderRuntimeManager", () => { attempt: 2, }); assert.strictEqual(outcome, "succeeded"); - assert.deepStrictEqual(yield* Ref.get(backgroundRuns), [true]); + assert.deepStrictEqual(yield* SubscriptionRef.get(requestLogRef), ["settled:succeeded"]); const operation = (yield* SubscriptionRef.get(providersRef))[0]?.connection?.runtime ?.operation; assert.strictEqual(operation?.status, "succeeded"); @@ -1098,11 +1144,15 @@ describe("ProviderRuntimeManager", () => { () => Effect.gen(function* () { const never = yield* Deferred.make(); + const runs = yield* Ref.make(0); const actions: ProviderManagedRuntimeActions = { getSummary: Effect.succeed(missingRuntime), plan: () => Effect.succeed(installPlan()), - run: (_action, _revision, _report, _await, options) => - options?.background ? Deferred.await(never) : Effect.void, + // Only the user's change, the second run, finishes; background runs wait. + run: () => + Ref.getAndUpdate(runs, (count) => count + 1).pipe( + Effect.flatMap((count) => (count === 1 ? Effect.void : Deferred.await(never))), + ), }; const other: ServerProvider = { ...provider, diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts index b61f492336..399e9d95bc 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts @@ -12,7 +12,10 @@ import { type ProviderRuntimeSummary, type ServerProvider, } from "@t3tools/contracts"; -import { managedRuntimeFailureClass } from "@scientfactory/provider-runtime"; +import { + managedRuntimeFailureClass, + managedRuntimeFailureRetryable, +} from "@scientfactory/provider-runtime"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; @@ -37,6 +40,7 @@ import { ProviderSessionManagerV2 } from "../../orchestration-v2/ProviderSession import { ProviderActivity } from "./ProviderActivity.ts"; import { ProviderConnectionActionError } from "./ProviderConnectionActions.ts"; import { ProviderLifecycleCoordinator } from "./ProviderLifecycleCoordinator.ts"; +import { ProviderRuntimeRequests } from "./ProviderRuntimeRequests.ts"; export interface ProviderRuntimeManagerShape { readonly plan: ( @@ -61,7 +65,7 @@ export interface ProviderRuntimeManagerShape { readonly reselect: (instanceId: ProviderInstanceId) => Effect.Effect; /** * Starts a change the user did not ask for and waits for it to end. It is marked - * `background`, its download has no time limit, and any start by the user stops it. + * `background`, and any start by the user stops it. */ readonly startInBackground: ( input: ProviderRuntimeStartInput & { readonly attempt: number }, @@ -188,6 +192,7 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { const providerSessions = yield* ProviderSessionManagerV2; const lifecycleCoordinator = yield* ProviderLifecycleCoordinator; const activity = yield* ProviderActivity; + const requests = yield* ProviderRuntimeRequests; const random = yield* SecureRandom.SecureRandom; const activeRef = yield* Ref.make>( new Map(), @@ -545,6 +550,14 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { message: "Another account is already changing this shared provider runtime.", }); } + if (background === undefined) { + yield* requests.requested({ + instanceId: input.instanceId, + driver: target.provider, + action: input.action, + catalogRevision: input.catalogRevision, + }); + } const cleanupStartupExit = (exit: Exit.Exit): Effect.Effect => Exit.isSuccess(exit) @@ -652,7 +665,6 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { input.catalogRevision, publishProgress, awaitActivationWindow, - { background: background !== undefined }, ), ).pipe(Effect.tap(() => Ref.set(committedRef, true))), ); @@ -723,6 +735,18 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { }) .pipe( Effect.catchCause(() => publishUnexpectedFailure), + Effect.andThen( + requests.settled({ + instanceId: input.instanceId, + driver: target.provider, + outcome: + completion._tag === "Success" + ? "succeeded" + : managedRuntimeFailureRetryable(completion.failure, lastStage) + ? "retryable" + : "failed", + }), + ), Effect.andThen( Deferred.succeed( outcome, @@ -862,6 +886,11 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { publishCancelled(input.instanceId, claim), "cancelled", ); + yield* requests.settled({ + instanceId: input.instanceId, + driver: target.provider, + outcome: "cancelled", + }); return { providers }; }, ); diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts new file mode 100644 index 0000000000..dbd08b8f43 --- /dev/null +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts @@ -0,0 +1,168 @@ +import * as NodeOS from "node:os"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, it } from "@effect/vitest"; +import { + ProviderDriverKind, + ProviderInstanceId, + type ProviderRuntimeSummary, + type ServerProvider, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; + +import * as ServerConfig from "../../config.ts"; +import { ProviderRegistry } from "../../provider/ProviderRegistry.ts"; +import { ProviderRuntimeManager, type ProviderRuntimeOutcome } from "./ProviderRuntimeManager.ts"; +import { make as makeRecovery } from "./ProviderRuntimeRecovery.ts"; +import { make as makeRequests, ProviderRuntimeRequests } from "./ProviderRuntimeRequests.ts"; + +const INSTANCE = ProviderInstanceId.make("codex"); +const CODEX = ProviderDriverKind.make("codex"); +const MINUTE = 60_000; + +const runtime: ProviderRuntimeSummary = { + source: "missing", + supportTier: "fully_assisted", + target: "darwin-arm64", + actions: ["install"], + managedVersion: null, + previousManagedVersion: null, + operation: null, + message: "Codex setup is available.", +}; + +const codex: ServerProvider = { + instanceId: INSTANCE, + driver: CODEX, + enabled: true, + installed: false, + version: null, + status: "error", + auth: { status: "unknown" }, + checkedAt: "2026-11-16T00:00:00.000Z", + models: [], + slashCommands: [], + skills: [], + connection: { methods: [], canDisconnect: false, operation: null, runtime }, +}; + +const plan = (catalogRevision: string) => ({ + instanceId: INSTANCE, + action: "install" as const, + target: "darwin-arm64", + version: "0.147.0", + downloadBytes: 100, + sourceLabel: "Official OpenAI release", + catalogRevision, + message: "Install reviewed Codex.", +}); + +const request = { + instanceId: INSTANCE, + driver: CODEX, + action: "install" as const, + catalogRevision: "reviewed:1", +}; + +describe("ProviderRuntimeRequests", () => { + it.effect("keeps the user's request across a restart without making it due at once", () => + Effect.gen(function* () { + const before = yield* makeRequests(); + yield* before.requested(request); + const after = yield* makeRequests(); + const [kept] = yield* after.list; + assert.deepInclude(kept, { ...request, retries: 0 }); + // A restart while the user's own download runs waits as long as a failure would. + assert.isAtLeast(kept!.dueAt, (yield* Clock.currentTimeMillis) + 4 * MINUTE); + yield* after.requested({ ...request, action: "remove" }); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-requests-" }).pipe( + Layer.provideMerge(NodeServices.layer), + ), + ), + ), + ); +}); + +describe("ProviderRuntimeRecovery", () => { + // Requests stay in memory here, so the loop's only waits are the clock and these calls. + const harness = (catalogRevision: string) => + Effect.gen(function* () { + const starts = yield* Ref.make>([]); + const requests = yield* makeRequests().pipe(Effect.provide(FileSystem.layerNoop({}))); + yield* requests.requested(request); + // The user's own try failed in a way a later try can fix. + yield* requests.settled({ instanceId: INSTANCE, driver: CODEX, outcome: "retryable" }); + yield* makeRecovery().pipe( + Effect.provideService(ProviderRuntimeRequests, requests), + Effect.provide( + Layer.mergeAll( + Layer.mock(ProviderRegistry)({ + getProviders: Effect.succeed([codex]), + streamChanges: Stream.never, + }), + Layer.mock(ProviderRuntimeManager)({ + plan: () => Effect.succeed(plan(catalogRevision)), + // As the manager does: a failure that may pass keeps the request. + startInBackground: (input) => + Effect.gen(function* () { + const at = yield* Clock.currentTimeMillis; + yield* Ref.update(starts, (all) => [...all, { attempt: input.attempt, at }]); + yield* requests.settled({ + instanceId: input.instanceId, + driver: CODEX, + outcome: "retryable", + }); + return "failed" as ProviderRuntimeOutcome; + }), + }), + ), + ), + ); + const runFor = (minutes: number) => + Effect.forEach(Array.from({ length: minutes }), () => TestClock.adjust("1 minute"), { + discard: true, + }); + return { starts, requests, runFor }; + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-recovery-" }).pipe( + Layer.provideMerge(NodeServices.layer), + ), + ), + ); + + it.effect("tries again about 5 minutes, then 30 minutes, then 2 hours after each failure", () => + Effect.gen(function* () { + const { starts, runFor } = yield* harness("reviewed:1"); + yield* runFor(4 * 60); + const tries = yield* Ref.get(starts); + assert.deepStrictEqual( + tries.map((entry) => entry.attempt), + [1, 2, 3], + ); + const [first, second, third] = tries.map((entry) => entry.at / MINUTE); + // Jittered by a fifth either way, then picked up at the next one-minute check. + assert.isTrue(first! >= 4 && first! <= 7, `first try at ${first}`); + assert.isTrue(second! - first! >= 24 && second! - first! <= 37, `second at ${second}`); + assert.isTrue(third! - second! >= 96 && third! - second! <= 145, `third at ${third}`); + }), + ); + + it.effect("drops the request when the plan changed instead of installing another one", () => + Effect.gen(function* () { + const { starts, requests, runFor } = yield* harness("reviewed:2"); + yield* runFor(10); + assert.deepStrictEqual(yield* Ref.get(starts), []); + assert.deepStrictEqual(yield* requests.list, []); + }), + ); +}); diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts new file mode 100644 index 0000000000..df759791a2 --- /dev/null +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts @@ -0,0 +1,92 @@ +import type { ProviderRuntimeOperation } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; + +import { ProviderRegistry } from "../../provider/ProviderRegistry.ts"; +import { ProviderRuntimeManager } from "./ProviderRuntimeManager.ts"; +import { nextTryAt, ProviderRuntimeRequests } from "./ProviderRuntimeRequests.ts"; + +const START_DELAY = "60 seconds"; +const POLL_INTERVAL = "1 minute"; +/** Stopped for the user's own change, or another change was running. */ +const BUSY_DELAY_MS = 5 * 60_000; + +const isRunning = (operation: ProviderRuntimeOperation | null | undefined) => + operation != null && + operation.finishedAt === null && + operation.status !== "failed" && + operation.status !== "cancelled" && + operation.status !== "succeeded"; + +/** + * Tries again, one at a time, the runtime changes the user asked for that a restart or a temporary + * failure stopped. Each try first checks that the instance is still enabled and offers the same + * reviewed plan; otherwise the request is dropped. + */ +export const make = Effect.fn("ProviderRuntimeRecovery.make")(function* () { + const requests = yield* ProviderRuntimeRequests; + const manager = yield* ProviderRuntimeManager; + const registry = yield* ProviderRegistry; + + const tryDue = Effect.fn("ProviderRuntimeRecovery.tryDue")(function* () { + const now = yield* Clock.currentTimeMillis; + const request = (yield* requests.list) + .filter((candidate) => candidate.dueAt <= now) + .toSorted((left, right) => left.dueAt - right.dueAt)[0]; + if (!request) return; + const providers = yield* registry.getProviders; + if (providers.some((provider) => isRunning(provider.connection?.runtime?.operation))) return; + const provider = providers.find((candidate) => candidate.instanceId === request.instanceId); + const planned = + provider?.enabled === true && provider.driver === request.driver + ? yield* manager + .plan({ instanceId: request.instanceId, action: request.action }) + .pipe(Effect.option) + : Option.none(); + if (Option.isNone(planned) || planned.value.catalogRevision !== request.catalogRevision) { + return yield* requests.update(request.instanceId, () => undefined); + } + const attempt = request.retries + 1; + const lease = yield* nextTryAt(attempt); + yield* requests.update(request.instanceId, (current) => ({ + ...current, + retries: attempt, + dueAt: lease, + })); + // The manager settles the request when the change ends; only a try that never ran is undone. + const outcome = yield* manager + .startInBackground({ + instanceId: request.instanceId, + action: request.action, + catalogRevision: request.catalogRevision, + attempt, + }) + .pipe(Effect.orElseSucceed(() => "busy" as const)); + if (outcome === "interrupted") return yield* Effect.interrupt; + if (outcome === "yielded" || outcome === "busy") { + const later = (yield* Clock.currentTimeMillis) + BUSY_DELAY_MS; + yield* requests.update(request.instanceId, (current) => + current.retries === attempt + ? { ...current, retries: request.retries, dueAt: later } + : current, + ); + } + }); + + yield* tryDue().pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Effect.logError("provider runtime recovery failed", { cause: Cause.pretty(cause) }), + ), + Effect.andThen(Effect.sleep(POLL_INTERVAL)), + Effect.forever, + Effect.delay(START_DELAY), + Effect.forkScoped, + ); +}); + +export const layer = Layer.effectDiscard(make()); diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts new file mode 100644 index 0000000000..aa26f9cbc8 --- /dev/null +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts @@ -0,0 +1,181 @@ +import { + ProviderDriverKind, + ProviderInstanceId, + type ProviderManagedRuntimeAction, +} from "@t3tools/contracts"; +import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Random from "effect/Random"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; + +import { ServerConfig } from "../../config.ts"; + +/** After the user's own try, then after each try Scient makes; the cooldown repeats after that. */ +const RETRY_DELAYS_MS = [5 * 60_000, 30 * 60_000, 2 * 60 * 60_000] as const; +const COOLDOWN_MS = 4 * 60 * 60_000; +const ACP_REGISTRY = ProviderDriverKind.make("acpRegistry"); + +/** An install, update or repair the user asked for that has not succeeded yet. */ +const RuntimeRequest = Schema.Struct({ + instanceId: ProviderInstanceId, + driver: ProviderDriverKind, + action: Schema.Literals(["install", "update", "repair"]), + /** The plan the user reviewed; a different plan is never installed in its place. */ + catalogRevision: Schema.String, + /** Tries Scient made on its own since the user's. */ + retries: Schema.Number, + /** No try starts before this (epoch ms). Set when a try starts too, so a crash cannot hurry the next. */ + dueAt: Schema.Number, +}); +export type RuntimeRequest = typeof RuntimeRequest.Type; +const RequestsJson = Schema.fromJsonString(Schema.Array(RuntimeRequest)); +const decodeRequests = Schema.decodeEffect(RequestsJson); +const encodeRequests = Schema.encodeEffect(RequestsJson); + +/** How a change of a runtime ended; `retryable` failed in a way a later try may not. */ +export type RuntimeChangeOutcome = "succeeded" | "cancelled" | "failed" | "retryable"; + +/** When the try after `retries` of Scient's own may start, spread so installs don't retry together. */ +export const nextTryAt = Effect.fnUntraced(function* (retries: number) { + const delay = RETRY_DELAYS_MS[retries] ?? COOLDOWN_MS; + const now = yield* Clock.currentTimeMillis; + return now + Math.round(delay * (0.8 + (yield* Random.next) * 0.4)); +}); + +// Instances of a driver share one managed runtime; each ACP Registry agent has its own. +const sameRuntime = ( + request: RuntimeRequest, + change: { readonly instanceId: ProviderInstanceId; readonly driver: ProviderDriverKind }, +) => + request.driver === change.driver && + (change.driver !== ACP_REGISTRY || request.instanceId === change.instanceId); + +/** + * Runtime changes the user asked for, kept across restarts until one succeeds, the user cancels or + * removes it, or chooses again. The runtime manager records them; recovery tries them again. + */ +export class ProviderRuntimeRequests extends Context.Service< + ProviderRuntimeRequests, + { + /** The user started a change: it replaces any request for the same runtime; Remove only clears. */ + readonly requested: (input: { + readonly instanceId: ProviderInstanceId; + readonly driver: ProviderDriverKind; + readonly action: ProviderManagedRuntimeAction; + readonly catalogRevision: string; + }) => Effect.Effect; + readonly settled: (input: { + readonly instanceId: ProviderInstanceId; + readonly driver: ProviderDriverKind; + readonly outcome: RuntimeChangeOutcome; + }) => Effect.Effect; + readonly list: Effect.Effect>; + /** Changes the instance's request if there still is one; `undefined` drops it. */ + readonly update: ( + instanceId: ProviderInstanceId, + change: (request: RuntimeRequest) => RuntimeRequest | undefined, + ) => Effect.Effect; + } +>()("t3/scient/providerLifecycle/ProviderRuntimeRequests") {} + +export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { + const config = yield* ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const platform = yield* HostProcess.Platform; + // Kept outside the runtime folders, which Remove deletes. + const filePath = path.join(config.baseDir, "provider-runtimes", ".background", "requests.json"); + const requestsRef = yield* Ref.make>( + yield* fs.readFileString(filePath).pipe( + Effect.flatMap(decodeRequests), + Effect.orElseSucceed((): ReadonlyArray => []), + ), + ); + const lock = yield* Semaphore.make(1); + + const change = ( + next: (requests: ReadonlyArray) => Effect.Effect>, + ) => + lock.withPermits(1)( + Effect.gen(function* () { + const updated = yield* next(yield* Ref.get(requestsRef)); + yield* Ref.set(requestsRef, updated); + yield* encodeRequests(updated).pipe( + Effect.flatMap((contents) => + writeFileStringAtomically({ filePath, contents, mode: 0o600 }), + ), + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(HostProcess.Platform, platform), + Effect.ignoreCause({ log: true }), + ); + }), + ); + + const updateEach = ( + requests: ReadonlyArray, + instanceId: ProviderInstanceId, + next: (request: RuntimeRequest) => RuntimeRequest | undefined, + ) => + requests.flatMap((request) => { + if (request.instanceId !== instanceId) return [request]; + const updated = next(request); + return updated ? [updated] : []; + }); + + const requested = Effect.fn("ProviderRuntimeRequests.requested")(function* ( + input: Parameters[0], + ) { + const dueAt = yield* nextTryAt(0); + yield* change((requests) => { + const others = requests.filter((request) => !sameRuntime(request, input)); + if (input.action === "remove") return Effect.succeed(others); + const { instanceId, driver, action, catalogRevision } = input; + return Effect.succeed([ + ...others, + { instanceId, driver, action, catalogRevision, retries: 0, dueAt }, + ]); + }); + }); + + const settled = Effect.fn("ProviderRuntimeRequests.settled")(function* ( + input: Parameters[0], + ) { + yield* change((requests) => + Effect.gen(function* () { + switch (input.outcome) { + case "succeeded": + return requests.filter((request) => !sameRuntime(request, input)); + case "cancelled": + case "failed": + return updateEach(requests, input.instanceId, () => undefined); + case "retryable": { + const request = requests.find((candidate) => candidate.instanceId === input.instanceId); + if (!request) return requests; + const dueAt = yield* nextTryAt(request.retries); + return updateEach(requests, input.instanceId, (current) => ({ ...current, dueAt })); + } + } + }), + ); + }); + + const update = Effect.fn("ProviderRuntimeRequests.update")(function* ( + instanceId: ProviderInstanceId, + next: (request: RuntimeRequest) => RuntimeRequest | undefined, + ) { + yield* change((requests) => Effect.succeed(updateEach(requests, instanceId, next))); + }); + + return ProviderRuntimeRequests.of({ requested, settled, list: Ref.get(requestsRef), update }); +}); + +export const layer = Layer.effect(ProviderRuntimeRequests, make()); diff --git a/apps/server/src/telemetry/ProviderLifecycleAnalytics.test.ts b/apps/server/src/telemetry/ProviderLifecycleAnalytics.test.ts index 046fe306db..acba665a54 100644 --- a/apps/server/src/telemetry/ProviderLifecycleAnalytics.test.ts +++ b/apps/server/src/telemetry/ProviderLifecycleAnalytics.test.ts @@ -147,31 +147,51 @@ describe("Provider lifecycle analytics", () => { ]); }); - it("says a background change started on its own and how its download went", () => { + it("says a background change started on its own and how far its download got", () => { const mapper = createProviderLifecycleAnalyticsMapper(); mapper.observe([provider()]); const background = { ...operation("downloading"), background: true, attempt: 2 }; - expect(mapper.observe([provider(background)])).toMatchObject([ - { name: "provider.lifecycle.started", properties: { trigger: "background", attempt: 2 } }, - ]); + const [started] = mapper.observe([provider(background)]); + expect(started).toMatchObject({ + name: "provider.lifecycle.started", + properties: { trigger: "background", attempt: 2 }, + }); + expect(started!.properties).not.toHaveProperty("failureClass"); expect( mapper.observe([ provider({ ...background, status: "failed", finishedAt: "2026-08-31T00:00:12.000Z", - download: { resumedBytes: 41_000_000, source: "fallback", compressed: true }, + failureClass: "timeout", + download: { + resumedBytes: 41_000_000, + source: "fallback", + compressed: true, + sourceKind: "github", + tries: 5, + progressPercent: 37, + bytesPerSecond: 410_000, + networkKind: "stalled", + }, }), ]), ).toMatchObject([ { name: "provider.lifecycle.failed", properties: { + // Older clients decode only `timeout`; the stall is told apart here. + failureClass: "stalled", trigger: "background", attempt: 2, resumedBytes: 41_000_000, downloadSource: "fallback", - compressed: true, + downloadVariant: "compressed", + sourceKind: "github", + downloadAttempts: 5, + progressPercent: 37, + bytesPerSecond: 410_000, + networkKind: "stalled", }, }, ]); diff --git a/apps/server/src/telemetry/ProviderLifecycleAnalytics.ts b/apps/server/src/telemetry/ProviderLifecycleAnalytics.ts index 5138ecde4a..ad3c2c82ac 100644 --- a/apps/server/src/telemetry/ProviderLifecycleAnalytics.ts +++ b/apps/server/src/telemetry/ProviderLifecycleAnalytics.ts @@ -1,4 +1,4 @@ -import type { ServerProvider } from "@t3tools/contracts"; +import type { ProviderRuntimeOperation, ServerProvider } from "@t3tools/contracts"; const MAX_INSTANCES = 1_000; @@ -30,6 +30,37 @@ interface Event { readonly properties: Readonly>; } +/** How the download went: closed values and numbers the contract buckets, never hosts or paths. */ +function downloadDetails( + download: NonNullable, +): Readonly> { + return { + resumedBytes: download.resumedBytes, + downloadSource: download.source, + downloadVariant: download.compressed ? "compressed" : "raw", + ...(download.sourceKind === undefined ? {} : { sourceKind: download.sourceKind }), + ...(download.tries === undefined ? {} : { downloadAttempts: download.tries }), + ...(download.progressPercent === undefined + ? {} + : { progressPercent: download.progressPercent }), + ...(download.bytesPerSecond === undefined ? {} : { bytesPerSecond: download.bytesPerSecond }), + ...(download.networkKind === undefined ? {} : { networkKind: download.networkKind }), + ...(download.compressedFailure === undefined + ? {} + : { compressedFailure: download.compressedFailure }), + }; +} + +/** Older clients cannot decode a new operation failure class, so a stall travels as `timeout`. */ +function runtimeFailureClass( + failureClass: NonNullable, + download: ProviderRuntimeOperation["download"], +): string { + return failureClass === "timeout" && download?.networkKind === "stalled" + ? "stalled" + : failureClass; +} + function snapshot(provider: ServerProvider): Snapshot { const runtime = provider.connection?.runtime; const operation = runtime?.operation; @@ -53,17 +84,13 @@ function snapshot(provider: ServerProvider): Snapshot { : operation.status === "failed" || operation.status === "cancelled" ? operation.status : null, - ...(operation.failureClass === undefined ? {} : { failureClass: operation.failureClass }), + ...(operation.failureClass === undefined + ? {} + : { failureClass: runtimeFailureClass(operation.failureClass, operation.download) }), details: { trigger: operation.background ? "background" : "user", ...(operation.attempt === undefined ? {} : { attempt: operation.attempt }), - ...(operation.download === undefined - ? {} - : { - resumedBytes: operation.download.resumedBytes, - downloadSource: operation.download.source, - compressed: operation.download.compressed, - }), + ...(operation.download === undefined ? {} : downloadDetails(operation.download)), }, } : null, @@ -126,7 +153,7 @@ function lifecycleEvent( durationMs, // The UI message is deliberately not inspected: it can contain paths, // credentials, provider output, or URLs. The owner sets the typed class. - failureClass: next.outcome === "failed" ? (next.failureClass ?? "unknown") : "unknown", + ...(next.outcome === "failed" ? { failureClass: next.failureClass ?? "unknown" } : {}), ...next.details, }, }; diff --git a/docs/internals/product-analytics.md b/docs/internals/product-analytics.md index 91e0821a00..c0cb908bbc 100644 --- a/docs/internals/product-analytics.md +++ b/docs/internals/product-analytics.md @@ -309,8 +309,19 @@ count user turns. Scient Agent uses the existing installation observations, installed-state changes, lifecycle operations, submissions, terminal outcomes and token usage. `provider.lifecycle.*` says who started a runtime change (`trigger`: `user` or -`background`), the background `attempt`, and for a finished download -`resumedBucket`, `downloadSource` (`primary` is Scient's copy) and `compressed`. +`background`) and the background `attempt`: Scient Agent's own provisioning, or Scient trying again a +change the user asked for (from 1). For a finished download, or as far as a failed one got, it adds +`resumedBucket`, `downloadSource` (`primary` is Scient's copy where one exists), `sourceKind` +(`scient_mirror`, `github` or `vendor`), `downloadVariant` (`compressed` or `raw`), +`downloadAttempts` (the downloader's own tries, not `attempt`), `progressBucket` (how much of the +file the last try had), `throughputBucket` (bytes received per second of request time in Mbit/s +buckets, kept bytes and waits between tries excluded), `networkKind` (how the last try's connection +failed: `dns`, `refused`, `reset`, `unreachable`, `connect_timeout`, `stalled`, `cut_off` or +`other`) and `compressedFailure` (why the compressed copy was given up for the raw one). Only +`provider.lifecycle.failed` carries `failureClass`; it reports `stalled` (30 seconds without data) +apart from `timeout`, while the operation itself still says `timeout` so older clients decode it. +No URL, host, path or error text is sent. Builds up to 0.6.29 sent `compressed` instead of +`downloadVariant`, and `failureClass: unknown` on the other lifecycle events. `provider.runtime.first_pick` is sent once per installation when the user first picks Scient Agent in the composer: `state` is `ready`, `in_progress` (with `percentBucket`), `failed` or `not_started`. Its `ready` share is the headline diff --git a/docs/user/providers.md b/docs/user/providers.md index 6bb7320544..9586677568 100644 --- a/docs/user/providers.md +++ b/docs/user/providers.md @@ -111,6 +111,14 @@ says when Scient last checked its qualified-release catalog. If the latest check says so with the reason and the time of the last successful check, and Scient keeps using the catalog it already has. When automatic update checks are off, the line says that instead. +A download has no overall time limit: on a slow connection it keeps going while data arrives, and +continues from what it already has after a dropped connection. If an install, update, or repair you +started fails in a way that usually passes (a lost connection, a busy server) or Scient closes +during it, Scient tries the same reviewed release again on its own about 5 minutes, 30 minutes, and +2 hours later, then every 4 hours. It stops once it succeeds, or when you cancel, remove the runtime, +disable the provider, or start another change to it; a release that changed in the meantime is not +installed in its place. + Scient downloads, verifies, and tests a new runtime while you keep working. Switching to it restarts that provider, so while one of its turns or background tasks (subagents, workflows, monitors) is running, the button shows **Waiting**; your conversations continue on the next message. Cancel a diff --git a/packages/contracts/src/providerLifecycle.ts b/packages/contracts/src/providerLifecycle.ts index 0fbf2582e9..33cf8a71ab 100644 --- a/packages/contracts/src/providerLifecycle.ts +++ b/packages/contracts/src/providerLifecycle.ts @@ -227,12 +227,35 @@ export const ProviderRuntimeOperation = Schema.Struct({ background: Schema.optionalKey(Schema.Boolean), /** Which background try this is, from 1, for analytics. */ attempt: Schema.optionalKey(PositiveInt), - /** How the finished download went, for analytics. */ + /** + * How the download went, for analytics: once complete, or as far as a failed try got. Fields + * after `compressed` are optional so older clients keep decoding the operation. + */ download: Schema.optionalKey( Schema.Struct({ resumedBytes: NonNegativeInt, source: Schema.Literals(["primary", "fallback"]), compressed: Schema.Boolean, + sourceKind: ForwardCompatibleOptional(Schema.Literals(["scient_mirror", "github", "vendor"])), + tries: Schema.optionalKey(PositiveInt), + progressPercent: Schema.optionalKey(NonNegativeInt), + /** Received bytes per second of request time; kept bytes and waits between tries excluded. */ + bytesPerSecond: Schema.optionalKey(NonNegativeInt), + /** How the latest try's connection failed; `stalled` is 30 seconds without data. */ + networkKind: ForwardCompatibleOptional( + Schema.Literals([ + "dns", + "refused", + "reset", + "unreachable", + "connect_timeout", + "stalled", + "cut_off", + "other", + ]), + ), + /** Why the zstd copy failed, when the raw binary was downloaded after it. */ + compressedFailure: ForwardCompatibleOptional(ProviderRuntimeFailureClass), }), ), }); diff --git a/packages/scient-analytics/fixtures/contract-v11.json b/packages/scient-analytics/fixtures/contract-v11.json index d676a7873e..37b6196d1c 100644 --- a/packages/scient-analytics/fixtures/contract-v11.json +++ b/packages/scient-analytics/fixtures/contract-v11.json @@ -869,7 +869,6 @@ "action": "unknown", "runtimeSource": "unknown", "stage": "unknown", - "failureClass": "unknown", "durationBucket": "unknown", "trigger": "user", "appVersion": "0.6.8", @@ -887,13 +886,18 @@ "action": "repair", "runtimeSource": "scient_managed", "stage": "downloading", - "failureClass": "permission", "durationBucket": "5-15s", "trigger": "agent", "attempt": 2, "downloadSource": "fallback", - "compressed": true, + "sourceKind": "github", + "downloadVariant": "raw", + "downloadAttempts": 5, + "networkKind": "stalled", + "compressedFailure": "checksum", "resumedBucket": "10-50mb", + "progressBucket": "25-49", + "throughputBucket": "1-5mbps", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -909,7 +913,6 @@ "action": "unknown", "runtimeSource": "unknown", "stage": "unknown", - "failureClass": "unknown", "durationBucket": "unknown", "trigger": "user", "appVersion": "0.6.8", @@ -927,13 +930,18 @@ "action": "repair", "runtimeSource": "scient_managed", "stage": "downloading", - "failureClass": "permission", "durationBucket": "5-15s", "trigger": "agent", "attempt": 2, "downloadSource": "fallback", - "compressed": true, + "sourceKind": "github", + "downloadVariant": "raw", + "downloadAttempts": 5, + "networkKind": "stalled", + "compressedFailure": "checksum", "resumedBucket": "10-50mb", + "progressBucket": "25-49", + "throughputBucket": "1-5mbps", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -972,8 +980,14 @@ "trigger": "agent", "attempt": 2, "downloadSource": "fallback", - "compressed": true, + "sourceKind": "github", + "downloadVariant": "raw", + "downloadAttempts": 5, + "networkKind": "stalled", + "compressedFailure": "checksum", "resumedBucket": "10-50mb", + "progressBucket": "25-49", + "throughputBucket": "1-5mbps", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -989,7 +1003,6 @@ "action": "unknown", "runtimeSource": "unknown", "stage": "unknown", - "failureClass": "unknown", "durationBucket": "unknown", "trigger": "user", "appVersion": "0.6.8", @@ -1007,13 +1020,18 @@ "action": "repair", "runtimeSource": "scient_managed", "stage": "downloading", - "failureClass": "permission", "durationBucket": "5-15s", "trigger": "agent", "attempt": 2, "downloadSource": "fallback", - "compressed": true, + "sourceKind": "github", + "downloadVariant": "raw", + "downloadAttempts": 5, + "networkKind": "stalled", + "compressedFailure": "checksum", "resumedBucket": "10-50mb", + "progressBucket": "25-49", + "throughputBucket": "1-5mbps", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2500,13 +2518,18 @@ "action": "repair", "runtimeSource": "scient_managed", "stage": "downloading", - "failureClass": "permission", "durationBucket": "5-15s", "trigger": "agent", "attempt": 2, "downloadSource": "fallback", - "compressed": true, + "sourceKind": "github", + "downloadVariant": "raw", + "downloadAttempts": 5, + "networkKind": "stalled", + "compressedFailure": "checksum", "resumedBucket": "10-50mb", + "progressBucket": "25-49", + "throughputBucket": "1-5mbps", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2609,13 +2632,18 @@ "action": "repair", "runtimeSource": "scient_managed", "stage": "downloading", - "failureClass": "permission", "durationBucket": "5-15s", "trigger": "agent", "attempt": 2, "downloadSource": "fallback", - "compressed": true, + "sourceKind": "github", + "downloadVariant": "raw", + "downloadAttempts": 5, + "networkKind": "stalled", + "compressedFailure": "checksum", "resumedBucket": "10-50mb", + "progressBucket": "25-49", + "throughputBucket": "1-5mbps", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2718,13 +2746,18 @@ "action": "repair", "runtimeSource": "scient_managed", "stage": "downloading", - "failureClass": "permission", "durationBucket": "5-15s", "trigger": "agent", "attempt": 2, "downloadSource": "fallback", - "compressed": true, + "sourceKind": "github", + "downloadVariant": "raw", + "downloadAttempts": 5, + "networkKind": "stalled", + "compressedFailure": "checksum", "resumedBucket": "10-50mb", + "progressBucket": "25-49", + "throughputBucket": "1-5mbps", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2827,13 +2860,18 @@ "action": "repair", "runtimeSource": "scient_managed", "stage": "downloading", - "failureClass": "permission", "durationBucket": "5-15s", "trigger": "agent", "attempt": 2, "downloadSource": "fallback", - "compressed": true, + "sourceKind": "github", + "downloadVariant": "raw", + "downloadAttempts": 5, + "networkKind": "stalled", + "compressedFailure": "checksum", "resumedBucket": "10-50mb", + "progressBucket": "25-49", + "throughputBucket": "1-5mbps", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" diff --git a/packages/scient-analytics/src/conformance.ts b/packages/scient-analytics/src/conformance.ts index 7062eacb46..36ad6b9f1f 100644 --- a/packages/scient-analytics/src/conformance.ts +++ b/packages/scient-analytics/src/conformance.ts @@ -71,7 +71,13 @@ const representativeProperties = { failureClass: "permission", attempt: 2, downloadSource: "fallback", - compressed: true, + sourceKind: "github", + downloadVariant: "raw", + downloadAttempts: 5, + networkKind: "stalled", + compressedFailure: "checksum", + progressPercent: 37, + bytesPerSecond: 410_000, resumedBytes: 41_000_000, percent: 64, component: "server", diff --git a/packages/scient-analytics/src/contract.ts b/packages/scient-analytics/src/contract.ts index 1bc3d33d9c..5bfcf4deb4 100644 --- a/packages/scient-analytics/src/contract.ts +++ b/packages/scient-analytics/src/contract.ts @@ -258,6 +258,18 @@ const LIFECYCLE_ACTIONS = new Set([ ]); const LIFECYCLE_TRIGGERS = new Set(["user", "background"]); const DOWNLOAD_SOURCES = new Set(["primary", "fallback"]); +const DOWNLOAD_SOURCE_KINDS = new Set(["scient_mirror", "github", "vendor"]); +const DOWNLOAD_VARIANTS = new Set(["compressed", "raw"]); +const DOWNLOAD_NETWORK_KINDS = new Set([ + "dns", + "refused", + "reset", + "unreachable", + "connect_timeout", + "stalled", + "cut_off", + "other", +]); const FIRST_PICK_STATES = new Set(["ready", "in_progress", "failed", "not_started"]); const LIFECYCLE_STAGES = new Set([ "preparing", @@ -302,6 +314,7 @@ const FAILURE_CLASSES = new Set([ "extract", "cancelled", "busy", + "stalled", "other", // LaTeX builds "compile_error", @@ -844,6 +857,17 @@ function byteBucket(value: unknown): string { return "over-100mb"; } +/** Bytes per second, bucketed in megabits per second as connections are quoted. */ +function throughputBucket(value: unknown): string { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return "unknown"; + const megabits = (value * 8) / 1_000_000; + if (megabits < 1) return "under-1mbps"; + if (megabits < 5) return "1-5mbps"; + if (megabits < 20) return "5-20mbps"; + if (megabits < 100) return "20-100mbps"; + return "over-100mbps"; +} + function percentBucket(value: unknown): string { if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return "unknown"; if (value < 25) return "0-24"; @@ -1039,29 +1063,43 @@ function normalizeEvent( case "provider.lifecycle.started": case "provider.lifecycle.completed": case "provider.lifecycle.failed": - case "provider.lifecycle.cancelled": + case "provider.lifecycle.cancelled": { + const failed = name === "provider.lifecycle.failed"; + // Read on every lifecycle event so a stray class never passes through on the others. + const failureClass = normalizedEnum(property(input, "failureClass"), FAILURE_CLASSES); return { name, - privacyLevel: name === "provider.lifecycle.failed" ? "essential" : "product", - priority: name === "provider.lifecycle.failed" ? "critical" : "core", + privacyLevel: failed ? "essential" : "product", + priority: failed ? "critical" : "core", properties: { provider, action: normalizedEnum(property(input, "action"), LIFECYCLE_ACTIONS), runtimeSource: normalizedEnum(property(input, "source"), RUNTIME_SOURCES), stage: normalizedEnum(property(input, "stage"), LIFECYCLE_STAGES), - failureClass: normalizedEnum(property(input, "failureClass"), FAILURE_CLASSES), + ...(failed ? { failureClass } : {}), durationBucket: durationBucket(property(input, "durationMs")), trigger: normalizedEnum(property(input, "trigger"), LIFECYCLE_TRIGGERS, "user"), ...optionalProperties(input, { attempt: "count", downloadSource: DOWNLOAD_SOURCES, - compressed: "boolean", + sourceKind: DOWNLOAD_SOURCE_KINDS, + downloadVariant: DOWNLOAD_VARIANTS, + downloadAttempts: "count", + networkKind: DOWNLOAD_NETWORK_KINDS, + compressedFailure: FAILURE_CLASSES, }), ...(property(input, "resumedBytes") === undefined ? {} : { resumedBucket: byteBucket(property(input, "resumedBytes")) }), + ...(property(input, "progressPercent") === undefined + ? {} + : { progressBucket: percentBucket(property(input, "progressPercent")) }), + ...(property(input, "bytesPerSecond") === undefined + ? {} + : { throughputBucket: throughputBucket(property(input, "bytesPerSecond")) }), }, }; + } case "provider.runtime.first_pick": return { name, diff --git a/packages/scient-provider-runtime/src/managedProviderRuntime.ts b/packages/scient-provider-runtime/src/managedProviderRuntime.ts index b6829a4f89..e03c92df30 100644 --- a/packages/scient-provider-runtime/src/managedProviderRuntime.ts +++ b/packages/scient-provider-runtime/src/managedProviderRuntime.ts @@ -15,6 +15,8 @@ import { downloadManagedRuntime, type ManagedRuntimeDownloadResult, type ManagedRuntimeDownloadRetry, + type ManagedRuntimeFailureClass, + managedRuntimeFailureClass, materializeManagedRuntimeArtifact, resolveManagedRuntimeArtifactPath, unpackManagedRuntimeZstd, @@ -45,14 +47,26 @@ export interface ManagedProviderRuntimeProgress { readonly totalBytes?: number | undefined; /** Set while the download waits to try again after a temporary failure. */ readonly retry?: ManagedRuntimeDownloadRetry | undefined; - /** Set once the download is complete, on the `verifying` stage. */ + /** How the download went: after each failed try, and once complete on the `verifying` stage. */ readonly download?: ManagedRuntimeDownloadFacts | undefined; } -/** How a finished download went, for analytics. */ +/** Who served a download: Scient's own copy, GitHub, or the provider's own host. */ +export type ManagedRuntimeSourceKind = "scient_mirror" | "github" | "vendor"; + +/** How a download went, for analytics. */ export interface ManagedRuntimeDownloadFacts extends ManagedRuntimeDownloadResult { /** The zstd copy, not the raw binary, was downloaded. */ readonly compressed: boolean; + /** Who served the latest try. */ + readonly sourceKind: ManagedRuntimeSourceKind; + /** Why the zstd copy failed, when the raw binary was downloaded after it. */ + readonly compressedFailure?: ManagedRuntimeFailureClass | undefined; +} + +function hostSourceKind(url: string): ManagedRuntimeSourceKind { + const host = new URL(url).hostname.toLowerCase(); + return host === "github.com" || host.endsWith(".githubusercontent.com") ? "github" : "vendor"; } export interface ManagedProviderRuntimeStateV1 { @@ -116,8 +130,6 @@ export interface ManagedProviderRuntimeInstallInput { readonly beforeActivate?: (signal: AbortSignal) => Promise; /** Told why the compressed copy failed, before the raw binary is downloaded; a throw stops the install. */ readonly onCompressedFallback?: (cause: unknown) => void; - /** Per download copy; `null` waits as long as data keeps arriving. Ten minutes by default. */ - readonly downloadTimeLimitMs?: number | null | undefined; } interface ManagedProviderRuntimeActivation { @@ -683,11 +695,11 @@ export class ManagedProviderRuntime { readonly signal: AbortSignal; readonly onProgress: ManagedProviderRuntimeInstallInput["onProgress"]; readonly onCompressedFallback: ManagedProviderRuntimeInstallInput["onCompressedFallback"]; - readonly timeLimitMs: ManagedProviderRuntimeInstallInput["downloadTimeLimitMs"]; }): Promise { const { artifact, archivePath, signal, onProgress } = input; const compressed = artifact.compressed; const unpack = this.#dependencies.unpackZstd; + let compressedFailure: ManagedRuntimeFailureClass | undefined; const fetchVerified = async (copy: { readonly url: string; readonly mirrorUrl?: string | undefined; @@ -696,7 +708,18 @@ export class ManagedProviderRuntime { readonly destination: string; readonly compressed: boolean; }) => { - onProgress?.({ stage: "downloading", downloadedBytes: 0, totalBytes: copy.size }); + const describe = (download: ManagedRuntimeDownloadResult): ManagedRuntimeDownloadFacts => ({ + ...download, + compressed: copy.compressed, + sourceKind: + download.source === "fallback" || !copy.mirrorUrl + ? hostSourceKind(copy.url) + : "scient_mirror", + ...(compressedFailure ? { compressedFailure } : {}), + }); + let downloadedBytes = 0; + // The downloader reports the bytes an earlier try kept as soon as it has checked them. + onProgress?.({ stage: "downloading", totalBytes: copy.size }); const result = await this.#dependencies.download({ ...(copy.mirrorUrl ? { @@ -708,18 +731,23 @@ export class ManagedProviderRuntime { checksum: copy.checksum, destination: copy.destination, partialPath: this.#partialDownloadPath(copy.checksum), - ...(input.timeLimitMs === undefined ? {} : { timeLimitMs: input.timeLimitMs }), expectedSize: copy.size, signal, - onProgress: (downloadedBytes, totalBytes) => - onProgress?.({ stage: "downloading", downloadedBytes, totalBytes }), + onProgress: (bytes, totalBytes) => { + downloadedBytes = bytes; + onProgress?.({ stage: "downloading", downloadedBytes: bytes, totalBytes }); + }, onRetry: (retry) => - onProgress?.({ stage: "downloading", downloadedBytes: 0, totalBytes: copy.size, retry }), - }); - onProgress?.({ - stage: "verifying", - ...(result ? { download: { ...result, compressed: copy.compressed } } : {}), + onProgress?.({ stage: "downloading", downloadedBytes, totalBytes: copy.size, retry }), + onFailedTry: (download) => + onProgress?.({ + stage: "downloading", + downloadedBytes, + totalBytes: copy.size, + download: describe(download), + }), }); + onProgress?.({ stage: "verifying", ...(result ? { download: describe(result) } : {}) }); }; if (compressed && unpack) { const compressedPath = `${archivePath}.zst`; @@ -737,6 +765,7 @@ export class ManagedProviderRuntime { } catch (cause) { if (signal.aborted) throw cause; await NodeFSP.rm(archivePath, { force: true }); + compressedFailure = managedRuntimeFailureClass(cause); input.onCompressedFallback?.(cause); } finally { await NodeFSP.rm(compressedPath, { force: true }); @@ -756,15 +785,7 @@ export class ManagedProviderRuntime { async #installOwned( input: ManagedProviderRuntimeInstallInput, ): Promise { - const { - artifact, - signal, - onProgress, - qualify, - beforeActivate, - onCompressedFallback, - downloadTimeLimitMs, - } = input; + const { artifact, signal, onProgress, qualify, beforeActivate, onCompressedFallback } = input; await NodeFSP.mkdir(this.#stagingDir, { recursive: true, mode: 0o700 }); // This process launches what it installs. Pruning needs the mutation lock held here. await this.#lease(artifact.version); @@ -789,7 +810,6 @@ export class ManagedProviderRuntime { signal, onProgress, onCompressedFallback, - timeLimitMs: downloadTimeLimitMs, }); onProgress?.({ stage: "installing" }); const stagedExecutable = await this.#dependencies.materialize({ diff --git a/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts b/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts index 0e8c18aa26..9d3143ce74 100644 --- a/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts +++ b/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts @@ -8,6 +8,7 @@ import * as NodeZlib from "node:zlib"; import { afterEach, describe, expect, it, vi } from "vite-plus/test"; +import type { ManagedProviderRuntimeProgress } from "./managedProviderRuntime.ts"; import { hydrateManagedRuntimeArtifact } from "./managedRuntimeArtifact.ts"; import { ManagedScientAgentRuntime } from "./managedScientAgentRuntime.ts"; import { resolveScientAgentArtifactPolicy } from "./scientAgentManifest.ts"; @@ -110,6 +111,41 @@ it("recovers the Scient download and still runs smoke, admission, and qualificat expect(await NodeFSP.readFile(otherState, "utf8")).toBe("keep"); }); +// regression: progress fell back to zero while a resumed download waited or started again. +it("reports progress from the bytes kept by an earlier try", async () => { + const { root } = await installedFixture(); + // Cut off after 5 bytes on Scient's copy and after 10 on GitHub, then whole on Scient's. + const ends = [5, 10, bytes.length]; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: URL, init: RequestInit) => { + const range = (init.headers as Record | undefined)?.range; + const from = Number(/bytes=(\d+)-/u.exec(range ?? "")?.[1] ?? 0); + const chunk = new Uint8Array(bytes.subarray(from, ends.shift())); + return from === 0 + ? new Response(chunk, { headers: { "content-length": String(bytes.length) } }) + : new Response(chunk, { + status: 206, + headers: { "content-range": `bytes ${from}-${bytes.length - 1}/${bytes.length}` }, + }); + }), + ); + const downloaded: number[] = []; + const runtime = new ManagedScientAgentRuntime(root, { smoke: async () => undefined }); + await runtime.install({ + artifact: artifact("0.2.0"), + signal: new AbortController().signal, + onProgress: (report) => { + if (report.stage === "downloading" && report.downloadedBytes !== undefined) + downloaded.push(report.downloadedBytes); + }, + }); + expect(downloaded.at(-1)).toBe(bytes.length); + expect(downloaded.slice(downloaded.indexOf(5))).toEqual( + downloaded.slice(downloaded.indexOf(5)).toSorted((left, right) => left - right), + ); +}); + it.each(["checksum", "smoke", "admission", "qualification", "cancellation"] as const)( "preserves the installed release when %s refuses an update after a transient response", async (failure) => { @@ -258,29 +294,46 @@ describe("installing from the compressed copy", () => { }); it.each([ - { failure: "a damaged copy on both servers", catalogCopy: packed, served: Buffer.from("junk") }, + { + failure: "a damaged copy on both servers", + catalogCopy: packed, + served: Buffer.from("junk"), + compressedFailure: "size_mismatch", + }, { failure: "a copy that unpacks to other bytes", catalogCopy: NodeZlib.zstdCompressSync(Buffer.from("other bytes")), served: NodeZlib.zstdCompressSync(Buffer.from("other bytes")), + compressedFailure: "other", }, - ])("falls back to the binary after $failure", async ({ catalogCopy, served }) => { - const { root } = await installedFixture(); - const urls: string[] = []; - vi.stubGlobal("fetch", serve(served, urls)); - const onCompressedFallback = vi.fn(); - const runtime = new ManagedScientAgentRuntime(root, { smoke: async () => undefined }); - await runtime.install({ - artifact: compressedArtifact("0.2.0", catalogCopy), - signal: new AbortController().signal, - onCompressedFallback, - }); - expect(onCompressedFallback).toHaveBeenCalledOnce(); - expect(urls.at(-1)).toBe( - "https://downloads.scientfactory.com/scient-agent/v0.2.0/scient-agent-darwin-arm64", - ); - await expectInstalled(runtime, root, "0.2.0"); - }); + ])( + "falls back to the binary after $failure", + async ({ catalogCopy, served, compressedFailure }) => { + const { root } = await installedFixture(); + const urls: string[] = []; + vi.stubGlobal("fetch", serve(served, urls)); + const onCompressedFallback = vi.fn(); + const progress: ManagedProviderRuntimeProgress[] = []; + const runtime = new ManagedScientAgentRuntime(root, { smoke: async () => undefined }); + await runtime.install({ + artifact: compressedArtifact("0.2.0", catalogCopy), + signal: new AbortController().signal, + onCompressedFallback, + onProgress: (report) => progress.push(report), + }); + expect(onCompressedFallback).toHaveBeenCalledOnce(); + // The binary's download still says why the compressed copy was given up. + expect(progress.findLast((report) => report.stage === "verifying")?.download).toMatchObject({ + compressed: false, + sourceKind: "scient_mirror", + compressedFailure, + }); + expect(urls.at(-1)).toBe( + "https://downloads.scientfactory.com/scient-agent/v0.2.0/scient-agent-darwin-arm64", + ); + await expectInstalled(runtime, root, "0.2.0"); + }, + ); it("stops without the binary when told not to fall back, or when cancelled", async () => { const { root, previousState } = await installedFixture(); diff --git a/packages/scient-provider-runtime/src/runtimeDownload.test.ts b/packages/scient-provider-runtime/src/runtimeDownload.test.ts index 4e59c46c06..047ed30351 100644 --- a/packages/scient-provider-runtime/src/runtimeDownload.test.ts +++ b/packages/scient-provider-runtime/src/runtimeDownload.test.ts @@ -1,4 +1,4 @@ -// @effect-diagnostics nodeBuiltinImport:off -- Synthetic responses exercise the private download boundary without network or provider credentials. +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off -- Synthetic responses exercise the private download boundary without network or provider credentials; the slow-download case paces its body on the test clock. import * as NodeCrypto from "node:crypto"; import * as NodeFSP from "node:fs/promises"; import * as NodeOS from "node:os"; @@ -11,6 +11,7 @@ import { downloadManagedRuntime, ManagedRuntimeFileError, managedRuntimeFailureClass, + managedRuntimeFailureRetryable, verifyManagedRuntimeChecksum, } from "./runtimeFiles.ts"; @@ -42,6 +43,7 @@ async function input(signal = new AbortController().signal) { } afterEach(async () => { + vi.useRealTimers(); vi.unstubAllGlobals(); await Promise.all( roots.splice(0).map((root) => NodeFSP.rm(root, { recursive: true, force: true })), @@ -348,31 +350,6 @@ describe("downloads with Scient's own copy first", () => { expect(urls(fetch)).toEqual([mirror, request.fallbackUrl, request.fallbackUrl]); }); - it("gives GitHub its own time limit when Scient's copy runs out of time", async () => { - const request = await twoSources(); - const limits: AbortController[] = []; - using _timeout = vi.spyOn(AbortSignal, "timeout").mockImplementation(() => { - const limit = new AbortController(); - limits.push(limit); - return limit.signal; - }); - const fetch = vi - .fn() - .mockImplementationOnce( - (_url: URL, init: RequestInit) => - new Promise((_resolve, reject) => { - init.signal!.addEventListener("abort", () => reject(init.signal!.reason)); - // Scient's copy is still busy when its own ten minutes end. - limits[0]!.abort(new DOMException("time limit", "TimeoutError")); - }), - ) - .mockResolvedValueOnce(ok()); - vi.stubGlobal("fetch", fetch); - await downloadManagedRuntime(request); - expect(urls(fetch)).toEqual([mirror, request.fallbackUrl]); - expect(limits).toHaveLength(2); - }); - it("fails only when both copies fail, and names both", async () => { const request = await twoSources(); const fetch = vi.fn().mockImplementation(async () => new Response("busy", { status: 504 })); @@ -427,9 +404,11 @@ describe("resumable downloads", () => { return partialContent(2, "llo"); }); vi.stubGlobal("fetch", fetch); - await expect(downloadManagedRuntime(request)).resolves.toEqual({ + await expect(downloadManagedRuntime(request)).resolves.toMatchObject({ resumedBytes: 2, source: "primary", + tries: 2, + progressPercent: 100, }); expect(await NodeFSP.readFile(request.destination, "utf8")).toBe(body); expect(await NodeFSP.readdir(NodePath.dirname(request.partialPath))).toEqual([]); @@ -511,36 +490,36 @@ describe("resumable downloads", () => { return partialContent(2, "llo"); }); vi.stubGlobal("fetch", fetch); - await expect(downloadManagedRuntime(request)).resolves.toEqual({ + await expect(downloadManagedRuntime(request)).resolves.toMatchObject({ resumedBytes: 2, source: "fallback", }); }); - it("keeps the bytes when the time limit ends and says trying again continues", async () => { + it("says how far a failed download got, and that a later try may finish it", async () => { const request = await resumable(); - const limit = new AbortController(); - using _timeout = vi.spyOn(AbortSignal, "timeout").mockReturnValue(limit.signal); - await NodeFSP.mkdir(NodePath.dirname(request.partialPath), { recursive: true }); - await NodeFSP.writeFile(request.partialPath, "he"); - const fetch = vi.fn().mockImplementationOnce( - (_url: URL, init: RequestInit) => - new Promise((_resolve, reject) => { - init.signal!.addEventListener("abort", () => reject(init.signal!.reason)); - limit.abort(new DOMException("time limit", "TimeoutError")); - }), - ); + const reset = Object.assign(new Error("private detail"), { code: "ECONNRESET" }); + const fetch = vi + .fn() + .mockResolvedValueOnce(new Response("he")) + .mockRejectedValue(new TypeError("fetch failed", { cause: reset })); vi.stubGlobal("fetch", fetch); - const failure = await downloadManagedRuntime(request).then( - () => expect.unreachable("the download should stop"), + const onFailedTry = vi.fn(); + const failure = await downloadManagedRuntime({ ...request, onFailedTry }).then( + () => expect.unreachable("the download should fail"), (cause: unknown) => cause, ); - expect(failure).toBeInstanceOf(ManagedRuntimeFileError); - expect((failure as Error).message).toBe( - "The download took longer than 10 minutes. Scient kept what it downloaded; trying again continues from there.", - ); - expect(managedRuntimeFailureClass(failure)).toBe("timeout"); - expect(await NodeFSP.readFile(request.partialPath, "utf8")).toBe("he"); + expect(onFailedTry).toHaveBeenCalledTimes(5); + expect(onFailedTry.mock.calls[0]![0]).toMatchObject({ tries: 1, networkKind: "cut_off" }); + expect(onFailedTry.mock.calls.at(-1)![0]).toMatchObject({ + resumedBytes: 2, + source: "primary", + tries: 5, + progressPercent: 40, + networkKind: "reset", + }); + expect(managedRuntimeFailureClass(failure)).toBe("network"); + expect(managedRuntimeFailureRetryable(failure)).toBe(true); }); it("starts the retries afresh while a slow line keeps bringing data", async () => { @@ -573,6 +552,54 @@ describe("resumable downloads", () => { }); }); +describe("downloads without an overall time limit", () => { + // regression: slow downloads failed after ten minutes while data still arrived. + it("keeps a slow download going past ten minutes while data arrives", async () => { + const request = { ...(await input()), expectedSize: 36 * 1024 }; + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"] }); + // A limit built on AbortSignal.timeout would otherwise ignore the test clock. + using _timeout = vi.spyOn(AbortSignal, "timeout").mockImplementation((ms) => { + const limit = new AbortController(); + setTimeout(() => limit.abort(new DOMException("time limit", "TimeoutError")), ms); + return limit.signal; + }); + let body!: ReadableStreamDefaultController; + const response = new Response( + new ReadableStream({ start: (controller) => (body = controller) }), + { + headers: { "content-length": String(request.expectedSize) }, + }, + ); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(response)); + let awaited: { readonly bytes: number; readonly resolve: () => void } | undefined; + const download = downloadManagedRuntime({ + ...request, + onProgress: (bytes) => { + if (awaited && bytes >= awaited.bytes) awaited.resolve(); + }, + }); + const ended = download.then( + () => undefined, + () => undefined, + ); + // Opening the file takes real time, so the clock moves on only once each KiB has arrived. + const written = (bytes: number) => + Promise.race([new Promise((resolve) => (awaited = { bytes, resolve })), ended]); + // 1 KiB every 20 seconds, inside the 30-second stall bound: twelve minutes in all. + for (let chunk = 1; chunk <= 36; chunk += 1) { + await vi.advanceTimersByTimeAsync(20_000); + body.enqueue(new Uint8Array(1024).fill(7)); + await written(chunk * 1024); + } + body.close(); + await expect(download).resolves.toMatchObject({ + tries: 1, + progressPercent: 100, + bytesPerSecond: Math.round(request.expectedSize / 720), + }); + }); +}); + describe("managed runtime failure classes", () => { const failureOf = (promise: Promise) => promise.then( @@ -607,6 +634,21 @@ describe("managed runtime failure classes", () => { ); }); + it.each([ + ["HTTP 429 on every try", () => new Response("slow down", { status: 429 }), true], + ["HTTP 501", () => new Response("never", { status: 501 }), false], + ["a missing file", () => new Response("missing", { status: 404 }), false], + [ + "an intercepted certificate", + () => Promise.reject(fetchFailure("SELF_SIGNED_CERT_IN_CHAIN")), + false, + ], + ])("says whether a later try may fix %s", async (_case, response, retryable) => { + vi.stubGlobal("fetch", vi.fn().mockImplementation(response)); + const failure = await failureOf(downloadManagedRuntime(await input())); + expect(managedRuntimeFailureRetryable(failure)).toBe(retryable); + }); + it("classifies a damaged download, cancellation and local disk errors without reading messages", async () => { const request = await input(); await NodeFSP.writeFile(request.destination, "ko"); @@ -634,6 +676,8 @@ describe("managed runtime failure classes", () => { new Error("Scient could not install it.", { cause: Object.assign(new Error("x"), { code }) }); expect(managedRuntimeFailureClass(wrapped("ENOSPC"))).toBe("disk_full"); expect(managedRuntimeFailureClass(wrapped("EACCES"))).toBe("permission"); + for (const local of [damaged, wrapped("ENOSPC"), wrapped("EACCES")]) + expect(managedRuntimeFailureRetryable(local)).toBe(false); expect( managedRuntimeFailureClass(new ManagedRuntimeFileError("Unsafe archive path"), "installing"), ).toBe("extract"); diff --git a/packages/scient-provider-runtime/src/runtimeFiles.ts b/packages/scient-provider-runtime/src/runtimeFiles.ts index 432622ce19..b35dfd4e57 100644 --- a/packages/scient-provider-runtime/src/runtimeFiles.ts +++ b/packages/scient-provider-runtime/src/runtimeFiles.ts @@ -38,7 +38,6 @@ const DEFAULT_EXTRACTION_LIMITS: ManagedRuntimeExtractionLimits = { maxExpandedBytes: 512 * 1024 * 1024, maxEntries: 32, }; -const DOWNLOAD_TIMEOUT_MS = 10 * 60 * 1_000; const DOWNLOAD_IDLE_TIMEOUT_MS = 30 * 1_000; /** A failed try that still brought this much of a resumable download starts the retries afresh. */ const RESUME_PROGRESS_BYTES = 1024 * 1024; @@ -62,16 +61,47 @@ export type ManagedRuntimeFailureClass = // @effect-diagnostics-next-line extendsNativeError:off - @scientfactory/provider-runtime has no Effect runtime; callers catch it as a native error. export class ManagedRuntimeFileError extends Error { readonly failureClass: ManagedRuntimeFailureClass | undefined; + /** Whether trying the same change again later may succeed, when the class alone can't say. */ + readonly retryable: boolean | undefined; constructor( message: string, - options?: ErrorOptions & { readonly failureClass?: ManagedRuntimeFailureClass }, + options?: ErrorOptions & { + readonly failureClass?: ManagedRuntimeFailureClass; + readonly retryable?: boolean; + }, ) { super(message, options); this.name = "ManagedRuntimeFileError"; this.failureClass = options?.failureClass; + this.retryable = options?.retryable; } } +/** How a try's connection failed, from its error code; never its message. */ +export type ManagedRuntimeNetworkKind = + | "dns" + | "refused" + | "reset" + | "unreachable" + | "connect_timeout" + | "stalled" + | "cut_off" + | "other"; + +const NETWORK_KINDS: Readonly> = { + ENOTFOUND: "dns", + EAI_AGAIN: "dns", + ECONNREFUSED: "refused", + ECONNRESET: "reset", + ECONNABORTED: "reset", + EPIPE: "reset", + UND_ERR_SOCKET: "reset", + ENETUNREACH: "unreachable", + EHOSTUNREACH: "unreachable", + ETIMEDOUT: "connect_timeout", + UND_ERR_CONNECT_TIMEOUT: "connect_timeout", +}; + function assertAllowedHttpsUrl(url: URL, allowedHosts: ReadonlySet): void { if (url.protocol !== "https:") { throw new ManagedRuntimeFileError("Managed runtime downloads must use HTTPS."); @@ -99,11 +129,12 @@ class TransientDownloadFailure extends Error { readonly reason: string; readonly retryAfterMs: number | null; readonly status: number | null; + readonly networkKind: ManagedRuntimeNetworkKind | undefined; constructor( kind: TransientDownloadFailure["kind"], reason: string, retryAfterMs: number | null = null, - options?: ErrorOptions & { readonly status?: number }, + options?: ErrorOptions & { readonly status?: number; readonly code?: string | undefined }, ) { super(reason, options); this.name = "TransientDownloadFailure"; @@ -111,6 +142,14 @@ class TransientDownloadFailure extends Error { this.reason = reason; this.retryAfterMs = retryAfterMs; this.status = options?.status ?? null; + this.networkKind = + kind === "stalled" + ? "stalled" + : kind === "cut-off" + ? "cut_off" + : kind === "connection" + ? ((options?.code === undefined ? undefined : NETWORK_KINDS[options.code]) ?? "other") + : undefined; } } @@ -160,12 +199,7 @@ export function managedRuntimeFailureClass( cause: unknown, stage?: string, ): ManagedRuntimeFailureClass { - const chain: unknown[] = []; - for (let current = cause; current !== undefined && chain.length < 8;) { - chain.push(current); - current = current instanceof Error ? current.cause : undefined; - } - const errors = chain.filter((entry): entry is Error => entry instanceof Error); + const errors = causeChain(cause); for (const error of errors) { if (error instanceof ManagedRuntimeFileError && error.failureClass !== undefined) return error.failureClass; @@ -184,6 +218,33 @@ export function managedRuntimeFailureClass( return "other"; } +function causeChain(cause: unknown): ReadonlyArray { + const errors: Error[] = []; + for (let current = cause; current instanceof Error && errors.length < 8; current = current.cause) + errors.push(current); + return errors; +} + +const RETRYABLE_FAILURE_CLASSES = new Set([ + "network", + "timeout", + "busy", +]); + +/** + * Whether the same change may succeed if tried again later, from error types: a server that stayed + * busy (HTTP 408, 429, most 5xx) or a lost connection may; a 501, certificate, checksum, disk or + * permission failure will not. + */ +export function managedRuntimeFailureRetryable(cause: unknown, stage?: string): boolean { + for (const error of causeChain(cause)) { + if (error instanceof ManagedRuntimeFileError && error.retryable !== undefined) + return error.retryable; + if (error instanceof TransientDownloadFailure) return true; + } + return RETRYABLE_FAILURE_CLASSES.has(managedRuntimeFailureClass(cause, stage)); +} + function retryAfterMs(header: string | null): number | null { if (!header) return null; const seconds = Number(header); @@ -209,7 +270,7 @@ function connectionFailure(cause: unknown): TransientDownloadFailure | ManagedRu "connection", code ? `connection failed, ${code}` : "connection failed", null, - { cause }, + { cause, code }, ); } @@ -291,21 +352,38 @@ interface ManagedRuntimeDownloadInput { * Name it by the release digest: both copies serve the same bytes. Needs `checksum`. */ readonly partialPath?: string | undefined; - /** Per copy; `null` waits as long as data keeps arriving. Ten minutes by default. */ - readonly timeLimitMs?: number | null | undefined; readonly allowedHosts: ReadonlyArray; readonly expectedSize: number; readonly signal: AbortSignal; + /** From the bytes kept by an earlier try, when a try starts, then as bytes arrive. */ readonly onProgress?: (downloaded: number, total: number) => void; readonly onRetry?: (retry: ManagedRuntimeDownloadRetry) => void; + /** After each failed try, before Scient waits or gives up, so a failed download says how far it got. */ + readonly onFailedTry?: (download: ManagedRuntimeDownloadResult) => void; } -/** What a finished download continued from, for analytics. */ +/** How a download went up to its latest try, for analytics. */ export interface ManagedRuntimeDownloadResult { - /** Bytes already on disk when the successful try started; 0 for a fresh download. */ + /** Bytes already on disk when the latest try started; 0 for a fresh download. */ readonly resumedBytes: number; - /** `fallback` when the second copy served the file. */ + /** The copy the latest try used; `fallback` is the second. */ readonly source: "primary" | "fallback"; + /** Tries made, the first included. */ + readonly tries: number; + /** How much of the file the latest try had when it ended, 0 to 100. */ + readonly progressPercent: number; + /** Received bytes per second of request time, kept bytes and waits excluded; absent with none. */ + readonly bytesPerSecond?: number; + /** How the latest try's connection failed. */ + readonly networkKind?: ManagedRuntimeNetworkKind; +} + +/** Shared by the tries of one download. */ +interface DownloadTransfer { + resumedBytes: number; + downloadedBytes: number; + receivedBytes: number; + activeMs: number; } function describeSourceFailure(failure: Error): string { @@ -323,10 +401,10 @@ function describeSourceFailure(failure: Error): string { } /** - * Downloads one reviewed artifact, retrying temporary failures for about a - * minute. With a fallback copy, any failure of one copy moves to the other. - * Cancellation, the time limit and local disk errors always stay final. - * With `partialPath`, a failed try keeps its bytes and the next try asks only for the rest. + * Downloads one reviewed artifact, retrying temporary failures (30 seconds without data among them) + * for about a minute; a fallback copy takes over from a failing one. No overall time limit: a slow + * download goes on while data arrives. Cancellation and local disk errors stay final. With + * `partialPath`, a failed try keeps its bytes and the next asks only for the rest. */ export async function downloadManagedRuntime( input: ManagedRuntimeDownloadInput, @@ -335,90 +413,70 @@ export async function downloadManagedRuntime( const allowedHosts = new Set(input.allowedHosts.map((host) => host.toLowerCase())); const expected = input.checksum ? expectedDigest(input.checksum) : undefined; const partialPath = input.checksum ? input.partialPath : undefined; - const timeLimitMs = input.timeLimitMs === undefined ? DOWNLOAD_TIMEOUT_MS : input.timeLimitMs; - // Each copy gets the whole time limit, so a slow copy never uses up the other's. - const deadlines = new Map(); - const deadline = (source: string): AbortSignal => { - const existing = deadlines.get(source); - if (existing) return existing; - const created = - timeLimitMs === null ? new AbortController().signal : AbortSignal.timeout(timeLimitMs); - deadlines.set(source, created); - return created; + const transfer: DownloadTransfer = { + resumedBytes: 0, + downloadedBytes: 0, + receivedBytes: 0, + activeMs: 0, }; - const waitSignal = - sources.length === 1 ? AbortSignal.any([input.signal, deadline(input.url)]) : input.signal; const lastFailure = new Map(); let usable = [...sources]; let round = [...usable]; let tries = 0; + const facts = (source: string, cause?: unknown): ManagedRuntimeDownloadResult => { + const networkKind = cause instanceof TransientDownloadFailure ? cause.networkKind : undefined; + return { + resumedBytes: transfer.resumedBytes, + source: source === input.url ? "primary" : "fallback", + tries, + progressPercent: Math.min( + 100, + Math.floor((transfer.downloadedBytes / input.expectedSize) * 100), + ), + ...(transfer.receivedBytes > 0 && transfer.activeMs > 0 + ? { bytesPerSecond: Math.round((transfer.receivedBytes * 1_000) / transfer.activeMs) } + : {}), + ...(networkKind ? { networkKind } : {}), + }; + }; for (let retry = 0; ;) { const source = round.shift()!; tries += 1; - const received = { bytes: 0 }; + const receivedBefore = transfer.receivedBytes; let cause: unknown; try { - const attempt = await downloadOnce( - input, - source, - allowedHosts, - deadline(source), - partialPath, - received, - ); - let resumedBytes = attempt.resumedBytes; + const attempt = await downloadOnce(input, source, allowedHosts, partialPath, transfer); if (expected !== undefined && attempt.digest !== expected) { await removePartialDownload(partialPath ?? input.destination); // Bytes kept from an earlier try may be what is wrong, so fetch the whole file once. - if (resumedBytes === 0) throw checksumMismatch(); - const fresh = await downloadOnce( - input, - source, - allowedHosts, - deadline(source), - partialPath, - received, - ); + if (attempt.resumedBytes === 0) throw checksumMismatch(); + const fresh = await downloadOnce(input, source, allowedHosts, partialPath, transfer); if (fresh.digest !== expected) { await removePartialDownload(partialPath ?? input.destination); throw checksumMismatch(); } - resumedBytes = 0; } if (partialPath) { await NodeFSP.rename(partialPath, input.destination); await NodeFSP.rm(partialValidatorPath(partialPath), { force: true }).catch(() => undefined); } - return { resumedBytes, source: source === input.url ? "primary" : "fallback" }; + return facts(source); } catch (caught) { cause = caught; } - // With a second copy, a problem with this one, running out of time - // included, only rules this copy out. Cancellation still ends everything. - const outOfTime = deadline(source).aborted && !input.signal.aborted; - const ruledOut = sources.length > 1 && (cause instanceof ManagedRuntimeFileError || outOfTime); - if (!(cause instanceof TransientDownloadFailure) && !ruledOut) { - if (outOfTime && partialPath && (await partialDownloadSize(partialPath)) > 0) { - throw new ManagedRuntimeFileError( - `The download took longer than ${describeTimeLimit(timeLimitMs)}. Scient kept what it downloaded; trying again continues from there.`, - { cause, failureClass: "timeout" }, - ); - } - throw cause; - } - lastFailure.set( - source, - outOfTime - ? new Error(`took longer than ${describeTimeLimit(timeLimitMs)}.`) - : (cause as Error), - ); + input.onFailedTry?.(facts(source, cause)); + // With a second copy, a problem with this one only rules this copy out. + // Cancellation still ends everything. + const ruledOut = sources.length > 1 && cause instanceof ManagedRuntimeFileError; + if (!(cause instanceof TransientDownloadFailure) && !ruledOut) throw cause; + lastFailure.set(source, cause as Error); if (ruledOut) { usable = usable.filter((candidate) => candidate !== source); round = round.filter((candidate) => candidate !== source); } if (usable.length > 0 && round.length > 0) continue; // Steady progress on a slow line is not a failing server; keep going while it lasts. - if (received.bytes >= RESUME_PROGRESS_BYTES) retry = 0; + if (transfer.receivedBytes - receivedBefore >= RESUME_PROGRESS_BYTES) retry = 0; const baseDelay = RETRY_DELAYS_MS[retry]; if (usable.length === 0 || baseDelay === undefined) { const kept = @@ -428,7 +486,11 @@ export async function downloadManagedRuntime( if (sources.length === 1) { throw new ManagedRuntimeFileError( `${exhaustedMessage(cause as TransientDownloadFailure, tries)}${kept}`, - { cause, failureClass: transientFailureClass(cause as TransientDownloadFailure) }, + { + cause, + failureClass: transientFailureClass(cause as TransientDownloadFailure), + retryable: true, + }, ); } const details = sources @@ -439,7 +501,13 @@ export async function downloadManagedRuntime( .join(" "); throw new ManagedRuntimeFileError( `Scient couldn't download it from either source after ${tries} tries. ${details}${kept}`, - { cause, failureClass: outOfTime ? "timeout" : managedRuntimeFailureClass(cause) }, + { + cause, + failureClass: managedRuntimeFailureClass(cause), + retryable: [...lastFailure.values()].some( + (failure) => failure instanceof TransientDownloadFailure, + ), + }, ); } const delayMs = @@ -452,16 +520,11 @@ export async function downloadManagedRuntime( reason: cause instanceof TransientDownloadFailure ? cause.reason : "download failed", delayMs, }); - await NodeTimersPromises.setTimeout(delayMs, undefined, { signal: waitSignal }); + await NodeTimersPromises.setTimeout(delayMs, undefined, { signal: input.signal }); round = [...usable]; } } -function describeTimeLimit(timeLimitMs: number | null): string { - const minutes = Math.round((timeLimitMs ?? 0) / 60_000); - return minutes === 1 ? "1 minute" : `${minutes} minutes`; -} - function checksumMismatch(): ManagedRuntimeFileError { return new ManagedRuntimeFileError("Managed runtime verification failed: checksum mismatch.", { failureClass: "checksum", @@ -535,9 +598,8 @@ async function downloadOnce( input: ManagedRuntimeDownloadInput, url: string, allowedHosts: ReadonlySet, - timeoutSignal: AbortSignal, partialPath: string | undefined, - received: { bytes: number }, + transfer: DownloadTransfer, ): Promise<{ readonly resumedBytes: number; readonly digest: string | undefined }> { const target = partialPath ?? input.destination; const newHash = () => @@ -549,10 +611,10 @@ async function downloadOnce( have = 0; } if (have > 0 && hash) await hashPrefix(target, have, hash); - if (have === input.expectedSize) { - input.onProgress?.(have, input.expectedSize); - return { resumedBytes: have, digest: hash?.digest("hex") }; - } + transfer.resumedBytes = have; + transfer.downloadedBytes = have; + input.onProgress?.(have, input.expectedSize); + if (have === input.expectedSize) return { resumedBytes: have, digest: hash?.digest("hex") }; const idleController = new AbortController(); let idleTimer: ReturnType | undefined; @@ -560,12 +622,12 @@ async function downloadOnce( if (idleTimer) clearTimeout(idleTimer); idleTimer = setTimeout(() => idleController.abort(), DOWNLOAD_IDLE_TIMEOUT_MS); }; - const signal = AbortSignal.any([input.signal, timeoutSignal, idleController.signal]); - // A stall is worth another try; cancellation and the overall limit are not. + const signal = AbortSignal.any([input.signal, idleController.signal]); + // A stall is worth another try; cancellation is not. const classify = (cause: unknown, fromNetwork: boolean): unknown => { if (cause instanceof ManagedRuntimeFileError || cause instanceof TransientDownloadFailure) return cause; - if (input.signal.aborted || timeoutSignal.aborted) return cause; + if (input.signal.aborted) return cause; if (idleController.signal.aborted) return new TransientDownloadFailure("stalled", "download stalled", null, { cause }); return fromNetwork ? connectionFailure(cause) : cause; @@ -585,6 +647,7 @@ async function downloadOnce( }); }; resetIdleTimer(); + const startedAt = Date.now(); try { let response = await request(have); @@ -595,6 +658,8 @@ async function downloadOnce( response = await request(0); } have = 0; + transfer.resumedBytes = 0; + transfer.downloadedBytes = 0; hash = newHash(); } if (!response.ok || !response.body) { @@ -628,7 +693,8 @@ async function downloadOnce( const source = NodeStream.Readable.fromWeb(response.body as never); source.on("data", (chunk: Buffer) => { downloaded += chunk.byteLength; - received.bytes += chunk.byteLength; + transfer.downloadedBytes = downloaded; + transfer.receivedBytes += chunk.byteLength; hash?.update(chunk); resetIdleTimer(); if (downloaded > MAX_DOWNLOAD_BYTES || downloaded > input.expectedSize) { @@ -665,6 +731,7 @@ async function downloadOnce( return { resumedBytes: have, digest: hash?.digest("hex") }; } finally { if (idleTimer) clearTimeout(idleTimer); + transfer.activeMs += Date.now() - startedAt; } } From 3c6909e47c004bc455432e61322de5753f60fe25 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Mon, 16 Nov 2026 23:07:09 +0200 Subject: [PATCH 02/12] feat(desktop): report app update outcomes and summaries Record one app.update.outcome per check, download, install and apply attempt and app.update.summary at launch, on status changes and daily, both Essential with closed values. Facts wait in a bounded persisted desktop journal and travel on the desktopUpdateStatus pipe to one backend, which queues them with their own ids and times and acknowledges only after the outbox holds them. Nothing is kept while analytics is off. The four-hour update poller survives an unexpected failure. Co-Authored-By: Claude Opus 5.5 --- UPSTREAM.md | 5 + .../src/backend/DesktopBackendManager.ts | 4 +- apps/desktop/src/main.ts | 8 +- .../updateOutcomes/DesktopUpdateFacts.test.ts | 183 +++++++++ .../updateOutcomes/DesktopUpdateFacts.ts | 218 +++++++++++ .../updateOutcomes/DesktopUpdateOutcomes.ts | 351 ++++++++++++++++++ .../updateOutcomes/updateFailureClass.test.ts | 80 ++++ .../updateOutcomes/updateFailureClass.ts | 135 +++++++ .../telemetry/DesktopTelemetryPublisher.ts | 82 +++- .../src/updates/DesktopUpdates.scient.test.ts | 172 +++++++++ apps/desktop/src/updates/DesktopUpdates.ts | 117 +++++- .../DesktopTelemetryReceiver.ts | 22 ++ .../AnalyticsServiceControls.test.ts | 1 + .../src/telemetry/DesktopUpdateFacts.test.ts | 34 ++ .../src/telemetry/DesktopUpdateFacts.ts | 22 ++ apps/server/src/telemetry/ScientAnalytics.ts | 17 + docs/internals/product-analytics.md | 46 ++- packages/contracts/src/resourceTelemetry.ts | 33 ++ .../fixtures/contract-v11.json | 171 +++++++++ packages/scient-analytics/src/conformance.ts | 66 ++++ packages/scient-analytics/src/contract.ts | 155 ++++++++ packages/scient-analytics/src/index.ts | 1 + packages/scient-analytics/src/runtime.test.ts | 58 +++ packages/scient-analytics/src/runtime.ts | 84 +++++ 24 files changed, 2039 insertions(+), 26 deletions(-) create mode 100644 apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts create mode 100644 apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.ts create mode 100644 apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts create mode 100644 apps/desktop/src/scient/updateOutcomes/updateFailureClass.test.ts create mode 100644 apps/desktop/src/scient/updateOutcomes/updateFailureClass.ts create mode 100644 apps/desktop/src/updates/DesktopUpdates.scient.test.ts create mode 100644 apps/server/src/telemetry/DesktopUpdateFacts.test.ts create mode 100644 apps/server/src/telemetry/DesktopUpdateFacts.ts diff --git a/UPSTREAM.md b/UPSTREAM.md index 4901c55b75..211ac76e6e 100644 --- a/UPSTREAM.md +++ b/UPSTREAM.md @@ -203,6 +203,11 @@ call live in `apps/web/src/scient/desktopUpdate/updateReadyNotice.tsx`; callers callers; do not reintroduce the confirmation during alignment. A managed policy's feed stays the update source on every channel. Install-on-quit and early native (Squirrel.Mac) preparation stay off: they need native-readiness tracking and packaged-build tests first. +Update outcomes and summaries (`app.update.*`, see `docs/internals/product-analytics.md`) live in +`apps/desktop/src/scient/updateOutcomes/`; `DesktopUpdates.ts`, `DesktopTelemetryPublisher.ts`, +`DesktopBackendManager.ts`, `main.ts`, the telemetry contract and `DesktopTelemetryReceiver.ts` +carry only marked calls into them. The four-hour poller contains an unexpected failure per +iteration instead of ending; keep that when aligning upstream's poller. The update pill shows only this app's own update. Other machines' servers and apps, provider CLIs and Scient-managed runtimes appear in a Scient-owned circle and card (`apps/web/src/scient/desktopUpdate/SidebarOtherUpdates.tsx`, `useOtherUpdates.ts`), mounted in diff --git a/apps/desktop/src/backend/DesktopBackendManager.ts b/apps/desktop/src/backend/DesktopBackendManager.ts index 8b4f154b5d..56f107f487 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.ts @@ -943,7 +943,9 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( const program = runBackendProcess({ ...config.value, - desktopTelemetryStream: desktopTelemetryPublisher.encoded, + // SCIENT-FORK: this backend's stream carries the update facts offered to it. + desktopTelemetryStream: + desktopTelemetryPublisher.encodedFor?.(spec.id) ?? desktopTelemetryPublisher.encoded, desktopBrowserStream: desktopBrowserHost.events, onDesktopBrowserCommand: desktopBrowserHost.handleCommandLine, onDesktopTelemetryControl: (message) => diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 6e7f708b89..2a3770c8d3 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -31,6 +31,9 @@ import * as DesktopVoice from "./app/DesktopVoice.ts"; // SCIENT-FORK:START — keep the computer awake while threads work. import * as DesktopKeepAwake from "./scient/keepAwake/DesktopKeepAwake.ts"; // SCIENT-FORK:END +// SCIENT-FORK:START — the update facts journal the updater writes and the telemetry pipe delivers. +import * as DesktopUpdateFacts from "./scient/updateOutcomes/DesktopUpdateFacts.ts"; +// SCIENT-FORK:END import * as ElectronApp from "./electron/ElectronApp.ts"; import * as ElectronDialog from "./electron/ElectronDialog.ts"; import * as ElectronMenu from "./electron/ElectronMenu.ts"; @@ -220,7 +223,8 @@ const layerDesktopPreview = PreviewManager.layer.pipe( ); const layerDesktopWindow = DesktopWindow.layer.pipe( - Layer.provideMerge(DesktopTelemetryPublisher.layer), + // SCIENT-FORK: the same journal layer as the updater's, so both share one journal. + Layer.provideMerge(DesktopTelemetryPublisher.layer.pipe(Layer.provide(DesktopUpdateFacts.layer))), Layer.provideMerge(DesktopShellEnvironment.layer), Layer.provideMerge(layerDesktopServerExposure), Layer.provideMerge(layerDesktopPreview), @@ -268,6 +272,8 @@ const layerDesktopApplication = Layer.mergeAll( Layer.provideMerge( DesktopUpdates.layer.pipe( Layer.provide(DesktopUpdatePolicy.layer), + // SCIENT-FORK: the same journal layer as the telemetry publisher's. + Layer.provide(DesktopUpdateFacts.layer), // The same layer as the foundation's, so the policy is still read once. Layer.provide(DesktopManagedPolicy.layer), ), diff --git a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts new file mode 100644 index 0000000000..fe717f8d92 --- /dev/null +++ b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts @@ -0,0 +1,183 @@ +// @effect-diagnostics nodeBuiltinImport:off -- a real temporary journal directory. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, it } from "@effect/vitest"; +import type { DesktopUpdateFact } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import type * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; + +import * as DesktopConfig from "../../app/DesktopConfig.ts"; +import * as DesktopEnvironment from "../../app/DesktopEnvironment.ts"; +import * as DesktopManagedPolicy from "../../app/DesktopManagedPolicy.ts"; +import { PRIMARY_INSTANCE_ID } from "../../backend/DesktopBackendPool.ts"; +import * as DesktopUpdateFacts from "./DesktopUpdateFacts.ts"; + +const WSL_SOURCE = "wsl:Ubuntu"; + +function journalLayer(home: string) { + const config = DesktopConfig.layerTest({ SCIENT_NEXT_HOME: home }); + const environment = DesktopEnvironment.layer({ + dirname: "/repo/apps/desktop/src", + homeDirectory: home, + platform: "darwin", + processArch: "arm64", + appVersion: "1.2.3", + appPath: "/repo", + isPackaged: true, + resourcesPath: "/missing/resources", + runningUnderArm64Translation: false, + }).pipe(Layer.provide(Layer.mergeAll(NodeServices.layer, config))); + return DesktopUpdateFacts.layer.pipe( + Layer.provide( + Layer.succeed(DesktopManagedPolicy.DesktopManagedPolicy, { + read: Effect.undefined, + }), + ), + Layer.provide(environment), + Layer.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ); +} + +/** One desktop process: its journal service, with analytics on or off for that launch. */ +function inLaunch( + home: string, + analytics: "true" | "false", + program: ( + facts: DesktopUpdateFacts.DesktopUpdateFacts["Service"], + ) => Effect.Effect, +) { + return Effect.scoped( + Effect.gen(function* () { + return yield* program(yield* DesktopUpdateFacts.DesktopUpdateFacts); + }), + ).pipe( + Effect.provide(journalLayer(home)), + Effect.provideService(HostProcess.Environment, { SCIENT_ANALYTICS_ENABLED: analytics }), + ); +} + +/** Collects the next offer each backend receives once `trigger` runs. */ +const nextOffers = Effect.fn("test.nextOffers")(function* ( + facts: DesktopUpdateFacts.DesktopUpdateFacts["Service"], + trigger: Effect.Effect, +) { + const watch = (sourceId: string) => + facts.deliveries(sourceId).pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped); + const primary = yield* watch(PRIMARY_INSTANCE_ID); + const wsl = yield* watch(WSL_SOURCE); + yield* Effect.yieldNow; + yield* trigger; + const offered = (yield* Fiber.join(primary)).flat(); + // Two backends have two analytics identities: an offer to both would count each fact twice. + yield* Fiber.interrupt(wsl); + assert.isFalse(Exit.isSuccess(yield* Fiber.await(wsl))); + return offered; +}); + +const ids = (facts: ReadonlyArray) => facts.map((fact) => fact.id); + +function temporaryHome(): string { + return NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "scient-update-facts-")); +} + +function journalFile(home: string): string { + return NodePath.join(home, "runtime", DesktopUpdateFacts.DESKTOP_UPDATE_FACTS_FILE); +} + +describe("DesktopUpdateFacts", () => { + it.effect( + "offers journaled facts to one backend, with the same ids after a restart, until acknowledged", + () => + Effect.gen(function* () { + const home = temporaryHome(); + const recorded = yield* inLaunch(home, "true", (facts) => + Effect.gen(function* () { + yield* facts.sourceReady(WSL_SOURCE); + return yield* nextOffers( + facts, + Effect.all([ + facts.sourceReady(PRIMARY_INSTANCE_ID), + facts.record("app.update.outcome", { + step: "install", + outcome: "handoff-attempted", + }), + ]).pipe(Effect.asVoid), + ); + }), + ); + assert.lengthOf(recorded, 1); + + // The app quit before any backend acknowledged; the next launch offers the same events. + const replayed = yield* inLaunch(home, "true", (facts) => + Effect.gen(function* () { + const offered = yield* nextOffers(facts, facts.sourceReady(PRIMARY_INSTANCE_ID)); + yield* facts.acknowledge(ids(offered)); + return offered; + }), + ); + assert.deepEqual(ids(replayed), ids(recorded)); + const journal = JSON.parse(NodeFS.readFileSync(journalFile(home), "utf8")) as { + readonly facts: ReadonlyArray; + }; + assert.lengthOf(journal.facts, 0); + }), + ); + + it.effect("never keeps or replays facts while analytics is off", () => + Effect.gen(function* () { + const home = temporaryHome(); + yield* inLaunch(home, "true", (facts) => + facts.record("app.update.outcome", { step: "apply", outcome: "not-applied" }), + ); + assert.isTrue(NodeFS.existsSync(journalFile(home))); + + yield* inLaunch(home, "false", (facts) => + facts.record("app.update.summary", { reason: "launch", status: "idle" }), + ); + assert.isFalse(NodeFS.existsSync(journalFile(home))); + + const offered = yield* inLaunch(home, "true", (facts) => + nextOffers( + facts, + facts + .sourceReady(PRIMARY_INSTANCE_ID) + .pipe( + Effect.andThen( + facts.record("app.update.summary", { reason: "launch", status: "idle" }), + ), + ), + ), + ); + assert.deepEqual( + offered.map((fact) => fact.name), + ["app.update.summary"], + ); + }), + ); + + it.effect("counts facts dropped for room in the next summary", () => + Effect.gen(function* () { + const home = temporaryHome(); + const journal = yield* inLaunch(home, "true", (facts) => + Effect.gen(function* () { + for (let index = 0; index < 66; index += 1) { + yield* facts.record("app.update.outcome", { step: "check", outcome: "up-to-date" }); + } + yield* facts.record("app.update.summary", { reason: "daily", status: "up-to-date" }); + return yield* nextOffers(facts, facts.sourceReady(PRIMARY_INSTANCE_ID)); + }), + ); + assert.lengthOf(journal, 64); + assert.equal(journal.at(-1)?.properties.droppedFactCount, 2); + }), + ); +}); diff --git a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.ts b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.ts new file mode 100644 index 0000000000..4fca357f3f --- /dev/null +++ b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.ts @@ -0,0 +1,218 @@ +/** + * DesktopUpdateFacts - persisted update facts, kept until one backend's analytics outbox holds + * them: installs run with every backend stopped, and apply outcomes precede any backend. + * @module DesktopUpdateFacts + */ +import { + DESKTOP_UPDATE_FACTS_LIMIT, + DesktopUpdateFact, + type DesktopUpdateFact as DesktopUpdateFactValue, +} from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as PubSub from "effect/PubSub"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; + +import * as DesktopEnvironment from "../../app/DesktopEnvironment.ts"; +import * as DesktopManagedPolicy from "../../app/DesktopManagedPolicy.ts"; +import * as DesktopObservability from "../../app/DesktopObservability.ts"; +// Read only at call time: the backend manager's stream imports this module. +import * as DesktopBackendManager from "../../backend/DesktopBackendManager.ts"; +import { scientAnalyticsMetadata } from "../../backend/scientAnalyticsMetadata.ts"; + +export const DESKTOP_UPDATE_FACTS_FILE = "desktop-update-facts.json"; +const MAX_FACT_AGE_MS = 14 * 86_400_000; + +const Journal = Schema.Struct({ + version: Schema.Literal(1), + /** Facts dropped for room or age since a summary last reported the count. */ + dropped: Schema.Number, + facts: Schema.Array(DesktopUpdateFact), +}); +type Journal = typeof Journal.Type; +const JournalJson = Schema.fromJsonString(Journal); +const decodeJournal = Schema.decodeUnknownEffect(JournalJson); +const encodeJournal = Schema.encodeEffect(JournalJson); + +export type DesktopUpdateFactProperties = Readonly>; + +export class DesktopUpdateFacts extends Context.Service< + DesktopUpdateFacts, + { + /** Journals a fact and offers the journal to the designated backend; off records nothing. */ + readonly record: ( + name: DesktopUpdateFactValue["name"], + properties: DesktopUpdateFactProperties, + ) => Effect.Effect; + /** The facts offered to one backend; each offer carries every pending fact. */ + readonly deliveries: (sourceId: string) => Stream.Stream>; + readonly sourceReady: (sourceId: string) => Effect.Effect; + readonly sourceRemoved: (sourceId: string) => Effect.Effect; + /** The backend's outbox holds these facts, or analytics there must never send them. */ + readonly acknowledge: (ids: ReadonlyArray) => Effect.Effect; + } +>()("@t3tools/desktop/scient/updateOutcomes/DesktopUpdateFacts") {} + +const { logWarning } = DesktopObservability.makeComponentLogger("desktop-update-facts"); + +const emptyJournal: Journal = { version: 1, dropped: 0, facts: [] }; + +/** Keeps the newest facts within the count and age bounds, counting what it drops. */ +function boundJournal(journal: Journal, nowMs: number): Journal { + const fresh = journal.facts.filter((fact) => { + const occurredAt = Date.parse(fact.occurredAt); + return Number.isFinite(occurredAt) && nowMs - occurredAt <= MAX_FACT_AGE_MS; + }); + const kept = fresh.slice(-DESKTOP_UPDATE_FACTS_LIMIT); + return { + version: 1, + dropped: journal.dropped + journal.facts.length - kept.length, + facts: kept, + }; +} + +/** Primary when it is ready; otherwise the earliest ready backend, as in a WSL-only setup. */ +function designatedSource(ready: ReadonlyArray): string | undefined { + const primary = DesktopBackendManager.PRIMARY_INSTANCE_ID; + return ready.includes(primary) ? primary : ready[0]; +} + +const make = Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const fileSystem = yield* FileSystem.FileSystem; + const random = yield* SecureRandom.SecureRandom; + const env = yield* HostProcess.Environment; + const metadata = scientAnalyticsMetadata(environment, env.SCIENT_ANALYTICS_ENABLED); + // Read once and shared with the updater's policy; a policy that cannot be used counts as off. + const telemetryAllowed = yield* (yield* DesktopManagedPolicy.DesktopManagedPolicy).read.pipe( + Effect.map((found) => found?.document?.limits?.telemetry !== false), + Effect.orElseSucceed(() => false), + ); + const enabled = metadata.SCIENT_ANALYTICS_ENABLED === "true" && telemetryAllowed; + const buildChannel = + metadata.SCIENT_ANALYTICS_BUILD_CHANNEL as DesktopUpdateFactValue["buildChannel"]; + + const directory = environment.path.join(environment.baseDir, "runtime"); + const journalPath = environment.path.join(directory, DESKTOP_UPDATE_FACTS_FILE); + const temporaryPath = `${journalPath}.tmp`; + const writeLock = yield* Semaphore.make(1); + + const persist = (journal: Journal) => + encodeJournal(journal).pipe( + Effect.flatMap((text) => + fileSystem + .makeDirectory(directory, { recursive: true }) + .pipe( + Effect.andThen(fileSystem.writeFileString(temporaryPath, text)), + Effect.andThen(fileSystem.rename(temporaryPath, journalPath)), + ), + ), + Effect.catch((error) => + logWarning("Could not save the update facts journal.", { errorTag: error._tag }), + ), + ); + + const nowMs = Clock.currentTimeMillis; + const loaded = yield* fileSystem.readFileString(journalPath).pipe( + Effect.flatMap(decodeJournal), + Effect.orElseSucceed(() => emptyJournal), + ); + // Facts are never kept for a session that may not send them, nor sent from one later. + const initial = enabled ? boundJournal(loaded, yield* nowMs) : emptyJournal; + if (!enabled && loaded.facts.length > 0) { + yield* fileSystem.remove(journalPath, { force: true }).pipe(Effect.ignore); + } + const journalRef = yield* Ref.make(initial); + const readyRef = yield* Ref.make>([]); + const offers = yield* PubSub.unbounded<{ + readonly sourceId: string; + readonly facts: ReadonlyArray; + }>(); + + const offer = Effect.gen(function* () { + const target = designatedSource(yield* Ref.get(readyRef)); + const { facts } = yield* Ref.get(journalRef); + if (target === undefined || facts.length === 0) return; + yield* PubSub.publish(offers, { sourceId: target, facts }); + }); + + const update = (f: (journal: Journal, nowMs: number) => Journal) => + writeLock.withPermits(1)( + Effect.gen(function* () { + const next = f(yield* Ref.get(journalRef), yield* nowMs); + yield* Ref.set(journalRef, next); + yield* persist(next); + }), + ); + + const record = Effect.fn("DesktopUpdateFacts.record")(function* ( + name: DesktopUpdateFactValue["name"], + properties: DesktopUpdateFactProperties, + ) { + if (!enabled) return; + const id = yield* random.uuidV4(); + const occurredAt = DateTime.formatIso(yield* DateTime.now); + yield* update((journal, now) => { + // A summary reports the drops since the last one; its own fact then carries the count. + const reportsDrops = name === "app.update.summary"; + const fact: DesktopUpdateFactValue = { + id, + name, + occurredAt, + appVersion: metadata.SCIENT_ANALYTICS_APP_VERSION, + buildChannel, + properties: reportsDrops + ? { ...properties, droppedFactCount: journal.dropped } + : properties, + }; + return boundJournal( + { + version: 1, + dropped: reportsDrops ? 0 : journal.dropped, + facts: [...journal.facts, fact], + }, + now, + ); + }); + yield* offer; + }); + + return DesktopUpdateFacts.of({ + record, + deliveries: (sourceId) => + Stream.fromPubSub(offers).pipe( + Stream.filter((delivery) => delivery.sourceId === sourceId), + Stream.map((delivery) => delivery.facts), + ), + sourceReady: Effect.fn("DesktopUpdateFacts.sourceReady")(function* (sourceId) { + yield* Ref.update(readyRef, (ready) => + ready.includes(sourceId) ? ready : [...ready, sourceId], + ); + yield* offer; + }), + sourceRemoved: Effect.fn("DesktopUpdateFacts.sourceRemoved")(function* (sourceId) { + const designated = yield* Ref.modify(readyRef, (ready) => [ + designatedSource(ready) === sourceId, + ready.filter((id) => id !== sourceId), + ]); + if (designated) yield* offer; + }), + acknowledge: Effect.fn("DesktopUpdateFacts.acknowledge")(function* (ids) { + yield* update((journal) => ({ + ...journal, + facts: journal.facts.filter((fact) => !ids.includes(fact.id)), + })); + }), + }); +}); + +export const layer = Layer.effect(DesktopUpdateFacts, make); diff --git a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts new file mode 100644 index 0000000000..cd45a43e6d --- /dev/null +++ b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts @@ -0,0 +1,351 @@ +/** + * DesktopUpdateOutcomes - what the desktop updater records about itself: one outcome per step + * attempt (`app.update.outcome`) and summaries of where an update stands (`app.update.summary`). + * Every fact goes through the persisted DesktopUpdateFacts journal. + * + * @module DesktopUpdateOutcomes + */ +import type { DesktopUpdateState } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; + +import * as DesktopUpdateFacts from "./DesktopUpdateFacts.ts"; +import { classifyUpdateFailure, type UpdateFailureClass } from "./updateFailureClass.ts"; + +export type UpdateTrigger = "user" | "background"; +export type UpdatePackageKind = "zip" | "nsis" | "appimage" | "deb"; +export type UpdateSourceKind = "github" | "scient_mirror" | "administrator"; + +/** Who asked for the update work running in this fiber; background loops provide their own. */ +export const CurrentUpdateTrigger = Context.Reference( + "@t3tools/desktop/scient/updateOutcomes/CurrentUpdateTrigger", + { defaultValue: () => "user" }, +); + +const BACKGROUND_CHECK_REASONS = new Set(["startup", "poll", "startup-recovery"]); +const SUMMARY_INTERVAL = "24 hours"; + +function checkTrigger(reason: string): UpdateTrigger { + return BACKGROUND_CHECK_REASONS.has(reason) ? "background" : "user"; +} + +export function updatePackageKind(input: { + readonly platform: NodeJS.Platform; + readonly appImage: boolean; + readonly debPackage: boolean; +}): UpdatePackageKind | undefined { + if (input.platform === "darwin") return "zip"; + if (input.platform === "win32") return "nsis"; + if (input.platform !== "linux") return undefined; + return input.appImage ? "appimage" : input.debPackage ? "deb" : undefined; +} + +/** The updater's feed: an administrator's, else Scient's own (GitHub releases or the mirror). */ +export function updateSourceKind(input: { + readonly administratorFeed: boolean; + readonly defaultProvider: string; +}): UpdateSourceKind { + if (input.administratorFeed) return "administrator"; + return input.defaultProvider === "github" ? "github" : "scient_mirror"; +} + +export interface UpdateInstallRecord { + readonly fromVersion: string; + readonly targetVersion: string; + readonly startedAt?: number | undefined; + readonly attemptRef?: string | undefined; +} + +/** What a launch observes about the install a previous run handed to the installer. */ +export function applyOutcome( + record: UpdateInstallRecord, + runningVersion: string, +): "completed" | "not-applied" | "superseded" | undefined { + if (record.fromVersion === record.targetVersion) return undefined; + if (runningVersion === record.targetVersion) return "completed"; + if (runningVersion === record.fromVersion) return "not-applied"; + return "superseded"; +} + +function disabledClass(state: DesktopUpdateState, policyOff: boolean): string { + if (state.channelBlockedByPolicy) return "channel-policy"; + if (policyOff) return "policy"; + const message = state.message ?? ""; + if (/safety policy/u.test(message)) return "safety"; + if (/no update feed/u.test(message)) return "no-feed"; + if (/packaged production builds/u.test(message)) return "development"; + if (/T3CODE_DISABLE_AUTO_UPDATE/u.test(message)) return "environment"; + if (/on Linux require/u.test(message)) return "linux-package"; + return "other"; +} + +/** Status that a summary reports a change of; a check in progress is not a change. */ +function summaryKey(state: DesktopUpdateState): string | undefined { + if (state.status === "checking") return undefined; + return state.status === "error" ? `error:${state.errorContext ?? ""}` : state.status; +} + +interface Attempt { + readonly ref: string; + readonly startedAt: number; + readonly trigger: UpdateTrigger; + readonly targetVersion: string | null; + readonly finished: boolean; +} + +type AttemptStep = "check" | "download" | "install"; + +export const make = Effect.fn("DesktopUpdateOutcomes.make")(function* (options: { + readonly facts: Option.Option; + readonly packageKind: UpdatePackageKind | undefined; + readonly sourceKind: UpdateSourceKind | undefined; + readonly policyOff: boolean; +}) { + const random = yield* SecureRandom.SecureRandom; + const attempts = yield* Ref.make>>({}); + const lastCheckRef = yield* Ref.make( + Option.none<{ readonly ok: boolean; readonly at: number }>(), + ); + const lastSuccessfulCheckAtRef = yield* Ref.make(Option.none()); + const context = { + ...(options.packageKind === undefined ? {} : { packageKind: options.packageKind }), + ...(options.sourceKind === undefined ? {} : { sourceKind: options.sourceKind }), + }; + + const record = ( + name: "app.update.outcome" | "app.update.summary", + properties: DesktopUpdateFacts.DesktopUpdateFactProperties, + ) => + Option.match(options.facts, { + onNone: () => Effect.void, + onSome: (facts) => facts.record(name, properties), + }); + + const newRef = random.uuidV4().pipe(Effect.map((uuid) => uuid.replaceAll("-", "").slice(0, 16))); + + const begin = Effect.fn("DesktopUpdateOutcomes.begin")(function* ( + step: AttemptStep, + targetVersion: string | null, + ) { + const attempt: Attempt = { + ref: yield* newRef, + startedAt: yield* Clock.currentTimeMillis, + trigger: yield* CurrentUpdateTrigger, + targetVersion, + finished: false, + }; + yield* Ref.update(attempts, (current) => ({ ...current, [step]: attempt })); + return attempt; + }); + + const outcome = ( + step: AttemptStep, + attempt: Attempt, + result: string, + details: { + readonly targetVersion?: string | null; + readonly failureClass?: UpdateFailureClass; + readonly durationMs?: number; + } = {}, + ) => { + const targetVersion = details.targetVersion ?? attempt.targetVersion; + return record("app.update.outcome", { + step, + outcome: result, + trigger: attempt.trigger, + attemptRef: attempt.ref, + ...context, + ...(targetVersion === null ? {} : { targetVersion }), + ...(details.failureClass === undefined ? {} : { failureClass: details.failureClass }), + ...(details.durationMs === undefined ? {} : { durationMs: details.durationMs }), + }); + }; + + /** Records the attempt's terminal outcome once, however many paths report its end. */ + const finish = Effect.fn("DesktopUpdateOutcomes.finish")(function* ( + step: AttemptStep, + result: string, + details: { readonly targetVersion?: string | null; readonly cause?: unknown } = {}, + ) { + const attempt = yield* Ref.modify(attempts, (current) => { + const active = current[step]; + return active === undefined || active.finished + ? [undefined, current] + : [active, { ...current, [step]: { ...active, finished: true } }]; + }); + if (attempt === undefined) return; + const now = yield* Clock.currentTimeMillis; + if (step === "check") { + yield* Ref.set(lastCheckRef, Option.some({ ok: result !== "failed", at: now })); + if (result !== "failed") yield* Ref.set(lastSuccessfulCheckAtRef, Option.some(now)); + } + yield* outcome(step, attempt, result, { + ...(details.targetVersion === undefined ? {} : { targetVersion: details.targetVersion }), + ...(result === "failed" ? { failureClass: classifyFailure(details.cause) } : {}), + ...(step === "install" ? {} : { durationMs: Math.max(0, now - attempt.startedAt) }), + }); + }); + + const summary = Effect.fn("DesktopUpdateOutcomes.summary")(function* ( + reason: "launch" | "change" | "daily", + state: DesktopUpdateState, + times: SummaryTimes, + ) { + const now = yield* Clock.currentTimeMillis; + const lastCheck = yield* Ref.get(lastCheckRef); + const lastSuccessfulCheckAt = yield* Ref.get(lastSuccessfulCheckAtRef); + const offered = state.availableVersion !== null || state.downloadedVersion !== null; + yield* record("app.update.summary", { + reason, + status: state.status, + channel: state.channel, + behind: offered ? "yes" : Option.isSome(lastCheck) && lastCheck.value.ok ? "no" : "unknown", + stageAgeMs: Math.max(0, now - times.statusSince), + ...Option.match(lastSuccessfulCheckAt, { + onNone: () => ({}), + onSome: (at) => ({ lastCheckAgeMs: Math.max(0, now - at) }), + }), + ...(state.status === "error" && state.errorContext !== null + ? { errorContext: state.errorContext } + : {}), + ...(state.status === "downloading" + ? { + downloadPercent: state.downloadPercent ?? 0, + sinceProgressMs: Math.max(0, now - times.progressAt), + } + : {}), + ...(state.status === "downloaded" + ? { readyAgeMs: Math.max(0, now - times.downloadedAt) } + : {}), + ...(state.status === "disabled" + ? { disabledClass: disabledClass(state, options.policyOff) } + : {}), + }); + }); + + return { + checkStarted: (reason: string) => + begin("check", null).pipe( + Effect.provideService(CurrentUpdateTrigger, checkTrigger(reason)), + Effect.asVoid, + ), + checkFinished: (state: DesktopUpdateState) => + finish("check", state.availableVersion === null ? "up-to-date" : "available", { + targetVersion: state.availableVersion, + }), + checkFailed: (cause: unknown) => finish("check", "failed", { cause }), + downloadStarted: (targetVersion: string | null) => + begin("download", targetVersion).pipe( + Effect.flatMap((attempt) => outcome("download", attempt, "started")), + ), + downloadCompleted: finish("download", "completed"), + downloadFailed: (cause: unknown) => finish("download", "failed", { cause }), + /** Starts an install attempt; its reference goes in the install record for the next launch. */ + installStarted: (targetVersion: string) => + Option.isNone(options.facts) + ? Effect.undefined + : begin("install", targetVersion).pipe(Effect.map((attempt) => attempt.ref)), + /** Before the installer takes over: the app may exit before anything else is recorded. */ + installHandoff: Ref.get(attempts).pipe( + Effect.flatMap((current) => + current.install === undefined + ? Effect.void + : outcome("install", current.install, "handoff-attempted"), + ), + ), + installFailed: (cause: unknown) => finish("install", "failed", { cause }), + applied: Effect.fn("DesktopUpdateOutcomes.applied")(function* ( + install: UpdateInstallRecord, + runningVersion: string, + ) { + const result = applyOutcome(install, runningVersion); + if (result === undefined) return; + const now = yield* Clock.currentTimeMillis; + yield* record("app.update.outcome", { + step: "apply", + outcome: result, + targetVersion: install.targetVersion, + ...context, + ...(install.attemptRef === undefined ? {} : { attemptRef: install.attemptRef }), + ...(install.startedAt === undefined + ? {} + : { durationMs: Math.max(0, now - install.startedAt) }), + }); + }), + /** Summaries at launch, when the status settles somewhere new, and once a day. */ + summaries: Effect.fn("DesktopUpdateOutcomes.summaries")(function* ( + latest: DesktopUpdateState, + changes: Stream.Stream, + ) { + if (Option.isNone(options.facts)) return; + const startedAt = yield* Clock.currentTimeMillis; + const stateRef = yield* Ref.make(latest); + const timesRef = yield* Ref.make({ + key: summaryKey(latest), + statusSince: startedAt, + progressAt: startedAt, + downloadedAt: startedAt, + }); + yield* summary("launch", latest, yield* Ref.get(timesRef)); + yield* Effect.sleep(SUMMARY_INTERVAL).pipe( + Effect.andThen( + Effect.all([Ref.get(stateRef), Ref.get(timesRef)]).pipe( + Effect.flatMap(([state, times]) => summary("daily", state, times)), + ), + ), + Effect.forever, + Effect.forkScoped, + ); + return yield* Stream.runForEach(changes, (state) => + Effect.gen(function* () { + const previous = yield* Ref.getAndSet(stateRef, state); + const now = yield* Clock.currentTimeMillis; + const key = summaryKey(state); + const [changed, times] = yield* Ref.modify(timesRef, (current) => { + const changed = key !== undefined && key !== current.key; + const next: SummaryTimes = { + key: changed ? key : current.key, + statusSince: changed ? now : current.statusSince, + progressAt: + state.status === "downloading" && + (previous.status !== "downloading" || + previous.downloadPercent !== state.downloadPercent) + ? now + : current.progressAt, + downloadedAt: + state.downloadedVersion !== null && + previous.downloadedVersion !== state.downloadedVersion + ? now + : current.downloadedAt, + }; + return [[changed, next] as const, next]; + }); + if (changed) yield* summary("change", state, times); + }), + ); + }), + }; +}); + +interface SummaryTimes { + readonly key: string | undefined; + readonly statusSince: number; + readonly progressAt: number; + readonly downloadedAt: number; +} + +export type DesktopUpdateOutcomes = Effect.Success>; + +/** A defect in the update code itself is internal, not an updater or transport failure. */ +function classifyFailure(cause: unknown): UpdateFailureClass { + return typeof cause === "object" && + cause !== null && + (cause as { readonly _tag?: unknown })._tag === "DesktopUpdateUnexpectedActionError" + ? "internal" + : classifyUpdateFailure(cause); +} diff --git a/apps/desktop/src/scient/updateOutcomes/updateFailureClass.test.ts b/apps/desktop/src/scient/updateOutcomes/updateFailureClass.test.ts new file mode 100644 index 0000000000..a610832dcd --- /dev/null +++ b/apps/desktop/src/scient/updateOutcomes/updateFailureClass.test.ts @@ -0,0 +1,80 @@ +import * as NodeModule from "node:module"; + +import { describe, expect, it } from "@effect/vitest"; + +import { classifyUpdateFailure } from "./updateFailureClass.ts"; + +// electron-updater's own error helpers, which apps/desktop does not declare. +const requireFromUpdater = NodeModule.createRequire( + NodeModule.createRequire(import.meta.url).resolve("electron-updater"), +); +const { newError, HttpError, CancellationError } = requireFromUpdater("builder-util-runtime") as { + newError: (message: string, code: string) => Error; + HttpError: new (statusCode: number, message?: string) => Error; + CancellationError: new () => Error; +}; + +const nodeError = (code: string, message: string) => Object.assign(new Error(message), { code }); + +describe("classifyUpdateFailure", () => { + it("classifies the errors electron-updater 6.8 reports", () => { + const cases: ReadonlyArray = [ + // GitHubProvider wraps the transport failure in a feed code; the transport cause wins. + [ + newError( + "Unable to find latest version on GitHub (https://github.com/x/y/releases/latest), please ensure a production release exists: Error: net::ERR_INTERNET_DISCONNECTED", + "ERR_UPDATER_LATEST_VERSION_NOT_FOUND", + ), + "network", + ], + [ + newError( + `Unable to find latest version on GitHub (https://github.com/x/y/releases/latest), please ensure a production release exists: HttpError: 403 "rate limited"`, + "ERR_UPDATER_LATEST_VERSION_NOT_FOUND", + ), + "http_4xx", + ], + // A release without its channel file is an incomplete feed, not a transport failure. + [ + newError( + "Cannot find latest-mac.yml in the latest release artifacts (https://github.com/x/y/releases/download/v1/latest-mac.yml): HttpError: 404 ", + "ERR_UPDATER_CHANNEL_FILE_NOT_FOUND", + ), + "feed", + ], + [new HttpError(503), "http_5xx"], + [ + new Error( + 'Cannot download "https://objects.githubusercontent.com/x", status 502: Bad Gateway', + ), + "http_5xx", + ], + [ + newError( + "New version 1.2.4 is not signed by the application owner: publisherNames: Scient, raw info: {}", + "ERR_UPDATER_INVALID_SIGNATURE", + ), + "signature", + ], + [ + new Error( + "Code signature at URL file:///private/var/update.zip did not pass validation: code object is not signed at all", + ), + "signature", + ], + [ + newError("sha512 checksum mismatch, expected a, got b", "ERR_CHECKSUM_MISMATCH"), + "checksum", + ], + [nodeError("ENOSPC", "ENOSPC: no space left on device, write"), "disk_full"], + [nodeError("EACCES", "spawn EACCES"), "permission"], + [new Error("net::ERR_CERT_AUTHORITY_INVALID"), "certificate"], + [nodeError("ETIMEDOUT", "connect ETIMEDOUT 140.82.112.3:443"), "timeout"], + [new CancellationError(), "cancelled"], + [new Error("something else entirely"), "unknown"], + ]; + for (const [error, expected] of cases) { + expect([String(error), classifyUpdateFailure(error)]).toEqual([String(error), expected]); + } + }); +}); diff --git a/apps/desktop/src/scient/updateOutcomes/updateFailureClass.ts b/apps/desktop/src/scient/updateOutcomes/updateFailureClass.ts new file mode 100644 index 0000000000..48826f5f07 --- /dev/null +++ b/apps/desktop/src/scient/updateOutcomes/updateFailureClass.ts @@ -0,0 +1,135 @@ +import * as Cause from "effect/Cause"; + +/** Why an update step failed, from the updater's own error codes, statuses and Node/Chromium errors. */ +export type UpdateFailureClass = + | "network" + | "timeout" + | "certificate" + | "http_4xx" + | "http_5xx" + | "feed" + | "checksum" + | "signature" + | "disk_full" + | "permission" + | "configuration" + | "cancelled" + | "internal" + | "unknown"; + +const FEED_CODES = new Set([ + "ERR_UPDATER_CHANNEL_FILE_NOT_FOUND", + "ERR_UPDATER_LATEST_VERSION_NOT_FOUND", + "ERR_UPDATER_NO_PUBLISHED_VERSIONS", + "ERR_UPDATER_RELEASE_NOT_FOUND", + "ERR_UPDATER_ASSET_NOT_FOUND", + "ERR_UPDATER_ZIP_FILE_NOT_FOUND", + "ERR_UPDATER_INVALID_UPDATE_INFO", + "ERR_UPDATER_INVALID_RELEASE_FEED", + "ERR_UPDATER_NO_FILES_PROVIDED", + "ERR_UPDATER_NO_CHECKSUM", + "ERR_UPDATER_INVALID_VERSION", +]); +const CONFIGURATION_CODES = new Set([ + "ERR_UPDATER_INVALID_PROVIDER_CONFIGURATION", + "ERR_UPDATER_UNSUPPORTED_PROVIDER", + "ERR_UPDATER_INVALID_CHANNEL", + "ERR_UPDATER_WEB_INSTALLER_DISABLED", +]); + +const SIGNATURE = /not signed by the application owner|code ?signature|codesign/iu; +const CHECKSUM = /checksum mismatch/iu; +const DISK_FULL = /\bENOSPC\b|no space left/iu; +const PERMISSION = /\b(?:EACCES|EPERM|EROFS)\b|permission denied|operation not permitted/iu; +const CERTIFICATE = + /net::ERR_(?:CERT_|SSL_)|\b(?:CERT_[A-Z_]+|UNABLE_TO_VERIFY_LEAF_SIGNATURE|UNABLE_TO_GET_ISSUER_CERT(?:_LOCALLY)?|SELF_SIGNED_CERT_IN_CHAIN|DEPTH_ZERO_SELF_SIGNED_CERT|ERR_TLS_CERT_ALTNAME_INVALID)\b/u; +const TIMEOUT = + /\b(?:ETIMEDOUT|ESOCKETTIMEDOUT)\b|net::ERR_(?:CONNECTION_)?TIMED_OUT|\btimed? ?out\b/iu; +const NETWORK = + /\b(?:ENOTFOUND|ECONNRESET|ECONNREFUSED|ECONNABORTED|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH|ENETDOWN|EPIPE)\b|net::ERR_[A-Z0-9_]+/u; +const HTTP_STATUS = /\b(?:HttpError: |status )([1-5]\d\d)\b/u; + +interface ErrorFacts { + readonly codes: ReadonlyArray; + readonly statuses: ReadonlyArray; + readonly names: ReadonlyArray; + /** Local only: classified here and never stored or sent. */ + readonly text: string; +} + +function collect(cause: unknown): ErrorFacts { + const codes: string[] = []; + const statuses: number[] = []; + const names: string[] = []; + const texts: string[] = []; + const visit = (value: unknown, depth: number): void => { + if (depth > 4 || value === null || value === undefined) return; + if (Cause.isCause(value)) { + for (const reason of value.reasons) { + if (reason._tag === "Fail") visit(reason.error, depth + 1); + else if (reason._tag === "Die") visit(reason.defect, depth + 1); + } + return; + } + if (typeof value === "string") { + texts.push(value); + return; + } + if (typeof value !== "object") return; + const error = value as { + readonly code?: unknown; + readonly statusCode?: unknown; + readonly name?: unknown; + readonly message?: unknown; + readonly cause?: unknown; + }; + if (typeof error.code === "string") codes.push(error.code); + if (typeof error.statusCode === "number") statuses.push(error.statusCode); + if (typeof error.name === "string") names.push(error.name); + // builder-util-runtime's CancellationError keeps the default name. + if (typeof value.constructor?.name === "string") names.push(value.constructor.name); + if (typeof error.message === "string") texts.push(error.message); + visit(error.cause, depth + 1); + }; + visit(cause, 0); + return { codes, statuses, names, text: texts.join("\n") }; +} + +function httpStatus(facts: ErrorFacts): number | undefined { + const status = + facts.statuses.find((value) => value >= 100 && value <= 599) ?? + facts.codes.map((code) => /^HTTP_ERROR_(\d{3})$/u.exec(code)?.[1]).find(Boolean) ?? + HTTP_STATUS.exec(facts.text)?.[1]; + return status === undefined ? undefined : Number(status); +} + +/** Classifies an electron-updater failure; feed errors wrap transport causes, so those win. */ +export function classifyUpdateFailure(cause: unknown): UpdateFailureClass { + try { + const facts = collect(cause); + const has = (code: string) => facts.codes.includes(code); + if (has("ERR_UPDATER_INVALID_SIGNATURE") || SIGNATURE.test(facts.text)) return "signature"; + if (has("ERR_CHECKSUM_MISMATCH") || CHECKSUM.test(facts.text)) return "checksum"; + if (has("ENOSPC") || DISK_FULL.test(facts.text)) return "disk_full"; + if (["EACCES", "EPERM", "EROFS"].some(has) || PERMISSION.test(facts.text)) return "permission"; + if (CERTIFICATE.test(facts.text) || facts.codes.some((code) => CERTIFICATE.test(code))) { + return "certificate"; + } + if (TIMEOUT.test(facts.text) || facts.codes.some((code) => TIMEOUT.test(code))) { + return "timeout"; + } + if (NETWORK.test(facts.text) || facts.codes.some((code) => NETWORK.test(code))) { + return "network"; + } + const feed = facts.codes.some((code) => FEED_CODES.has(code)); + const status = httpStatus(facts); + if (feed && (status === undefined || status === 404)) return "feed"; + if (status !== undefined && status >= 400) return status >= 500 ? "http_5xx" : "http_4xx"; + if (feed) return "feed"; + if (facts.codes.some((code) => CONFIGURATION_CODES.has(code))) return "configuration"; + if (facts.names.includes("CancellationError")) return "cancelled"; + return "unknown"; + } catch { + return "unknown"; + } +} diff --git a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts index c6868aec83..28b93af1c1 100644 --- a/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts +++ b/apps/desktop/src/telemetry/DesktopTelemetryPublisher.ts @@ -26,6 +26,9 @@ import * as ElectronApp from "../electron/ElectronApp.ts"; import * as ElectronPowerMonitor from "../electron/ElectronPowerMonitor.ts"; import * as DesktopRendererHistory from "./DesktopRendererHistory.ts"; import * as DesktopShellEnvironment from "../shell/DesktopShellEnvironment.ts"; +// SCIENT-FORK:START — journaled update facts reach one backend's analytics. +import * as DesktopUpdateFacts from "../scient/updateOutcomes/DesktopUpdateFacts.ts"; +// SCIENT-FORK:END const LIVE_SAMPLE_INTERVAL = Duration.seconds(1); const BATTERY_SAMPLE_INTERVAL = Duration.seconds(5); @@ -66,6 +69,10 @@ export class DesktopTelemetryPublisher extends Context.Service< readonly latest: Effect.Effect>; readonly changes: Stream.Stream; readonly encoded: Stream.Stream; + // SCIENT-FORK:START — one backend's stream, which also carries the update facts offered to it; + // without it a backend reads `encoded`. + readonly encodedFor?: (sourceId: string) => Stream.Stream; + // SCIENT-FORK:END readonly handleControlForSource: ( sourceId: string, message: DesktopTelemetryControlMessage, @@ -150,6 +157,8 @@ export const make = Effect.fn("desktop.telemetryPublisher.make")(function* () { const powerMonitor = yield* ElectronPowerMonitor.ElectronPowerMonitor; const rendererHistory = yield* DesktopRendererHistory.DesktopRendererHistory; const shellEnvironment = yield* DesktopShellEnvironment.DesktopShellEnvironment; + // SCIENT-FORK: absent in tests and tools that never deliver update facts. + const updateFacts = yield* Effect.serviceOption(DesktopUpdateFacts.DesktopUpdateFacts); yield* electronApp.whenReady; const initialPowerState: PowerState = { @@ -355,8 +364,18 @@ export const make = Effect.fn("desktop.telemetryPublisher.make")(function* () { return Queue.offer(updateCommitQueue, message).pipe(Effect.asVoid); case "cancelDesktopUpdate": return Queue.offer(updateCancellationQueue, message).pipe(Effect.asVoid); + // SCIENT-FORK:START — a ready backend can take journaled update facts. case "serverReady": - return Effect.void; + return Option.match(updateFacts, { + onNone: () => Effect.void, + onSome: (facts) => facts.sourceReady(sourceId), + }); + case "ackDesktopUpdateFacts": + return Option.match(updateFacts, { + onNone: () => Effect.void, + onSome: (facts) => facts.acknowledge(message.ids), + }); + // SCIENT-FORK:END } }; const removeControlSource: DesktopTelemetryPublisher["Service"]["removeControlSource"] = ( @@ -373,6 +392,13 @@ export const make = Effect.fn("desktop.telemetryPublisher.make")(function* () { ? Effect.void : Queue.offer(sampleTriggers, undefined).pipe(Effect.asVoid), ), + // SCIENT-FORK: pending update facts move to another ready backend. + Effect.andThen( + Option.match(updateFacts, { + onNone: () => Effect.void, + onSome: (facts) => facts.sourceRemoved(sourceId), + }), + ), ); const snapshots = Stream.unwrap( @@ -401,17 +427,47 @@ export const make = Effect.fn("desktop.telemetryPublisher.make")(function* () { ); }), ); - const encoded = Stream.concat( - Stream.make({ - version: 1, - type: "desktopTelemetryHello", - electronPid: process.pid, - } as const), - Stream.merge( - Stream.merge(Stream.merge(snapshots, updateReports), shellEnvironment.updates), - Stream.fromPubSub(appHealthChanges), - ), - ).pipe(Stream.map((message) => textEncoder.encode(`${encodeMessage(message)}\n`))); + // SCIENT-FORK:START — a backend's own stream adds the update facts offered to it. + const encodedWith = (extra: Stream.Stream) => + Stream.concat( + Stream.make({ + version: 1, + type: "desktopTelemetryHello", + electronPid: process.pid, + } as const), + Stream.merge( + Stream.merge( + Stream.merge(Stream.merge(snapshots, updateReports), shellEnvironment.updates), + Stream.fromPubSub(appHealthChanges), + ), + extra, + ), + ).pipe(Stream.map((message) => textEncoder.encode(`${encodeMessage(message)}\n`))); + const encoded = encodedWith(Stream.empty); + const encodedFor = (sourceId: string) => + encodedWith( + Option.match(updateFacts, { + onNone: () => Stream.empty, + onSome: (facts) => + facts.deliveries(sourceId).pipe( + Stream.mapEffect((delivered) => + Ref.get(latestUpdateReport).pipe( + Effect.map( + Option.map((report): DesktopUpdateStatusReport => ({ + version: 1, + type: "desktopUpdateStatus", + state: report.state, + facts: delivered, + })), + ), + ), + ), + Stream.filter(Option.isSome), + Stream.map((report) => report.value), + ), + }), + ); + // SCIENT-FORK:END const publishUpdateReport: DesktopTelemetryPublisher["Service"]["publishUpdateReport"] = ( report, @@ -425,6 +481,8 @@ export const make = Effect.fn("desktop.telemetryPublisher.make")(function* () { latest: Ref.get(latest), changes: Stream.fromPubSub(changes), encoded, + // SCIENT-FORK: per-backend stream. + encodedFor, handleControlForSource, removeControlSource, publishUpdateReport, diff --git a/apps/desktop/src/updates/DesktopUpdates.scient.test.ts b/apps/desktop/src/updates/DesktopUpdates.scient.test.ts new file mode 100644 index 0000000000..6f0ca2580d --- /dev/null +++ b/apps/desktop/src/updates/DesktopUpdates.scient.test.ts @@ -0,0 +1,172 @@ +import * as NodeModule from "node:module"; + +import { assert, describe, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; + +import * as ElectronUpdater from "../electron/ElectronUpdater.ts"; +import * as DesktopUpdateFacts from "../scient/updateOutcomes/DesktopUpdateFacts.ts"; +import * as DesktopUpdates from "./DesktopUpdates.ts"; +import { flushCallbacks, makeHarness, type UpdatesHarnessOptions } from "./updatesTestHarness.ts"; + +const requireFromUpdater = NodeModule.createRequire( + NodeModule.createRequire(import.meta.url).resolve("electron-updater"), +); +const { newError } = requireFromUpdater("builder-util-runtime") as { + newError: (message: string, code: string) => Error; +}; + +type Recorded = { readonly name: string; readonly properties: Record }; + +/** The updater with a journal that keeps what it was given. */ +function recordingHarness(options: UpdatesHarnessOptions = {}) { + const recorded: Recorded[] = []; + const harness = makeHarness(options); + const facts = Layer.succeed( + DesktopUpdateFacts.DesktopUpdateFacts, + DesktopUpdateFacts.DesktopUpdateFacts.of({ + record: (name, properties) => Effect.sync(() => void recorded.push({ name, properties })), + deliveries: () => Stream.empty, + sourceReady: () => Effect.void, + sourceRemoved: () => Effect.void, + acknowledge: () => Effect.void, + }), + ); + const outcomes = (step: string) => + recorded + .filter((fact) => fact.name === "app.update.outcome" && fact.properties.step === step) + .map((fact) => fact.properties); + return { + harness, + outcomes, + layer: Layer.merge(TestClock.layer(), harness.layer.pipe(Layer.provide(facts))), + }; +} + +describe("DesktopUpdates outcomes", () => { + it.effect("records one outcome for a check the updater reports twice", () => { + const offline = newError( + "Unable to find latest version on GitHub (https://github.com/x/y/releases/latest), please ensure a production release exists: Error: net::ERR_INTERNET_DISCONNECTED", + "ERR_UPDATER_LATEST_VERSION_NOT_FOUND", + ); + // electron-updater emits `error` and also rejects the check's promise. + const { harness, outcomes, layer } = recordingHarness({ + checkForUpdates: Effect.suspend(() => { + harness.emit("error", offline); + return Effect.fail( + new ElectronUpdater.ElectronUpdaterCheckForUpdatesError({ + channel: "latest", + cause: offline, + }), + ); + }), + }); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + yield* updates.check("menu"); + yield* flushCallbacks; + const checks = outcomes("check"); + assert.lengthOf(checks, 1); + assert.include(checks[0], { outcome: "failed", failureClass: "network", trigger: "user" }); + }), + ).pipe(Effect.provide(layer)); + }); + + it.effect("records one failed install after the handoff, however many paths report it", () => { + const refused = Object.assign(new Error("spawn EACCES"), { code: "EACCES" }); + const { harness, outcomes, layer } = recordingHarness({ + quitAndInstall: Effect.suspend(() => { + harness.emit("error", refused); + return Effect.fail( + new ElectronUpdater.ElectronUpdaterQuitAndInstallError({ + channel: "latest", + isSilent: true, + isForceRunAfter: true, + cause: refused, + }), + ); + }), + }); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* flushCallbacks; + yield* updates.install; + yield* flushCallbacks; + const installs = outcomes("install"); + assert.deepEqual( + installs.map((install) => install.outcome), + ["handoff-attempted", "failed"], + ); + assert.include(installs[1], { + failureClass: "permission", + targetVersion: "1.2.4", + attemptRef: installs[0]?.attemptRef, + }); + }), + ).pipe(Effect.provide(layer)); + }); + + it.effect("reports an install that did not apply once, not at every launch", () => + Effect.gen(function* () { + const record = JSON.stringify({ + fromVersion: "1.2.3", + targetVersion: "1.2.4", + startedAt: 0, + attemptRef: "0123456789abcdef", + }); + const first = recordingHarness({ installRecord: record }); + yield* Effect.scoped( + DesktopUpdates.DesktopUpdates.pipe(Effect.flatMap((updates) => updates.configure)), + ).pipe(Effect.provide(first.layer)); + assert.deepEqual(first.outcomes("apply"), [ + { + step: "apply", + outcome: "not-applied", + targetVersion: "1.2.4", + packageKind: "zip", + attemptRef: "0123456789abcdef", + durationMs: 0, + }, + ]); + const kept = [...first.harness.installRecords.values()][0]; + assert.isDefined(kept); + + const second = recordingHarness({ installRecord: kept }); + yield* Effect.scoped( + DesktopUpdates.DesktopUpdates.pipe(Effect.flatMap((updates) => updates.configure)), + ).pipe(Effect.provide(second.layer)); + assert.include(kept, '"outcomeReported":true'); + assert.lengthOf(second.outcomes("apply"), 0); + }), + ); + + it.effect("keeps polling after a poll fails unexpectedly", () => { + let checks = 0; + const { harness, layer } = recordingHarness({ + checkForUpdates: Effect.suspend(() => + ++checks === 2 ? Effect.die(new Error("unexpected poll failure")) : Effect.void, + ), + }); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + yield* TestClock.adjust("15 seconds"); + yield* TestClock.adjust("4 hours"); + assert.equal(harness.checkCount(), 2); + yield* TestClock.adjust("4 hours"); + assert.equal(harness.checkCount(), 3); + }), + ).pipe(Effect.provide(layer)); + }); +}); diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index c9693b2507..f6cda6c961 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -37,6 +37,10 @@ import * as DesktopState from "../app/DesktopState.ts"; import * as ElectronUpdater from "../electron/ElectronUpdater.ts"; import * as ElectronWindow from "../electron/ElectronWindow.ts"; import * as IpcChannels from "../ipc/channels.ts"; +// SCIENT-FORK:START — update outcomes and summaries for product analytics. +import * as DesktopUpdateFacts from "../scient/updateOutcomes/DesktopUpdateFacts.ts"; +import * as DesktopUpdateOutcomes from "../scient/updateOutcomes/DesktopUpdateOutcomes.ts"; +// SCIENT-FORK:END import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts"; import * as DesktopSshEnvironment from "../ssh/DesktopSshEnvironment.ts"; import * as DesktopUpdatePolicy from "./DesktopUpdatePolicy.ts"; @@ -108,8 +112,12 @@ const DesktopUpdateInstallRecord = Schema.fromJsonString( Schema.Struct({ fromVersion: Schema.String, targetVersion: Schema.String, - // SCIENT-FORK: when the install began, so the next launch can report how long it took. + // SCIENT-FORK:START — when the install began, so the next launch can report how long it + // took; which attempt it was, and whether that launch's outcome was already recorded. startedAt: Schema.optionalKey(Schema.Number), + attemptRef: Schema.optionalKey(Schema.String), + outcomeReported: Schema.optionalKey(Schema.Boolean), + // SCIENT-FORK:END }), ); const decodeInstallRecord = Schema.decodeUnknownEffect(DesktopUpdateInstallRecord); @@ -490,6 +498,24 @@ export const make = Effect.gen(function* () { ) : false; + // SCIENT-FORK:START — update outcomes and summaries for product analytics. + const outcomes = yield* DesktopUpdateOutcomes.make({ + facts: yield* Effect.serviceOption(DesktopUpdateFacts.DesktopUpdateFacts), + packageKind: DesktopUpdateOutcomes.updatePackageKind({ + platform: environment.platform, + appImage: Option.isSome(config.appImagePath), + debPackage: isDebPackage, + }), + sourceKind: mockUpdates + ? undefined + : DesktopUpdateOutcomes.updateSourceKind({ + administratorFeed: policy.feedUrl !== undefined, + defaultProvider: scientDesktopUpdateFeed("latest").provider, + }), + policyOff: policy.mode === "off", + }); + // SCIENT-FORK:END + const hasUpdateFeedConfig = Ref.get(appUpdateYmlConfigRef).pipe( Effect.map( (appUpdateYmlConfig) => @@ -632,9 +658,13 @@ export const make = Effect.gen(function* () { yield* setState(reduceDesktopUpdateStateOnCheckStart(state, yield* currentIsoTimestamp)); } yield* logUpdaterInfo("checking for updates", { reason }); + // SCIENT-FORK: one outcome per check. + yield* outcomes.checkStarted(reason); return yield* runUpdaterCheck(allowDowngrade).pipe( Effect.as(true), + // SCIENT-FORK: the updater's events applied the check's result before it resolved. + Effect.tap(() => Ref.get(updateStateRef).pipe(Effect.flatMap(outcomes.checkFinished))), Effect.catchTags({ ElectronUpdaterCheckForUpdatesError: Effect.fn( "desktop.updates.handleCheckForUpdatesFailure", @@ -645,12 +675,16 @@ export const make = Effect.gen(function* () { reduceDesktopUpdateStateOnNoUpdate(current, checkedAt), ); yield* logUpdaterInfo("no beta releases published yet"); + // SCIENT-FORK: an empty Beta feed is a completed check. + yield* outcomes.checkFinished(yield* Ref.get(updateStateRef)); return true; } const failedAt = yield* currentIsoTimestamp; yield* updateState((current) => reduceDesktopUpdateStateOnCheckFailure(current, error.message, failedAt), ); + // SCIENT-FORK: the rejected check, not the updater's matching error event, is its outcome. + yield* outcomes.checkFailed(error.cause); yield* logUpdaterError(error.message, { errorTag: error._tag, cause: describeUpdateErrorCause(error.cause), @@ -720,11 +754,15 @@ export const make = Effect.gen(function* () { return { accepted: true, completed: false }; } yield* setState(reduceDesktopUpdateStateOnDownloadStart(state)); + // SCIENT-FORK: a download that never ends still counts as started. + yield* outcomes.downloadStarted(state.availableVersion); yield* electronUpdater.setDisableDifferentialDownload( isArm64HostRunningIntelBuild(environment.runtimeInfo), ); yield* logUpdaterInfo("downloading update"); yield* electronUpdater.downloadUpdate; + // SCIENT-FORK: one outcome per download. + yield* outcomes.downloadCompleted; return { accepted: true, completed: true }; }).pipe( Effect.catchTags({ @@ -734,6 +772,8 @@ export const make = Effect.gen(function* () { yield* updateState((current) => reduceDesktopUpdateStateOnDownloadFailure(current, `${error.message} ${cause}`), ); + // SCIENT-FORK: one outcome per download. + yield* outcomes.downloadFailed(error.cause); yield* logUpdaterError(error.message, { errorTag: error._tag, cause, @@ -757,6 +797,8 @@ export const make = Effect.gen(function* () { yield* updateState((current) => reduceDesktopUpdateStateOnDownloadFailure(current, error.message), ); + // SCIENT-FORK: one outcome per download. + yield* outcomes.downloadFailed(error); yield* logUpdaterError(error.message, { errorTag: error._tag, cause: describeUpdateErrorCause(error.cause), @@ -805,7 +847,9 @@ export const make = Effect.gen(function* () { continue; } // A check offered this update just now, so the download needs no refresh first. - const result = yield* downloadAvailableUpdate(false); + const result = yield* downloadAvailableUpdate(false).pipe( + Effect.provideService(DesktopUpdateOutcomes.CurrentUpdateTrigger, "background"), + ); if (result.completed) { yield* Ref.set(autoDownloadBackoffRef, Option.none()); return; @@ -915,16 +959,26 @@ export const make = Effect.gen(function* () { updateRestartMarkerDir, DESKTOP_UPDATE_INSTALL_RECORD_FILE, ); - const writeInstallRecord = (targetVersion: string) => - Clock.currentTimeMillis.pipe( - Effect.flatMap((startedAt) => - encodeInstallRecord({ fromVersion: environment.appVersion, targetVersion, startedAt }), - ), + // SCIENT-FORK:START — the record also names the install attempt. + const saveInstallRecord = (record: typeof DesktopUpdateInstallRecord.Type) => + encodeInstallRecord(record).pipe( Effect.flatMap((text) => fileSystem.writeFileString(installRecordPath, text)), Effect.catch((error) => logUpdaterWarning("Could not record the update install.", { errorTag: error._tag }), ), ); + const writeInstallRecord = (targetVersion: string, attemptRef: string | undefined) => + Clock.currentTimeMillis.pipe( + Effect.flatMap((startedAt) => + saveInstallRecord({ + fromVersion: environment.appVersion, + targetVersion, + startedAt, + ...(attemptRef === undefined ? {} : { attemptRef }), + }), + ), + ); + // SCIENT-FORK:END const removeInstallRecord = fileSystem .remove(installRecordPath, { force: true }) .pipe(Effect.ignore); @@ -947,12 +1001,31 @@ export const make = Effect.gen(function* () { ), ) : Effect.void; + // A record that did not apply stays for the next launch's notice, so it is reported once. + const reportInstall = (record: typeof DesktopUpdateInstallRecord.Type) => + logAppliedInstall(record).pipe( + Effect.andThen( + record.outcomeReported === true + ? Effect.void + : outcomes + .applied(record, environment.appVersion) + .pipe( + Effect.andThen( + DesktopUpdateOutcomes.applyOutcome(record, environment.appVersion) === + "not-applied" + ? saveInstallRecord({ ...record, outcomeReported: true }) + : Effect.void, + ), + ), + ), + ); // SCIENT-FORK:END const readNotAppliedInstallVersion = fileSystem.readFileString(installRecordPath).pipe( Effect.flatMap(decodeInstallRecord), Effect.option, - Effect.tap((record) => (Option.isSome(record) ? logAppliedInstall(record.value) : Effect.void)), + // SCIENT-FORK: the outcome of the install a previous run handed over. + Effect.tap((record) => (Option.isSome(record) ? reportInstall(record.value) : Effect.void)), Effect.map((record) => Option.flatMap(record, (value) => Option.fromNullishOr(resolveNotAppliedInstallVersion(value, environment.appVersion)), @@ -1069,7 +1142,13 @@ export const make = Effect.gen(function* () { yield* writeUpdateRestartMarker; yield* writeUpdateInProgressMarker; const { downloadedVersion } = yield* Ref.get(updateStateRef); - if (downloadedVersion !== null) yield* writeInstallRecord(downloadedVersion); + // SCIENT-FORK: the install attempt starts with its record. + if (downloadedVersion !== null) { + yield* writeInstallRecord( + downloadedVersion, + yield* outcomes.installStarted(downloadedVersion), + ); + } // Stop pooled backends and SSH tunnels before quitAndInstall can outrun their scope finalizers. const instances = yield* pool.list; // stop clears desiredRunning, so retain the recovery set beforehand. @@ -1089,6 +1168,8 @@ export const make = Effect.gen(function* () { ], { concurrency: "unbounded" }, ); + // SCIENT-FORK: the installer may end this process before anything else is recorded. + yield* outcomes.installHandoff; yield* electronUpdater.quitAndInstall({ isSilent: true, isForceRunAfter: true, @@ -1098,6 +1179,8 @@ export const make = Effect.gen(function* () { Effect.catchTags({ ElectronUpdaterQuitAndInstallError: Effect.fn("desktop.updates.handleInstallFailure")( function* (error) { + // SCIENT-FORK: one outcome per install, however many paths report its failure. + yield* outcomes.installFailed(error.cause); yield* recoverFailedInstall(error.message); yield* logUpdaterError(error.message, { errorTag: error._tag, @@ -1117,6 +1200,8 @@ export const make = Effect.gen(function* () { return yield* Effect.failCause(cause); } const error = new DesktopUpdateUnexpectedActionError({ action: "install", cause }); + // SCIENT-FORK: one outcome per install, however many paths report its failure. + yield* outcomes.installFailed(error); yield* recoverFailedInstall(error.message); yield* logUpdaterError(error.message, { errorTag: error._tag, @@ -1204,10 +1289,10 @@ export const make = Effect.gen(function* () { ); yield* Effect.sleep(AUTO_UPDATE_POLL_INTERVAL).pipe( Effect.andThen(checkForUpdates("poll")), - Effect.forever, + // SCIENT-FORK:START — an unexpected failure ends one poll, not polling. Effect.catchCause((cause) => { if (Cause.hasInterruptsOnly(cause)) { - return Effect.void; + return Effect.failCause(cause); } const error = new DesktopUpdatePollerError({ poller: "poll", cause }); return logUpdaterError(error.message, { @@ -1216,6 +1301,8 @@ export const make = Effect.gen(function* () { poller: error.poller, }); }), + Effect.forever, + // SCIENT-FORK:END Effect.forkScoped, ); }).pipe(Effect.withSpan("desktop.updates.startPollers")); @@ -1301,6 +1388,8 @@ export const make = Effect.gen(function* () { cause, }); if (Option.isSome(activeAction) && activeAction.value === "install") { + // SCIENT-FORK: installers report failure here; one outcome per install. + yield* outcomes.installFailed(cause); yield* recoverFailedInstall(error.message); yield* logUpdaterError(error.message, { errorTag: error._tag, @@ -1464,6 +1553,12 @@ export const make = Effect.gen(function* () { Option.orElse(disabledReason, () => channelBlockedReason(settings.updateChannel)), ), ); + // SCIENT-FORK:START — summaries for the app's lifetime, also when updates are off. + const summaryChanges = yield* PubSub.subscribe(stateChanges); + yield* outcomes + .summaries(yield* Ref.get(updateStateRef), Stream.fromSubscription(summaryChanges)) + .pipe(Effect.forkScoped); + // SCIENT-FORK:END if (Option.isSome(disabledReason)) { return; } diff --git a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts index 9acf95990f..a0c643040c 100644 --- a/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts +++ b/apps/server/src/resourceTelemetry/DesktopTelemetryReceiver.ts @@ -37,6 +37,9 @@ import { eventVersion as messageVersion } from "./NativeTelemetryClient.ts"; import * as ServerConfig from "../config.ts"; import * as ServerSettings from "../serverSettings.ts"; import { AnalyticsService } from "../telemetry/AnalyticsService.ts"; +// SCIENT-FORK:START — journaled desktop update facts +import { ingestDesktopUpdateFacts } from "../telemetry/DesktopUpdateFacts.ts"; +// SCIENT-FORK:END import { subscribeBeforeSnapshotWithoutMutex } from "../utils/subscribeBeforeSnapshot.ts"; const INITIAL_SAMPLE_DEADLINE_MS = 90_000; @@ -604,6 +607,25 @@ export const make = Effect.fn("resourceTelemetry.desktopTelemetryReceiver.make") return recordContact.pipe(Effect.andThen(applyShellSnapshot(message))); } + // SCIENT-FORK:START — a report carrying facts delivers them; the desktop forgets them only + // after this backend's outbox holds them, and their status is not a change. + if (message.type === "desktopUpdateStatus" && message.facts !== undefined) { + const facts = message.facts; + return recordContact.pipe( + Effect.andThen( + ingestDesktopUpdateFacts(analytics, facts).pipe( + Effect.flatMap((ids) => + ids.length === 0 + ? Effect.void + : sendControlMessage({ version: 1, type: "ackDesktopUpdateFacts", ids }), + ), + Effect.ignoreCause(), + ), + ), + Effect.asVoid, + ); + } + // SCIENT-FORK:END if (message.type === "desktopUpdateStatus") { return recordContact.pipe( Effect.andThen(Ref.set(latestUpdateReport, Option.some(message))), diff --git a/apps/server/src/telemetry/AnalyticsServiceControls.test.ts b/apps/server/src/telemetry/AnalyticsServiceControls.test.ts index 8aa9e74f0c..ac96a4a3d2 100644 --- a/apps/server/src/telemetry/AnalyticsServiceControls.test.ts +++ b/apps/server/src/telemetry/AnalyticsServiceControls.test.ts @@ -35,6 +35,7 @@ function fakeRuntime(enabled = true) { } return alive; }, + ingest: async () => true, flush: async () => 0, setConsent: async () => 0, pendingCount: async () => 0, diff --git a/apps/server/src/telemetry/DesktopUpdateFacts.test.ts b/apps/server/src/telemetry/DesktopUpdateFacts.test.ts new file mode 100644 index 0000000000..f9813071c2 --- /dev/null +++ b/apps/server/src/telemetry/DesktopUpdateFacts.test.ts @@ -0,0 +1,34 @@ +import type { DesktopUpdateFact } from "@t3tools/contracts"; +import { assert, describe, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import { ingestDesktopUpdateFacts } from "./DesktopUpdateFacts.ts"; +import { disabledAnalyticsService } from "./ScientAnalytics.ts"; + +const fact: DesktopUpdateFact = { + id: "6f1c2a9e-4b7d-4e21-9a3f-0c5d8e7b1a24", + name: "app.update.outcome", + occurredAt: "2026-11-16T10:00:00.000Z", + appVersion: "0.6.29", + buildChannel: "stable", + properties: { step: "install", outcome: "handoff-attempted" }, +}; + +describe("ingestDesktopUpdateFacts", () => { + it.effect("acknowledges facts only once the outbox settled them", () => + Effect.gen(function* () { + const unconfirmed = yield* ingestDesktopUpdateFacts( + Option.some({ ...disabledAnalyticsService, ingestRecorded: () => Effect.succeed(false) }), + [fact], + ); + assert.deepEqual(unconfirmed, []); + + const persisted = yield* ingestDesktopUpdateFacts( + Option.some({ ...disabledAnalyticsService, ingestRecorded: () => Effect.succeed(true) }), + [fact], + ); + assert.deepEqual(persisted, [fact.id]); + }), + ); +}); diff --git a/apps/server/src/telemetry/DesktopUpdateFacts.ts b/apps/server/src/telemetry/DesktopUpdateFacts.ts new file mode 100644 index 0000000000..16370ea688 --- /dev/null +++ b/apps/server/src/telemetry/DesktopUpdateFacts.ts @@ -0,0 +1,22 @@ +/** + * Update facts the desktop app journaled (outcomes of check, download, install and apply, and + * update summaries). They keep the desktop's event ids and times, so a fact offered again after a + * lost acknowledgement stays one event in the outbox and at the gateway. + * + * @module DesktopUpdateFacts + */ +import type { DesktopUpdateFact } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import type { AnalyticsService } from "./AnalyticsService.ts"; + +/** The ids the desktop may forget: held by the outbox, or never to be sent from here. */ +export const ingestDesktopUpdateFacts = Effect.fn("DesktopUpdateFacts.ingest")(function* ( + analytics: Option.Option, + facts: ReadonlyArray, +) { + const ingest = Option.getOrUndefined(analytics)?.ingestRecorded; + const settled = ingest === undefined ? true : yield* ingest(facts); + return settled ? facts.map((fact) => fact.id) : []; +}); diff --git a/apps/server/src/telemetry/ScientAnalytics.ts b/apps/server/src/telemetry/ScientAnalytics.ts index a366c50a8e..228ff3f5c1 100644 --- a/apps/server/src/telemetry/ScientAnalytics.ts +++ b/apps/server/src/telemetry/ScientAnalytics.ts @@ -8,6 +8,7 @@ import { AnalyticsConsent, createAnalyticsRuntime, type AnalyticsRuntime, + type RecordedAnalyticsEvent, } from "@scientfactory/analytics"; import * as NodePath from "node:path"; import * as NodeProcess from "node:process"; @@ -61,6 +62,14 @@ export interface ScientAnalyticsControls { /** Internal observer fence; never part of a wire event or public identity. */ readonly collectionEpoch: Effect.Effect; + /** + * Queues events another Scient process recorded, keeping their ids and times. True once the + * outbox holds them or analytics must never send them; false asks the sender to offer again. + */ + readonly ingestRecorded?: ( + events: ReadonlyArray, + ) => Effect.Effect; + /** In-memory boundary watermark acquisition; not telemetry or a public API. */ readonly observeCollectionBoundary?: ( listener: (epoch: number, phase: AnalyticsCollectionBoundaryPhase) => Promise, @@ -113,6 +122,7 @@ export const disabledAnalyticsService: AnalyticsService["Service"] = { record: () => Effect.void, recordOwners: () => Effect.void, flush: Effect.void, + ingestRecorded: () => Effect.succeed(true), status: Effect.succeed({ available: false, consent: "off" }), collectionEpoch: Effect.succeed(0), }; @@ -233,6 +243,12 @@ export const make = Effect.gen(function* () { Effect.asVoid, ); + const ingestRecorded: NonNullable = (events) => + Effect.tryPromise({ + try: () => runtime.ingest(events), + catch: () => "analytics-ingest-failed" as const, + }).pipe(Effect.orElseSucceed(() => false)); + const status: AnalyticsService["Service"]["status"] = Effect.sync(() => ({ available: analyticsConfig.enabled, consent: analyticsConfig.enabled ? consent : "off", @@ -242,6 +258,7 @@ export const make = Effect.gen(function* () { record, recordOwners, flush, + ingestRecorded, status, collectionEpoch: Effect.succeed(collectionEpoch), observeCollectionBoundary, diff --git a/docs/internals/product-analytics.md b/docs/internals/product-analytics.md index c0cb908bbc..b091ed5e37 100644 --- a/docs/internals/product-analytics.md +++ b/docs/internals/product-analytics.md @@ -515,7 +515,7 @@ kind. Its private summary and payload are never read; the failure class remains memory. This covers reported outcomes, not an unreported server crash. Registered operation kinds without producers are not usage evidence. Non-PDF -document/figure exports, desktop updater, +document/figure exports, migration, desktop main-process crash, and cloud/mobile coverage still require their own authoritative seams and proof. `client.connected`/`client.turn.requested` remain unregistered as described below. Failure categories not known from a safe typed @@ -754,6 +754,50 @@ The contract reads these fields with closed lists, so free text becomes `unknown The gateway accepts them unchanged; reference queries on the ledger live in the website's `workers/events/src/attributionPipeline.test.ts`. +## Desktop app updates + +The desktop updater records two Essential events, both fully closed: their normalizers read every +field, so nothing passes through as recorded. Versions are public release numbers or omitted; +URLs, hosts, paths, raw errors and the updater's `message` never enter them. + +`app.update.outcome` is one outcome per step attempt, with `step` and `outcome`: `check` +(`available`, `up-to-date`, `failed`), `download` (`started`, `completed`, `failed`), `install` +(`handoff-attempted`, `failed`) and `apply` (`completed`, `not-applied`, `superseded`). `trigger` +is `user` or `background` (startup and four-hourly checks, the background download). +`failureClass` (failed only) comes from electron-updater's codes, HTTP statuses and Node/Chromium +network errors: `network`, `timeout`, `certificate`, `http_4xx`, `http_5xx`, `feed` (the feed lacks +a channel file, release or asset), `checksum`, `signature`, `disk_full`, `permission`, +`configuration`, `cancelled`, `internal` or `unknown`; a feed error wrapping a transport cause +reports the transport cause. `targetVersion`, `sourceKind` (`github`, `scient_mirror`, +`administrator`), `packageKind` (`zip`, `nsis`, `appimage`, `deb`; omitted when unknown), a random +per-attempt `attemptRef` and `durationBucket` complete it. The rejected promise, not the updater's +matching `error` event, is a check's or download's outcome; an install records one failure +however many paths report it. `handoff-attempted` is recorded just before the installer takes +over; the next launch reports `apply` with the same `attemptRef`: `completed` when the target +runs, `not-applied` when the old version runs again (once, not at every launch), `superseded` +when another build runs. A handoff without a later outcome is unresolved, not a failure; there is +no timer that restarts servers after the handoff. + +`app.update.summary` is sent at launch, when the status settles somewhere new (a check in +progress is not a change) and daily while running: `reason`, `status`, `channel`, `behind` +(`yes` when a newer offer is known, `no` after a successful check found none, otherwise +`unknown`), `stageAgeBucket`, `lastCheckAgeBucket` (last successful check, or `never`), +`errorContext` on errors, `progressBucket` and `sinceProgressBucket` while downloading, +`readyAgeBucket` when downloaded, `disabledClass` when off and `droppedFactsBucket`. A ready +update waiting for days is not a failed install. + +Installs run with every backend stopped and an apply outcome is known before any backend starts, +so both events wait in a persisted journal in the desktop process +([`DesktopUpdateFacts.ts`](../../apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.ts): +at most 64 facts and 14 days; drops are counted into the next summary). It records nothing while +analytics is off for that launch (release metadata, `SCIENT_ANALYTICS_ENABLED`, or a managed +policy without telemetry) and deletes what an earlier launch kept. Facts travel on the telemetry +pipe's `desktopUpdateStatus` report to one backend (the primary when ready, else the first ready +one; two backends have two installation identities). That backend's `ingestRecorded` queues them +with their own event ids and times, through the normal consent and shape checks, and the desktop +forgets them only after the outbox persisted them or analytics there must never send them. A fact +offered again stays one event: the outbox and gateway both ignore a repeated id. + ## Privacy invariants Analytics must never include prompts, responses, file contents or paths, URLs (except a public skill diff --git a/packages/contracts/src/resourceTelemetry.ts b/packages/contracts/src/resourceTelemetry.ts index 69e1f158be..f3458d9eaf 100644 --- a/packages/contracts/src/resourceTelemetry.ts +++ b/packages/contracts/src/resourceTelemetry.ts @@ -285,6 +285,22 @@ export const DesktopUpdateRemoteOutcome = Schema.Literals([ ]); export type DesktopUpdateRemoteOutcome = typeof DesktopUpdateRemoteOutcome.Type; +// SCIENT-FORK:START — update facts the desktop journaled, delivered to one backend's analytics. +export const DESKTOP_UPDATE_FACTS_LIMIT = 64; +export const DesktopUpdateFact = Schema.Struct({ + id: TrimmedNonEmptyString, + name: Schema.Literals(["app.update.outcome", "app.update.summary"]), + occurredAt: TrimmedNonEmptyString, + appVersion: TrimmedNonEmptyString, + buildChannel: Schema.Literals(["stable", "beta", "nightly", "development", "unknown"]), + properties: Schema.Record( + Schema.String, + Schema.Union([Schema.String, Schema.Number, Schema.Boolean]), + ), +}); +export type DesktopUpdateFact = typeof DesktopUpdateFact.Type; +// SCIENT-FORK:END + export const DesktopUpdateStatusReport = Schema.Struct({ version: Schema.Literal(1), type: Schema.Literal("desktopUpdateStatus"), @@ -293,6 +309,11 @@ export const DesktopUpdateStatusReport = Schema.Struct({ outcome: Schema.optionalKey(DesktopUpdateRemoteOutcome), reason: Schema.optionalKey(TrimmedNonEmptyString), state: DesktopUpdateStateSchema, + // SCIENT-FORK:START — a report carrying facts delivers them; it is not a status change. + facts: Schema.optionalKey( + Schema.Array(DesktopUpdateFact).check(Schema.isMaxLength(DESKTOP_UPDATE_FACTS_LIMIT)), + ), + // SCIENT-FORK:END }); export type DesktopUpdateStatusReport = typeof DesktopUpdateStatusReport.Type; @@ -407,6 +428,16 @@ export const DesktopTelemetryServerReady = Schema.Struct({ }); export type DesktopTelemetryServerReady = typeof DesktopTelemetryServerReady.Type; +// SCIENT-FORK:START — the backend's analytics outbox holds these facts, or must never send them. +export const DesktopTelemetryAckDesktopUpdateFacts = Schema.Struct({ + version: Schema.Literal(1), + type: Schema.Literal("ackDesktopUpdateFacts"), + ids: Schema.Array(TrimmedNonEmptyString).check(Schema.isMaxLength(DESKTOP_UPDATE_FACTS_LIMIT)), +}); +export type DesktopTelemetryAckDesktopUpdateFacts = + typeof DesktopTelemetryAckDesktopUpdateFacts.Type; +// SCIENT-FORK:END + export const DesktopTelemetryControlMessage = Schema.Union([ DesktopTelemetryServerReady, DesktopTelemetrySetDiagnosticsDemand, @@ -414,6 +445,8 @@ export const DesktopTelemetryControlMessage = Schema.Union([ DesktopTelemetryRequestDesktopUpdate, DesktopTelemetryCommitDesktopUpdate, DesktopTelemetryCancelDesktopUpdate, + // SCIENT-FORK: acknowledges journaled update facts. + DesktopTelemetryAckDesktopUpdateFacts, ]); export type DesktopTelemetryControlMessage = typeof DesktopTelemetryControlMessage.Type; diff --git a/packages/scient-analytics/fixtures/contract-v11.json b/packages/scient-analytics/fixtures/contract-v11.json index 37b6196d1c..7fba884507 100644 --- a/packages/scient-analytics/fixtures/contract-v11.json +++ b/packages/scient-analytics/fixtures/contract-v11.json @@ -118,6 +118,69 @@ "contractRevision": "11" } }, + { + "case": "app.update.outcome:fallbacks", + "name": "app.update.outcome", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "step": "unknown", + "outcome": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "app.update.outcome:representative", + "name": "app.update.outcome", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "step": "unknown", + "outcome": "unknown", + "durationBucket": "under-1m", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "app.update.summary:fallbacks", + "name": "app.update.summary", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "reason": "unknown", + "status": "unknown", + "channel": "unknown", + "behind": "unknown", + "stageAgeBucket": "unknown", + "lastCheckAgeBucket": "never", + "droppedFactsBucket": "0", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "app.update.summary:representative", + "name": "app.update.summary", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "reason": "unknown", + "status": "unknown", + "channel": "unknown", + "behind": "unknown", + "stageAgeBucket": "unknown", + "lastCheckAgeBucket": "never", + "droppedFactsBucket": "0", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, { "case": "server.boot.heartbeat:fallbacks", "name": "server.boot.heartbeat", @@ -2079,6 +2142,114 @@ "contractRevision": "11" } }, + { + "case": "app.update.outcome:download-failed", + "name": "app.update.outcome", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "step": "download", + "outcome": "failed", + "failureClass": "signature", + "trigger": "background", + "targetVersion": "0.6.30", + "sourceKind": "github", + "packageKind": "nsis", + "attemptRef": "3f9a0c1d2e4b5a67", + "durationBucket": "1-10m", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "app.update.outcome:apply-not-applied", + "name": "app.update.outcome", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "step": "apply", + "outcome": "not-applied", + "targetVersion": "0.6.30", + "packageKind": "zip", + "attemptRef": "3f9a0c1d2e4b5a67", + "durationBucket": "1-3d", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "app.update.outcome:outcome-hostile", + "name": "app.update.outcome", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "step": "install", + "outcome": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "app.update.summary:summary-downloading", + "name": "app.update.summary", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "reason": "change", + "status": "downloading", + "channel": "latest", + "behind": "yes", + "stageAgeBucket": "10-60m", + "lastCheckAgeBucket": "10-60m", + "droppedFactsBucket": "0", + "progressBucket": "25-49", + "sinceProgressBucket": "10-60m", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "app.update.summary:summary-ready", + "name": "app.update.summary", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "reason": "daily", + "status": "downloaded", + "channel": "beta", + "behind": "yes", + "stageAgeBucket": "3-7d", + "lastCheckAgeBucket": "1-6h", + "droppedFactsBucket": "11-50", + "readyAgeBucket": "3-7d", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "app.update.summary:summary-disabled", + "name": "app.update.summary", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "reason": "launch", + "status": "disabled", + "channel": "latest", + "behind": "unknown", + "stageAgeBucket": "under-1m", + "lastCheckAgeBucket": "never", + "droppedFactsBucket": "0", + "disabledClass": "policy", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, { "case": "scient.operation.completed:capture-v2-retained", "name": "scient.operation.completed", diff --git a/packages/scient-analytics/src/conformance.ts b/packages/scient-analytics/src/conformance.ts index 36ad6b9f1f..72bac98b8b 100644 --- a/packages/scient-analytics/src/conformance.ts +++ b/packages/scient-analytics/src/conformance.ts @@ -336,6 +336,72 @@ export function buildAnalyticsConformanceFixture() { failedCount: "2", }), ); + // Desktop app updates: closed steps and outcomes, validated release versions, buckets only. + cases.push( + fixtureCase("download-failed", "app.update.outcome", { + step: "download", + outcome: "failed", + failureClass: "signature", + trigger: "background", + targetVersion: "0.6.30", + sourceKind: "github", + packageKind: "nsis", + attemptRef: "3f9a0c1d2e4b5a67", + durationMs: 95_000, + ...privateProperties, + }), + fixtureCase("apply-not-applied", "app.update.outcome", { + step: "apply", + outcome: "not-applied", + trigger: "user", + targetVersion: "0.6.30", + packageKind: "zip", + attemptRef: "3f9a0c1d2e4b5a67", + durationMs: 2 * 86_400_000, + }), + fixtureCase("outcome-hostile", "app.update.outcome", { + step: "install", + outcome: "completed", + failureClass: "PRIVATE-CONTENT", + trigger: "agent", + targetVersion: "0.6.30-private.build", + sourceKind: "https://example.invalid/feed", + packageKind: "dmg", + attemptRef: "PRIVATE", + message: "PRIVATE-CONTENT", + }), + fixtureCase("summary-downloading", "app.update.summary", { + reason: "change", + status: "downloading", + channel: "latest", + behind: "yes", + stageAgeMs: 25 * 60_000, + downloadPercent: 40, + sinceProgressMs: 12 * 60_000, + lastCheckAgeMs: 26 * 60_000, + droppedFactCount: 0, + ...privateProperties, + }), + fixtureCase("summary-ready", "app.update.summary", { + reason: "daily", + status: "downloaded", + channel: "beta", + behind: "yes", + stageAgeMs: 4 * 86_400_000, + readyAgeMs: 4 * 86_400_000, + lastCheckAgeMs: 3 * 3_600_000, + droppedFactCount: 12, + }), + fixtureCase("summary-disabled", "app.update.summary", { + reason: "launch", + status: "disabled", + channel: "latest", + behind: "unknown", + stageAgeMs: 0, + disabledClass: "policy", + message: "Updates are turned off by PRIVATE-CONTENT.", + }), + ); for (const captureOutcome of ["retained", "partial", "none", "failed", "unobserved"]) { const rejected = captureOutcome === "partial" || captureOutcome === "failed"; const retained = captureOutcome === "partial" || captureOutcome === "retained"; diff --git a/packages/scient-analytics/src/contract.ts b/packages/scient-analytics/src/contract.ts index 5bfcf4deb4..2253e7b794 100644 --- a/packages/scient-analytics/src/contract.ts +++ b/packages/scient-analytics/src/contract.ts @@ -22,6 +22,8 @@ export const ANALYTICS_EVENT_NAMES = [ "app.session.ended", "app.health", "app.diagnostics", + "app.update.outcome", + "app.update.summary", "server.boot.heartbeat", "install.owner_observed", "install.account_observed", @@ -336,6 +338,50 @@ const HEALTH_OPERATIONS = new Set([ "update", ]); const HEALTH_OUTCOMES = new Set(["started", "completed", "failed", "abnormal"]); +// Desktop app updates: each step's outcomes, and why a step failed. +const UPDATE_STEP_OUTCOMES: Readonly>> = { + check: ["available", "up-to-date", "failed"], + download: ["started", "completed", "failed"], + install: ["handoff-attempted", "failed"], + apply: ["completed", "not-applied", "superseded"], +}; +const UPDATE_FAILURE_CLASSES = [ + "network", + "timeout", + "certificate", + "http_4xx", + "http_5xx", + "feed", + "checksum", + "signature", + "disk_full", + "permission", + "configuration", + "cancelled", + "internal", + "unknown", +]; +const UPDATE_STATUSES = [ + "disabled", + "idle", + "checking", + "up-to-date", + "available", + "downloading", + "downloaded", + "error", +]; +const UPDATE_DISABLED_CLASSES = [ + "policy", + "channel-policy", + "safety", + "no-feed", + "development", + "environment", + "linux-package", + "other", +]; +const UPDATE_ATTEMPT_REF_PATTERN = /^[0-9a-f]{16}$/u; const OPERATION_KINDS = new Set([ "file-preview", "pdf-open", @@ -887,6 +933,98 @@ export function durationBucket(value: unknown): string { return "over-10m"; } +/** How long an update stage or offer has lasted: minutes while transferring, days while waiting. */ +function updateAgeBucket(value: unknown): string { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return "unknown"; + if (value < 60_000) return "under-1m"; + if (value < 600_000) return "1-10m"; + if (value < 3_600_000) return "10-60m"; + if (value < 21_600_000) return "1-6h"; + if (value < 86_400_000) return "6-24h"; + if (value < 3 * 86_400_000) return "1-3d"; + if (value < 7 * 86_400_000) return "3-7d"; + return "over-7d"; +} + +/** Copies a field only when it is one of `allowed`; any other value stays absent. */ +function optionalClosed( + input: Readonly>, + key: string, + allowed: ReadonlyArray, +): Record { + const value = property(input, key); + return typeof value === "string" && allowed.includes(value) ? { [key]: value } : {}; +} + +/** Marks every field read, so an event sends only what its normalizer sets. */ +function closedEvent(input: Readonly>): void { + for (const key of Object.keys(input)) property(input, key); +} + +function updateOutcomeProperties(input: Readonly>) { + closedEvent(input); + const step = closedCategory(property(input, "step"), Object.keys(UPDATE_STEP_OUTCOMES)); + const outcome = closedCategory(property(input, "outcome"), UPDATE_STEP_OUTCOMES[step] ?? []); + const failureClass = closedCategory(property(input, "failureClass"), UPDATE_FAILURE_CLASSES); + const trigger = property(input, "trigger"); + const targetVersion = property(input, "targetVersion"); + const attemptRef = property(input, "attemptRef"); + const durationMs = property(input, "durationMs"); + return { + step, + outcome, + ...(outcome === "failed" ? { failureClass } : {}), + ...(step !== "apply" && (trigger === "user" || trigger === "background") ? { trigger } : {}), + ...(typeof targetVersion === "string" && analyticsAppVersion(targetVersion) !== "unknown" + ? { targetVersion } + : {}), + ...optionalClosed(input, "sourceKind", ["github", "scient_mirror", "administrator"]), + ...optionalClosed(input, "packageKind", ["zip", "nsis", "appimage", "deb"]), + ...(typeof attemptRef === "string" && UPDATE_ATTEMPT_REF_PATTERN.test(attemptRef) + ? { attemptRef } + : {}), + ...(durationMs === undefined ? {} : { durationBucket: updateAgeBucket(durationMs) }), + }; +} + +function updateSummaryProperties(input: Readonly>) { + closedEvent(input); + const status = closedCategory(property(input, "status"), UPDATE_STATUSES); + const lastCheckAgeMs = property(input, "lastCheckAgeMs"); + return { + reason: closedCategory(property(input, "reason"), ["launch", "change", "daily"]), + status, + channel: closedCategory(property(input, "channel"), ["latest", "beta", "nightly"]), + behind: closedCategory(property(input, "behind"), ["yes", "no", "unknown"]), + stageAgeBucket: updateAgeBucket(property(input, "stageAgeMs")), + lastCheckAgeBucket: lastCheckAgeMs === undefined ? "never" : updateAgeBucket(lastCheckAgeMs), + droppedFactsBucket: countBucket(property(input, "droppedFactCount") ?? 0), + ...(status === "error" + ? { + errorContext: closedCategory(property(input, "errorContext"), [ + "check", + "download", + "install", + ]), + } + : {}), + ...(status === "downloading" + ? { + progressBucket: percentBucket(property(input, "downloadPercent")), + sinceProgressBucket: updateAgeBucket(property(input, "sinceProgressMs")), + } + : {}), + ...(status === "downloaded" + ? { readyAgeBucket: updateAgeBucket(property(input, "readyAgeMs")) } + : {}), + ...(status === "disabled" + ? { + disabledClass: closedCategory(property(input, "disabledClass"), UPDATE_DISABLED_CLASSES), + } + : {}), + }; +} + function turnTerminalStatus(value: unknown): string { if (value === "cancelled" || value === "interrupted") return "stopped"; return normalizedEnum(value, new Set(["completed", "failed", "stopped"]), "other"); @@ -1152,6 +1290,23 @@ function normalizeEvent( ...optionalProperties(input, { stoppedCount: "count" }), }, }; + case "app.update.outcome": { + const properties = updateOutcomeProperties(input); + return { + name, + privacyLevel: "essential", + priority: + properties.step === "install" || properties.step === "apply" ? "critical" : "core", + properties, + }; + } + case "app.update.summary": + return { + name, + privacyLevel: "essential", + priority: "summary", + properties: updateSummaryProperties(input), + }; case "app.session.started": return { name, diff --git a/packages/scient-analytics/src/index.ts b/packages/scient-analytics/src/index.ts index eb7cfec51b..32a8f422ee 100644 --- a/packages/scient-analytics/src/index.ts +++ b/packages/scient-analytics/src/index.ts @@ -15,5 +15,6 @@ export { createAnalyticsRuntime, type AnalyticsRuntime, type AnalyticsRuntimeOptions, + type RecordedAnalyticsEvent, } from "./runtime.ts"; export type { AnalyticsWorkerInput } from "./workerProtocol.ts"; diff --git a/packages/scient-analytics/src/runtime.test.ts b/packages/scient-analytics/src/runtime.test.ts index 14f9a4aec9..521cfdc330 100644 --- a/packages/scient-analytics/src/runtime.test.ts +++ b/packages/scient-analytics/src/runtime.test.ts @@ -500,4 +500,62 @@ describe("Scient analytics background runtime", () => { expect(server.deletionBodies).toHaveLength(1); expect(server.bodies).toHaveLength(1); }); + + it("ingests a recorded event once, keeping its id, time and release", async () => { + const outboxPath = fixturePath(); + const runtime = createAnalyticsRuntime( + options({ outboxPath, now: () => new Date("2026-11-16T12:00:00.000Z") }), + ); + const recorded = { + id: "6f1c2a9e-4b7d-4e21-9a3f-0c5d8e7b1a24", + name: "app.update.outcome", + occurredAt: "2026-11-16T11:00:00.000Z", + appVersion: "0.6.29", + buildChannel: "stable", + properties: { step: "install", outcome: "handoff-attempted", targetVersion: "0.6.30" }, + } as const; + try { + expect(await runtime.ingest([recorded])).toBe(true); + // A replay after a lost acknowledgement finds the same row. + expect(await runtime.ingest([recorded])).toBe(true); + expect(await runtime.pendingCount()).toBe(1); + } finally { + await runtime.close(); + } + const database = new NodeSqlite.DatabaseSync(outboxPath); + const row = database + .prepare("SELECT id, occurred_at, privacy_level, properties_json FROM analytics_outbox") + .get() as { + readonly id: string; + readonly occurred_at: string; + readonly privacy_level: string; + readonly properties_json: string; + }; + database.close(); + expect(row).toMatchObject({ + id: recorded.id, + occurred_at: recorded.occurredAt, + privacy_level: "essential", + }); + expect(JSON.parse(row.properties_json)).toMatchObject({ + appVersion: "0.6.29", + step: "install", + targetVersion: "0.6.30", + }); + }); + + it("settles recorded events it must not send without queueing them", async () => { + const runtime = createAnalyticsRuntime(options({ consent: "off" })); + const recorded = { + id: "6f1c2a9e-4b7d-4e21-9a3f-0c5d8e7b1a25", + name: "app.update.summary", + occurredAt: "2026-11-16T11:00:00.000Z", + appVersion: "0.6.29", + buildChannel: "stable", + properties: { reason: "launch", status: "idle" }, + } as const; + expect(await runtime.ingest([recorded])).toBe(true); + expect(await runtime.pendingCount()).toBe(0); + await runtime.close(); + }); }); diff --git a/packages/scient-analytics/src/runtime.ts b/packages/scient-analytics/src/runtime.ts index b5b438038d..daf392a1c8 100644 --- a/packages/scient-analytics/src/runtime.ts +++ b/packages/scient-analytics/src/runtime.ts @@ -6,6 +6,7 @@ import { type AnalyticsConsent, type AnalyticsPriority, consentAllows, + type EventPrivacyLevel, type NormalizationContext, normalizeInheritedEvent, } from "./contract.ts"; @@ -25,6 +26,9 @@ const MAX_IN_FLIGHT_BATCHES = 2; const WRITE_COALESCE_MS = 100; const CONTROL_TIMEOUT_MS = 3_500; const LOCAL_SHUTDOWN_BUDGET_MS = 300; +const RECORDED_EVENT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u; +const RECORDED_EVENT_MAX_AGE_MS = 30 * 86_400_000; +const RECORDED_EVENT_MAX_SKEW_MS = 86_400_000; const PRIORITY_RANK: Readonly> = { critical: 0, @@ -46,9 +50,22 @@ export interface AnalyticsRuntimeOptions extends NormalizationContext { readonly controlTimeoutMs?: number; } +/** An event another Scient process recorded; it keeps its own id, time and release. */ +export interface RecordedAnalyticsEvent extends NormalizationContext { + readonly id: string; + readonly name: string; + readonly occurredAt: string; + readonly properties: Readonly>; +} + export interface AnalyticsRuntime { readonly enabled: boolean; readonly record: (name: string, properties?: Readonly>) => boolean; + /** + * True once the outbox holds these events or none of them may be sent (consent, shape, age); + * false when that is unconfirmed, so the caller keeps them to offer again. + */ + readonly ingest: (events: ReadonlyArray) => Promise; readonly flush: () => Promise; readonly setConsent: (consent: AnalyticsConsent) => Promise; readonly pendingCount: () => Promise; @@ -60,6 +77,7 @@ function disabledRuntime(): AnalyticsRuntime { return { enabled: false, record: () => false, + ingest: async () => true, flush: async () => 0, setConsent: async () => 0, pendingCount: async () => 0, @@ -115,6 +133,7 @@ export function createAnalyticsRuntime(options: AnalyticsRuntimeOptions): Analyt let nextRequestId = 1; const queue: AnalyticsEventDraft[] = []; const inFlight = new Map>(); + const ingestions = new Map void>(); const requests = new Map< number, { @@ -144,6 +163,8 @@ export function createAnalyticsRuntime(options: AnalyticsRuntimeOptions): Analyt } queue.length = 0; inFlight.clear(); + for (const settle of ingestions.values()) settle(false); + ingestions.clear(); settleRequests(); }; @@ -220,6 +241,8 @@ export function createAnalyticsRuntime(options: AnalyticsRuntimeOptions): Analyt return; case "persisted": inFlight.delete(response.batchId); + ingestions.get(response.batchId)?.(true); + ingestions.delete(response.batchId); drain(); return; case "result": { @@ -280,11 +303,72 @@ export function createAnalyticsRuntime(options: AnalyticsRuntimeOptions): Analyt return accepted; }; + const recordedDraft = ( + event: RecordedAnalyticsEvent, + level: EventPrivacyLevel, + ): AnalyticsEventDraft | null => { + const occurredAt = Date.parse(event.occurredAt); + const nowMs = now().valueOf(); + if ( + !RECORDED_EVENT_ID.test(event.id) || + !Number.isFinite(occurredAt) || + occurredAt > nowMs + RECORDED_EVENT_MAX_SKEW_MS || + occurredAt < nowMs - RECORDED_EVENT_MAX_AGE_MS + ) { + return null; + } + const normalized = normalizeInheritedEvent(event.name, event.properties, { + appVersion: event.appVersion, + buildChannel: event.buildChannel, + }); + if (!normalized || !consentAllows(level, normalized.privacyLevel)) return null; + return { + id: event.id, + name: normalized.name, + session_id: sessionId, + occurred_at: new Date(occurredAt).toISOString(), + privacy_level: normalized.privacyLevel, + consent_level: level, + properties: normalized.properties, + priority: normalized.priority, + }; + }; + + const ingest: AnalyticsRuntime["ingest"] = async (events) => { + if (closed || deleting || !available) return false; + const level = consent; + if (level === "off" || purpose === "deletion") return true; + const drafts = events.flatMap((event) => recordedDraft(event, level) ?? []); + if (drafts.length === 0) return true; + await waitForCondition(() => ready || !available, Date.now() + controlTimeoutMs); + if (!ready || !available || closed || deleting) return false; + const batchId = nextBatchId++; + return new Promise((resolve) => { + const timer = setTimeout(() => { + ingestions.delete(batchId); + resolve(false); + }, controlTimeoutMs); + timer.unref(); + ingestions.set(batchId, (persisted) => { + clearTimeout(timer); + resolve(persisted); + }); + inFlight.set(batchId, drafts); + if (!post({ type: "enqueue", batchId, events: drafts })) { + inFlight.delete(batchId); + ingestions.delete(batchId); + clearTimeout(timer); + resolve(false); + } + }); + }; + return { get enabled() { return available && !closed; }, record, + ingest, flush: async () => { if (closed || deleting || !available || consent === "off" || purpose === "deletion") return 0; await waitForCondition(() => ready || !available, Date.now() + controlTimeoutMs); From 6182a7e752019b2eebd6463bf05d62dc9bc03733 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Mon, 16 Nov 2026 23:02:00 +0200 Subject: [PATCH 03/12] ci(release): require the pinned Windows publisher on every channel (C2) A signed Windows app pins its publisher: electron-updater installs a later update only if the installer's signer CN matches publisherName in the installed app-update.yml. An unsigned or differently signed release would then fail every pinned client's update, on Stable and on Beta (clients switch channels). - Build: the Windows signing attestation records the configured publisher (AZURE_TRUSTED_SIGNING_PUBLISHER_NAME) when signed. - Native Windows verifier: a signed installer must be Valid, signed by that CN, and the packaged app-update.yml must pin exactly that name; an unsigned build must pin none. - Handoff: records windows { signed, publisherName } for later gates. - Publication gate: reads the newest published Stable and Beta handoffs; if either pins a publisher, the candidate must be signed by it and allow_unsigned_windows (including Beta's automatic exception) cannot override. Older handoffs carry no record and pin nothing, so today's unsigned releases keep working until the first signed one ships. No job is added or renamed, so no required-check names change. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 68 ++++++- docs/operations/release.md | 16 ++ scripts/create-scient-release-handoff.ts | 15 ++ scripts/windows-publication-signing.test.ts | 196 ++++++++++++++++++++ scripts/windows-publication-signing.ts | 127 +++++++++++++ 5 files changed, 414 insertions(+), 8 deletions(-) create mode 100644 scripts/windows-publication-signing.test.ts create mode 100644 scripts/windows-publication-signing.ts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a2279457cb..a970b25e5d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -883,8 +883,15 @@ jobs: if [[ "$RELEASE_PLATFORM" == "mac" && "$RELEASE_ARCH" == "x64" ]]; then for manifest in release-publish/*-mac.yml; do mv "$manifest" "${manifest%.yml}-x64.yml"; done fi - printf '{"platform":"%s","arch":"%s","signed":%s}\n' \ - "$RELEASE_PLATFORM" "$RELEASE_ARCH" "$signed" \ + # A signed Windows build pins this publisher in the installed app's app-update.yml. + publisher="" + if [[ "$RELEASE_PLATFORM" == "win" && "$signed" == "true" ]]; then + publisher="$AZURE_TRUSTED_SIGNING_PUBLISHER_NAME" + fi + jq -cn --arg platform "$RELEASE_PLATFORM" --arg arch "$RELEASE_ARCH" \ + --argjson signed "$signed" --arg publisher "$publisher" \ + '{platform: $platform, arch: $arch, signed: $signed} + + (if $publisher == "" then {} else {publisherName: $publisher} end)' \ > "release-publish/signing-$RELEASE_PLATFORM-$RELEASE_ARCH.json" if [[ "$RELEASE_PLATFORM" == "mac" ]]; then echo "SCIENT_MAC_SIGNING_READY=$signed" >> "$GITHUB_ENV" @@ -1007,9 +1014,37 @@ jobs: run: | $attestation = Get-Content verify/signing.json -Raw | ConvertFrom-Json if ($attestation.platform -ne 'win' -or $attestation.arch -ne 'x64') { throw "Invalid signing attestation target" } + # electron-updater checks each later installer against the publisherName list in the + # installed app's app-update.yml, comparing the signer certificate's CN. + $pinned = @() + $inList = $false + $updateConfig = 'verify/win-unpacked/resources/app-update.yml' + $lines = if (Test-Path $updateConfig) { Get-Content $updateConfig } else { @() } + if ($attestation.signed -and $lines.Count -eq 0) { throw "The signed Windows app has no app-update.yml" } + foreach ($line in $lines) { + if ($line -match '^publisherName:\s*(\S.*)?$') { + $inList = $true + if ($Matches[1]) { $pinned += $Matches[1].Trim().Trim("'").Trim('"') } + continue + } + if ($inList -and $line -match '^\s+-\s+(.+)$') { $pinned += $Matches[1].Trim().Trim("'").Trim('"'); continue } + $inList = $false + } if ($attestation.signed) { + $expected = [string]$attestation.publisherName + if (-not $expected) { throw "The signed Windows attestation names no publisher" } + if ($pinned.Count -ne 1 -or $pinned[0] -cne $expected) { + throw "The installed app would pin publisher '$($pinned -join "', '")', not the configured '$expected'" + } $signature = Get-AuthenticodeSignature (Resolve-Path verify/installer.exe) if ($signature.Status -ne "Valid") { throw "Invalid Windows signature: $($signature.Status)" } + $subject = $signature.SignerCertificate.Subject + if ($subject -notmatch '(?:^|,\s*)CN=("(?:[^"]|"")*"|[^,]+)') { throw "The installer's signer has no CN: $subject" } + $signer = $Matches[1].Trim() + if ($signer.StartsWith('"')) { $signer = $signer.Substring(1, $signer.Length - 2).Replace('""', '"') } + if ($signer -cne $expected) { throw "The installer is signed by '$signer', not the configured publisher '$expected'" } + } elseif ($pinned.Count -gt 0) { + throw "An unsigned Windows build must not pin publisher '$($pinned -join "', '")'" } # Needs every target's resource monitor and SyncTeX runtime, not their installers. @@ -1163,6 +1198,7 @@ jobs: shell: bash env: ALLOW_UNSIGNED_WINDOWS: ${{ needs.preflight.outputs.allow_unsigned_windows }} + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail for attestation in release-assets/signing-mac-arm64.json release-assets/signing-mac-x64.json; do @@ -1171,16 +1207,31 @@ jobs: exit 1 } done - if ! jq -e '.signed == true' release-assets/signing-win-x64.json >/dev/null; then - if [[ "$ALLOW_UNSIGNED_WINDOWS" != "true" ]]; then - echo "Windows publication is unsigned; rerun only with explicit allow_unsigned_windows=true." >&2 - exit 1 - fi - echo "WARNING: publishing an unsigned Windows installer by explicit release approval." + # Clients on either channel can switch to the other, so a publisher pinned by the newest + # Stable or Beta release binds this publication, whatever its channel or exception. + published="$RUNNER_TEMP/published-handoffs" + mkdir -p "$published" + handoffs=() + stable_tag="$(gh api "repos/$SCIENT_DESKTOP_CANONICAL_REPOSITORY/releases/latest" --jq .tag_name)" + gh release download "$stable_tag" --repo "$SCIENT_DESKTOP_CANONICAL_REPOSITORY" \ + --pattern scient-release-handoff.json --output "$published/stable.json" + handoffs+=(--published-handoff "$published/stable.json") + beta_tag="$(gh api "repos/$SCIENT_DESKTOP_BETA_REPOSITORY/releases?per_page=100" \ + --jq '[.[] | select(.draft == false and (.tag_name | contains("-beta.")))][0].tag_name // empty')" + if [[ -n "$beta_tag" ]]; then + gh release download "$beta_tag" --repo "$SCIENT_DESKTOP_BETA_REPOSITORY" \ + --pattern scient-release-handoff.json --output "$published/beta.json" + handoffs+=(--published-handoff "$published/beta.json") fi + node scripts/windows-publication-signing.ts \ + --attestation release-assets/signing-win-x64.json \ + --allow-unsigned "$ALLOW_UNSIGNED_WINDOWS" \ + "${handoffs[@]}" - name: Remove internal build attestations shell: bash run: | + # The handoff keeps the Windows signing record for the next publication's gate. + cp release-assets/signing-win-x64.json "$RUNNER_TEMP/signing-win-x64.json" rm -f release-assets/signing-*.json rm -f release-assets/builder-debug.yml - name: Verify complete updater and server artifact set @@ -1207,6 +1258,7 @@ jobs: --channel "$RELEASE_CHANNEL" \ --source-sha "$SOURCE_SHA" \ --source-tree "$SOURCE_TREE" \ + --windows-signing "$RUNNER_TEMP/signing-win-x64.json" \ --output "release-assets/scient-release-handoff.json" - id: release_candidate name: Upload immutable release candidate diff --git a/docs/operations/release.md b/docs/operations/release.md index 578bc4a7b4..61be74d8ab 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -281,6 +281,22 @@ It labels them unsigned and cannot publish them unless the explicit the same owner-approved Windows exception automatically. That exception applies only to Windows; macOS signing and notarization remain mandatory. +A signed Windows app pins its publisher: electron-updater installs a later update +only if its installer's signer CN matches the `publisherName` in the installed +`app-update.yml`. So the exception ends once a signed build ships: + +- The native Windows verifier requires a signed installer's Authenticode status to + be Valid, its signer CN to equal `AZURE_TRUSTED_SIGNING_PUBLISHER_NAME`, and the + packaged `app-update.yml` to pin exactly that name. An unsigned build must pin none. +- Each release handoff records `windows: { signed, publisherName }`. +- Before publishing on either channel, the gate reads the newest published Stable + and Beta handoffs. If either pins a publisher, the candidate must be signed by + that publisher, whatever `allow_unsigned_windows` says: users switch between + Stable and Beta, so a pin on one binds both. Handoffs from before this record + pin nothing, so today's unsigned releases are unaffected until the first signed + one ships. Changing the publisher later needs a planned transition, not a + release that installed clients would reject. + Release jobs expose Apple credentials only to macOS packaging and Azure credentials only to the Linux-hosted Windows packaging job. Publication additionally verifies the final Windows Authenticode signature and the final macOS app signatures, diff --git a/scripts/create-scient-release-handoff.ts b/scripts/create-scient-release-handoff.ts index db009b7822..28e6e1aaad 100644 --- a/scripts/create-scient-release-handoff.ts +++ b/scripts/create-scient-release-handoff.ts @@ -10,6 +10,8 @@ import { scientReleaseRepository, } from "@t3tools/shared/scientRelease"; +import { readWindowsSigningAttestation } from "./windows-publication-signing.ts"; + interface ReleaseAsset { readonly name: string; readonly size: number; @@ -45,6 +47,12 @@ export function createScientReleaseHandoff(args: ReadonlyArray): string const sourceSha = value(args, "--source-sha"); const sourceTree = value(args, "--source-tree"); const output = NodePath.resolve(value(args, "--output")); + // Later publications read this to learn which publisher installed Windows clients pin. + const windows = args.includes("--windows-signing") + ? readWindowsSigningAttestation( + JSON.parse(NodeFS.readFileSync(value(args, "--windows-signing"), "utf8")) as unknown, + ) + : null; const excluded = new Set([NodePath.basename(output), "SHA256SUMS.txt"]); const assets: ReleaseAsset[] = NodeFS.readdirSync(assetsDir) .filter((name) => !excluded.has(name)) @@ -70,6 +78,13 @@ export function createScientReleaseHandoff(args: ReadonlyArray): string version, tag: `v${version}`, distribution: { repository: distributionRepository, channel }, + ...(windows === null + ? {} + : { + windows: windows.signed + ? { signed: true, publisherName: windows.publisherName } + : { signed: false }, + }), source: { repository: sourceRepository, commit: sourceSha, diff --git a/scripts/windows-publication-signing.test.ts b/scripts/windows-publication-signing.test.ts new file mode 100644 index 0000000000..7616a1cf00 --- /dev/null +++ b/scripts/windows-publication-signing.test.ts @@ -0,0 +1,196 @@ +// @effect-diagnostics nodeBuiltinImport:off - Rehearse the publication gate with temporary files and a local gh fixture. +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import { assert, describe, it } from "@effect/vitest"; +import { parse } from "yaml"; + +import { createScientReleaseHandoff } from "./create-scient-release-handoff.ts"; +import { + decideWindowsPublication, + pinnedWindowsPublishers, + readWindowsSigningAttestation, +} from "./windows-publication-signing.ts"; + +const PUBLISHER = "ScientFactory Ltd"; +const signed = { signed: true, publisherName: PUBLISHER } as const; +const unsigned = { signed: false, publisherName: null } as const; + +describe("Windows publication signing gate", () => { + it("keeps today's unsigned exception while no installed client pins a publisher", () => { + // Handoffs published before the signing record pin nothing. + const pinned = pinnedWindowsPublishers([{ version: "0.6.29" }, { windows: { signed: false } }]); + assert.deepStrictEqual(pinned, []); + assert.throws( + () => decideWindowsPublication({ candidate: unsigned, pinned, allowUnsigned: false }), + /allow_unsigned_windows=true/, + ); + assert.match( + decideWindowsPublication({ candidate: unsigned, pinned, allowUnsigned: true }).warning ?? "", + /unsigned Windows installer/, + ); + assert.isNull( + decideWindowsPublication({ candidate: signed, pinned, allowUnsigned: false }).warning, + ); + }); + + it("refuses an unsigned or differently signed installer once any channel pins a publisher", () => { + const pinned = pinnedWindowsPublishers([ + { windows: { signed: false } }, + { windows: { signed: true, publisherName: PUBLISHER } }, + ]); + assert.deepStrictEqual(pinned, [PUBLISHER]); + assert.throws( + () => decideWindowsPublication({ candidate: unsigned, pinned, allowUnsigned: true }), + /cannot override/, + ); + assert.throws( + () => + decideWindowsPublication({ + candidate: { signed: true, publisherName: "Someone Else" }, + pinned, + allowUnsigned: true, + }), + /signed by 'Someone Else'/, + ); + assert.isNull( + decideWindowsPublication({ candidate: signed, pinned, allowUnsigned: false }).warning, + ); + }); + + it("fails closed on attestations and records it cannot read", () => { + assert.throws( + () => readWindowsSigningAttestation({ platform: "win", arch: "x64", signed: true }), + /must name its publisher/, + ); + assert.throws( + () => + readWindowsSigningAttestation({ + platform: "win", + arch: "x64", + signed: false, + publisherName: PUBLISHER, + }), + /cannot name a publisher/, + ); + assert.throws(() => pinnedWindowsPublishers([{ windows: { signed: "yes" } }])); + }); + + it("records the signing attestation in the handoff the next publication reads", () => { + const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "scient-windows-handoff-")); + try { + const assets = NodePath.join(root, "assets"); + NodeFS.mkdirSync(assets); + NodeFS.writeFileSync(NodePath.join(assets, "Scient-0.6.30-x64.exe"), "installer"); + const attestation = NodePath.join(root, "signing-win-x64.json"); + const output = NodePath.join(assets, "scient-release-handoff.json"); + const create = () => + createScientReleaseHandoff([ + "--assets-dir", + assets, + "--version", + "0.6.30", + "--channel", + "stable", + "--repository", + "ScientFactory/scient-desktop", + "--source-sha", + "a".repeat(40), + "--source-tree", + "b".repeat(40), + "--windows-signing", + attestation, + "--output", + output, + ]); + const pinnedAfter = (record: object) => { + NodeFS.writeFileSync( + attestation, + JSON.stringify({ platform: "win", arch: "x64", ...record }), + ); + create(); + return pinnedWindowsPublishers([JSON.parse(NodeFS.readFileSync(output, "utf8"))]); + }; + + assert.deepStrictEqual(pinnedAfter({ signed: true, publisherName: PUBLISHER }), [PUBLISHER]); + assert.deepStrictEqual(pinnedAfter({ signed: false }), []); + } finally { + NodeFS.rmSync(root, { recursive: true, force: true }); + } + }); +}); + +const workflow = parse( + NodeFS.readFileSync( + NodePath.join(import.meta.dirname, "../.github/workflows/release.yml"), + "utf8", + ), +) as { jobs: { assemble: { steps: Array<{ name?: string; run?: string }> } } }; +const gateStep = workflow.jobs.assemble.steps.find( + (entry) => entry.name === "Require signed macOS and explicitly approved Windows publication", +)?.run; + +// oxlint-disable-next-line t3code/no-global-process-runtime -- This synchronous platform gate skips the Linux Bash step on native Windows. +describe.skipIf(process.platform === "win32")("Windows publication gate step", () => { + it("reads the newest Stable and Beta handoffs before deciding", () => { + if (!gateStep) throw new Error("Missing the publication signing step."); + const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "scient-windows-gate-")); + try { + const bin = NodePath.join(root, "bin"); + NodeFS.mkdirSync(bin); + NodeFS.mkdirSync(NodePath.join(root, "release-assets")); + NodeFS.symlinkSync(import.meta.dirname, NodePath.join(root, "scripts")); + for (const arch of ["arm64", "x64"]) { + NodeFS.writeFileSync( + NodePath.join(root, `release-assets/signing-mac-${arch}.json`), + JSON.stringify({ platform: "mac", arch, signed: true }), + ); + } + NodeFS.writeFileSync( + NodePath.join(root, "release-assets/signing-win-x64.json"), + JSON.stringify({ platform: "win", arch: "x64", signed: false }), + ); + // Only the newest Beta is signed: its clients can switch to Stable, so Stable is bound too. + NodeFS.writeFileSync( + NodePath.join(bin, "gh"), + `#!/usr/bin/env node +const fs = require('node:fs'); +const args = process.argv.slice(2); +if (args[0] === 'api' && args[1].endsWith('/scient-desktop/releases/latest')) process.stdout.write('v0.6.30\\n'); +else if (args[0] === 'api' && args[1].startsWith('repos/ScientFactory/scient-desktop-beta/releases')) process.stdout.write(process.env.BETA_TAG + '\\n'); +else if (args[0] === 'release' && args[1] === 'download') { + const handoff = args[2] === 'v0.6.30' ? { version: '0.6.30' } : { windows: { signed: true, publisherName: 'ScientFactory Ltd' } }; + fs.writeFileSync(args[args.indexOf('--output') + 1], JSON.stringify(handoff)); +} else process.exit(2); +`, + { mode: 0o700 }, + ); + const run = (betaTag: string) => + NodeChildProcess.spawnSync("bash", ["-c", gateStep], { + cwd: root, + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + RUNNER_TEMP: root, + BETA_TAG: betaTag, + ALLOW_UNSIGNED_WINDOWS: "true", + SCIENT_DESKTOP_CANONICAL_REPOSITORY: "ScientFactory/scient-desktop", + SCIENT_DESKTOP_BETA_REPOSITORY: "ScientFactory/scient-desktop-beta", + }, + encoding: "utf8", + }); + + const pinned = run("v0.6.31-beta.20261116.1"); + assert.notStrictEqual(pinned.status, 0); + assert.match(pinned.stderr, /pin publisher 'ScientFactory Ltd'/); + // With no Beta published, the explicit exception still allows today's unsigned build. + const none = run(""); + assert.strictEqual(none.status, 0, none.stderr); + assert.match(none.stdout, /WARNING: Publishing an unsigned Windows installer/); + } finally { + NodeFS.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/scripts/windows-publication-signing.ts b/scripts/windows-publication-signing.ts new file mode 100644 index 0000000000..4c88de5410 --- /dev/null +++ b/scripts/windows-publication-signing.ts @@ -0,0 +1,127 @@ +#!/usr/bin/env node +// @effect-diagnostics nodeBuiltinImport:off - Standalone release gate reads build attestations and published handoffs before publication. + +import * as NodeFS from "node:fs"; +import * as NodeURL from "node:url"; + +/** What a release records about its Windows installer's Authenticode signature. */ +export interface WindowsSigningRecord { + readonly signed: boolean; + /** The certificate CN the installed app pins in app-update.yml; null when unsigned. */ + readonly publisherName: string | null; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +/** The Windows build's signing attestation, as the release workflow writes it. */ +export function readWindowsSigningAttestation(value: unknown): WindowsSigningRecord { + if ( + !isRecord(value) || + value.platform !== "win" || + value.arch !== "x64" || + typeof value.signed !== "boolean" + ) { + throw new Error("Invalid Windows signing attestation."); + } + const publisherName = value.publisherName; + if (value.signed) { + if (typeof publisherName !== "string" || publisherName.trim() === "") { + throw new Error("A signed Windows attestation must name its publisher."); + } + return { signed: true, publisherName }; + } + if (publisherName !== undefined) { + throw new Error("An unsigned Windows attestation cannot name a publisher."); + } + return { signed: false, publisherName: null }; +} + +/** + * Publishers that installed Windows clients already pin, from each channel's newest published + * handoff. Once a signed build ships, every later build is signed by the same publisher, so the + * newest release of each channel carries the pin. Handoffs older than this record name none. + */ +export function pinnedWindowsPublishers(handoffs: ReadonlyArray): ReadonlyArray { + const pinned = new Set(); + for (const handoff of handoffs) { + if (!isRecord(handoff) || handoff.windows === undefined) continue; + const record = readWindowsSigningAttestation({ + platform: "win", + arch: "x64", + ...(handoff.windows as object), + }); + if (record.publisherName !== null) pinned.add(record.publisherName); + } + return [...pinned]; +} + +/** + * Clients switch between Stable and Beta, so a pin on either binds both, and electron-updater + * rejects an installer whose signer misses the pin: no exception may override it. Before any + * pin, an unsigned build still needs the explicit unsigned-Windows exception. + */ +export function decideWindowsPublication(input: { + readonly candidate: WindowsSigningRecord; + readonly pinned: ReadonlyArray; + readonly allowUnsigned: boolean; +}): { readonly warning: string | null } { + const { candidate, pinned, allowUnsigned } = input; + if (pinned.length > 0) { + const names = pinned.map((name) => `'${name}'`).join(", "); + if (!candidate.signed) { + throw new Error( + `Installed Windows clients pin publisher ${names}; an unsigned installer would fail their update signature check on Stable and Beta. allow_unsigned_windows cannot override this.`, + ); + } + if (!pinned.includes(candidate.publisherName!)) { + throw new Error( + `Installed Windows clients pin publisher ${names}; this installer is signed by '${candidate.publisherName}' and would fail their update signature check.`, + ); + } + return { warning: null }; + } + if (candidate.signed) return { warning: null }; + if (!allowUnsigned) { + throw new Error( + "Windows publication is unsigned; rerun only with explicit allow_unsigned_windows=true.", + ); + } + return { warning: "Publishing an unsigned Windows installer by explicit release approval." }; +} + +function values(args: ReadonlyArray, flag: string): ReadonlyArray { + return args.flatMap((arg, index) => (arg === flag && args[index + 1] ? [args[index + 1]!] : [])); +} + +function readJson(path: string): unknown { + return JSON.parse(NodeFS.readFileSync(path, "utf8")) as unknown; +} + +export function runWindowsPublicationGate(args: ReadonlyArray): string | null { + const [attestation] = values(args, "--attestation"); + if (!attestation) throw new Error("--attestation is required."); + const allowUnsigned = values(args, "--allow-unsigned")[0] === "true"; + const { warning } = decideWindowsPublication({ + candidate: readWindowsSigningAttestation(readJson(attestation)), + pinned: pinnedWindowsPublishers(values(args, "--published-handoff").map(readJson)), + allowUnsigned, + }); + return warning; +} + +// Real paths, so an indirect invocation still runs the gate rather than silently passing it. +const isMain = process.argv[1] + ? NodeFS.realpathSync(process.argv[1]) === + NodeFS.realpathSync(NodeURL.fileURLToPath(import.meta.url)) + : false; +if (isMain) { + try { + const warning = runWindowsPublicationGate(process.argv.slice(2)); + if (warning !== null) process.stdout.write(`WARNING: ${warning}\n`); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} From 08a0392abf436d4f5832a256306cd17ad4c5f7b4 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Mon, 16 Nov 2026 23:10:30 +0200 Subject: [PATCH 04/12] test(analytics): regenerate the contract corpus for both update and runtime events Co-Authored-By: Claude Opus 5.5 --- packages/scient-analytics/fixtures/contract-v11.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/scient-analytics/fixtures/contract-v11.json b/packages/scient-analytics/fixtures/contract-v11.json index 7fba884507..68a19a270e 100644 --- a/packages/scient-analytics/fixtures/contract-v11.json +++ b/packages/scient-analytics/fixtures/contract-v11.json @@ -139,6 +139,7 @@ "properties": { "step": "unknown", "outcome": "unknown", + "sourceKind": "github", "durationBucket": "under-1m", "appVersion": "0.6.8", "buildChannel": "stable", From 5f6fd4190517dd9f57a14dee9cb7386e5fb495e0 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Mon, 16 Nov 2026 23:40:39 +0200 Subject: [PATCH 05/12] fix(recovery): preserve cancellation, foreground priority and source cooldowns --- .../ProviderRuntimeManager.test.ts | 210 ++++++- .../ProviderRuntimeManager.ts | 518 +++++++++++++----- .../ProviderRuntimeRecovery.test.ts | 32 +- .../ProviderRuntimeRecovery.ts | 8 +- .../ProviderRuntimeRequests.ts | 112 ++-- .../ProviderRuntimeSection.browser.test.tsx | 46 ++ .../ProviderRuntimeSection.tsx | 60 +- docs/internals/provider-lifecycle.md | 26 +- packages/contracts/src/providerLifecycle.ts | 18 +- .../src/managedProviderRuntime.ts | 52 +- .../src/managedScientAgentDownload.test.ts | 45 ++ .../src/runtimeDownload.test.ts | 256 +++++++-- .../src/runtimeFiles.ts | 190 ++++--- 13 files changed, 1235 insertions(+), 338 deletions(-) diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts index 38e1bfdb7b..9c5b35cf06 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts @@ -1,4 +1,5 @@ import { assert, describe, it } from "@effect/vitest"; +import * as NodeOS from "node:os"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -48,7 +49,9 @@ import { make, ProviderRuntimeManager, } from "./ProviderRuntimeManager.ts"; -import { ProviderRuntimeRequests } from "./ProviderRuntimeRequests.ts"; +import { make as makeRequests, ProviderRuntimeRequests } from "./ProviderRuntimeRequests.ts"; +import { make as makeRecovery } from "./ProviderRuntimeRecovery.ts"; +import * as ServerConfig from "../../config.ts"; const CODEX = ProviderDriverKind.make("codex"); const INSTANCE = ProviderInstanceId.make("codex"); @@ -135,6 +138,7 @@ function makeHarness( readonly useProductionLayer?: boolean; readonly isBusy?: ProviderActivityShape["isBusy"]; readonly closeInstance?: ProviderSessionManagerV2Shape["closeInstance"]; + readonly requests?: ProviderRuntimeRequests["Service"]; } = {}, ) { return Effect.gen(function* () { @@ -226,12 +230,16 @@ function makeHarness( const managerScope = yield* Scope.make(); yield* Effect.addFinalizer(() => Scope.close(managerScope, Exit.void)); const requestLogRef = yield* SubscriptionRef.make>([]); - const requestsLayer = Layer.mock(ProviderRuntimeRequests)({ - requested: (input) => - SubscriptionRef.update(requestLogRef, (log) => [...log, `requested:${input.action}`]), - settled: (input) => - SubscriptionRef.update(requestLogRef, (log) => [...log, `settled:${input.outcome}`]), - }); + const requestsLayer = hooks.requests + ? Layer.succeed(ProviderRuntimeRequests, hooks.requests) + : Layer.mock(ProviderRuntimeRequests)({ + list: Effect.succeed([]), + changes: Stream.empty, + requested: (input) => + SubscriptionRef.update(requestLogRef, (log) => [...log, `requested:${input.action}`]), + settled: (input) => + SubscriptionRef.update(requestLogRef, (log) => [...log, `settled:${input.outcome}`]), + }); const providerSessionsLayer = Layer.mock(ProviderSessionManagerV2)({ closeInstance: (instanceId) => Ref.update(stopCountRef, (count) => count + 1).pipe( @@ -274,6 +282,7 @@ function makeHarness( ); return { manager, + registry, providersRef, reloadCountRef, reloadedInstancesRef, @@ -1189,6 +1198,193 @@ describe("ProviderRuntimeManager", () => { }), ); + it.effect.each(["same instance", "another provider"] as const)( + "gives foreground startup priority during background summary discovery: %s", + (choice) => + Effect.gen(function* () { + const summaryStarted = yield* Deferred.make(); + const summaries = yield* Ref.make(0); + const runs = yield* SubscriptionRef.make(0); + const actions: ProviderManagedRuntimeActions = { + getSummary: Ref.getAndUpdate(summaries, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 0 + ? Deferred.succeed(summaryStarted, undefined).pipe(Effect.andThen(Effect.never)) + : Effect.succeed(missingRuntime), + ), + ), + plan: () => Effect.succeed(installPlan()), + run: () => + SubscriptionRef.update(runs, (count) => count + 1).pipe(Effect.andThen(Effect.never)), + }; + const other: ServerProvider = { + ...provider, + instanceId: ProviderInstanceId.make("claude"), + driver: ProviderDriverKind.make("claudeAgent"), + }; + const { manager, providersRef, coordinator, lifecycleReleaseCountRef } = yield* makeHarness( + actions, + [provider, other], + ); + const background = yield* manager + .startInBackground({ + instanceId: INSTANCE, + action: "install", + catalogRevision: "reviewed:1", + attempt: 1, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(summaryStarted); + const reserved = yield* coordinator.current(INSTANCE); + assert.ok(reserved); + assert.strictEqual( + (yield* SubscriptionRef.get(providersRef))[0]?.connection?.runtime?.operation, + null, + ); + const foregroundInstance = choice === "same instance" ? INSTANCE : other.instanceId; + const foreground = yield* manager.start({ + instanceId: foregroundInstance, + action: "install", + catalogRevision: "reviewed:1", + }); + assert.strictEqual(yield* Fiber.join(background), "yielded"); + const operation = foreground.providers.find( + (candidate) => candidate.instanceId === foregroundInstance, + )?.connection?.runtime?.operation; + assert.ok(operation); + assert.notStrictEqual(operation.operationId, reserved.operationId); + assert.strictEqual( + (yield* coordinator.current(foregroundInstance))?.operationId, + operation.operationId, + ); + yield* awaitState(runs, (count) => count === 1); + assert.strictEqual(yield* SubscriptionRef.get(lifecycleReleaseCountRef), 1); + if (choice === "another provider") + assert.strictEqual(yield* coordinator.current(INSTANCE), undefined); + yield* manager.cancel({ + instanceId: foregroundInstance, + operationId: operation.operationId, + }); + assert.strictEqual(yield* coordinator.current(foregroundInstance), undefined); + assert.strictEqual(yield* SubscriptionRef.get(lifecycleReleaseCountRef), 2); + }), + ); + + it.effect.each(["waiting", "starting"] as const)( + "exposes and durably cancels restarted pending recovery while %s", + (phase) => + Effect.gen(function* () { + const before = yield* makeRequests(); + const request = { + requestId: "requested-before-restart", + instanceId: INSTANCE, + driver: CODEX, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + yield* before.requested(request); + yield* before.settled({ ...request, outcome: "retryable" }); + const requests = yield* makeRequests(); + const summaryStarted = yield* Deferred.make(); + const runs = yield* Ref.make(0); + const actions: ProviderManagedRuntimeActions = { + getSummary: Deferred.succeed(summaryStarted, undefined).pipe( + Effect.andThen(Effect.never), + ), + plan: () => Effect.succeed(installPlan()), + run: () => Ref.update(runs, (count) => count + 1), + }; + const { manager, registry, providersRef, coordinator } = yield* makeHarness( + actions, + [provider], + () => Effect.void, + undefined, + Effect.void, + { requests }, + ); + const pending = (yield* SubscriptionRef.get(providersRef))[0]?.connection?.runtime; + assert.strictEqual(pending?.operation, null); + assert.strictEqual(pending?.pendingRecovery?.requestId, request.requestId); + const background = + phase === "starting" + ? yield* manager.startInBackground({ ...request, attempt: 1 }).pipe(Effect.forkChild) + : undefined; + if (background) yield* Deferred.await(summaryStarted); + const cancelled = yield* manager.cancel({ + instanceId: INSTANCE, + requestId: request.requestId, + }); + assert.isNull(cancelled.providers[0]?.connection?.runtime?.pendingRecovery); + assert.strictEqual(cancelled.providers[0]?.connection?.runtime?.operation, null); + if (background) assert.strictEqual(yield* Fiber.join(background), "cancelled"); + assert.strictEqual(yield* coordinator.current(INSTANCE), undefined); + assert.deepStrictEqual(yield* requests.list, []); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + yield* makeRecovery().pipe( + Effect.provideService(ProviderRuntimeRequests, requests), + Effect.provideService(ProviderRuntimeManager, manager), + Effect.provideService(ProviderRegistry, registry), + ); + yield* TestClock.adjust("10 minutes"); + assert.strictEqual(yield* Ref.get(runs), 0); + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-restarted-recovery-" }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), + ); + + it.effect("cancels a requested foreground startup before its operation is published", () => + Effect.gen(function* () { + const requests = yield* makeRequests(); + const summaryStarted = yield* Deferred.make(); + const runs = yield* Ref.make(0); + const actions: ProviderManagedRuntimeActions = { + getSummary: Deferred.succeed(summaryStarted, undefined).pipe(Effect.andThen(Effect.never)), + plan: () => Effect.succeed(installPlan()), + run: () => Ref.update(runs, (count) => count + 1), + }; + const { manager, providersRef, coordinator } = yield* makeHarness( + actions, + [provider], + () => Effect.void, + undefined, + Effect.void, + { requests }, + ); + const caller = yield* manager + .start({ + instanceId: INSTANCE, + action: "install", + catalogRevision: "reviewed:1", + }) + .pipe(Effect.forkChild); + yield* Deferred.await(summaryStarted); + const pending = (yield* awaitState( + providersRef, + (providers) => providers[0]?.connection?.runtime?.pendingRecovery != null, + ))[0]!.connection!.runtime!; + assert.strictEqual(pending.operation, null); + const cancelled = yield* manager.cancel({ + instanceId: INSTANCE, + requestId: pending.pendingRecovery!.requestId, + }); + assert.isTrue(Exit.isFailure(yield* Fiber.await(caller))); + assert.isNull(cancelled.providers[0]?.connection?.runtime?.pendingRecovery); + assert.strictEqual(yield* coordinator.current(INSTANCE), undefined); + assert.strictEqual(yield* Ref.get(runs), 0); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-cancel-startup-" }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), + ); + it.effect("holds the lifecycle reservation until cancelled runtime state is published", () => Effect.gen(function* () { const cancellationPublicationStarted = yield* Deferred.make(); diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts index 399e9d95bc..10a11b9615 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts @@ -15,6 +15,7 @@ import { import { managedRuntimeFailureClass, managedRuntimeFailureRetryable, + managedRuntimeRetryAfterMs, } from "@scientfactory/provider-runtime"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; @@ -30,6 +31,7 @@ import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; import * as Scope from "effect/Scope"; import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; import type { ProviderManagedRuntimeActions, @@ -40,7 +42,7 @@ import { ProviderSessionManagerV2 } from "../../orchestration-v2/ProviderSession import { ProviderActivity } from "./ProviderActivity.ts"; import { ProviderConnectionActionError } from "./ProviderConnectionActions.ts"; import { ProviderLifecycleCoordinator } from "./ProviderLifecycleCoordinator.ts"; -import { ProviderRuntimeRequests } from "./ProviderRuntimeRequests.ts"; +import { ProviderRuntimeRequests, type RuntimeRequest } from "./ProviderRuntimeRequests.ts"; export interface ProviderRuntimeManagerShape { readonly plan: ( @@ -64,11 +66,11 @@ export interface ProviderRuntimeManagerShape { */ readonly reselect: (instanceId: ProviderInstanceId) => Effect.Effect; /** - * Starts a change the user did not ask for and waits for it to end. It is marked - * `background`, and any start by the user stops it. + * Waits for a background recovery or Scient Agent provisioning attempt. A foreground + * start preempts it until its runtime change commits. */ readonly startInBackground: ( - input: ProviderRuntimeStartInput & { readonly attempt: number }, + input: ProviderRuntimeStartInput & { readonly attempt: number; readonly requestId?: string }, ) => Effect.Effect; } @@ -101,12 +103,20 @@ interface ActiveRuntimeOperation { readonly committedRef: Ref.Ref; readonly fiberRef: Ref.Ref | undefined>; readonly transitionLock: Semaphore.Semaphore; - readonly background: { readonly attempt: number } | undefined; + readonly background: { readonly attempt: number; readonly requestId?: string } | undefined; + readonly requestId: string | undefined; readonly outcome: Deferred.Deferred; /** Set once the download is complete. */ readonly downloadRef: Ref.Ref; } +interface RuntimeStartup { + readonly instanceId: ProviderInstanceId; + readonly background: ActiveRuntimeOperation["background"]; + readonly interrupt: Effect.Effect; + readonly outcome: Deferred.Deferred; +} + type ActiveRuntimeCleanupMode = "complete" | "interrupt"; const nowIso = DateTime.now.pipe(Effect.map(DateTime.formatIso)); @@ -197,6 +207,73 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { const activeRef = yield* Ref.make>( new Map(), ); + const startupsRef = yield* Ref.make>(new Map()); + const publicationLock = yield* Semaphore.make(1); + + const pendingRecovery = ( + runtime: ProviderRuntimeSummary, + request: RuntimeRequest | undefined, + ): NonNullable | null => + request && !ACTIVE_RUNTIME_STATUSES.has(runtime.operation?.status ?? "failed") + ? { + requestId: request.requestId, + action: request.action, + nextRetryAt: DateTime.formatIso(DateTime.makeUnsafe(request.dueAt)), + } + : null; + + const publishRuntime: typeof providerRegistry.setProviderManagedRuntimeSummary = Effect.fn( + "ProviderRuntimeManager.publishRuntime", + )(function* (input) { + return yield* publicationLock.withPermits(1)( + Effect.gen(function* () { + const request = (yield* requests.list).find( + (candidate) => candidate.instanceId === input.instanceId, + ); + return yield* providerRegistry.setProviderManagedRuntimeSummary({ + ...input, + runtime: input.runtime + ? { ...input.runtime, pendingRecovery: pendingRecovery(input.runtime, request) } + : null, + }); + }), + ); + }); + + const syncPendingRecovery = Effect.fn("ProviderRuntimeManager.syncPendingRecovery")(function* () { + yield* publicationLock.withPermits(1)( + Effect.gen(function* () { + const providers = yield* providerRegistry.getProviders; + const pending = yield* requests.list; + for (const request of pending) { + const provider = providers.find( + (candidate) => candidate.instanceId === request.instanceId, + ); + if (!provider?.enabled || provider.driver !== request.driver) { + yield* requests.update(request.requestId, () => undefined); + } + } + const current = yield* requests.list; + for (const provider of providers) { + const runtime = provider.connection?.runtime; + if (!runtime) continue; + const next = pendingRecovery( + runtime, + current.find((request) => request.instanceId === provider.instanceId), + ); + if ( + (runtime.pendingRecovery?.requestId ?? null) === (next?.requestId ?? null) && + runtime.pendingRecovery?.nextRetryAt === next?.nextRetryAt + ) + continue; + yield* providerRegistry.setProviderManagedRuntimeSummary({ + instanceId: provider.instanceId, + runtime: { ...runtime, pendingRecovery: next }, + }); + } + }), + ); + }); const readTarget = Effect.fn("ProviderRuntimeManager.readTarget")(function* ( instanceId: ProviderInstanceId, @@ -286,7 +363,7 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { Effect.orElseSucceed(() => active.baseSummary), ); const finishedAt = yield* nowIso; - return yield* providerRegistry.setProviderManagedRuntimeSummary({ + return yield* publishRuntime({ instanceId, runtime: { ...latestSummary, @@ -306,7 +383,14 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { const cleanupInterruptedStart = Effect.fn("ProviderRuntimeManager.cleanupInterruptedStart")( function* (instanceId: ProviderInstanceId, operationId: string) { - const active = yield* takeIfCurrent(instanceId, operationId); + const candidate = (yield* Ref.get(activeRef)).get(instanceId); + if (candidate?.operationId !== operationId) return; + const active = yield* candidate.transitionLock.withPermits(1)( + Effect.gen(function* () { + if (yield* Ref.get(candidate.committedRef)) return undefined; + return yield* takeIfCurrent(instanceId, operationId); + }), + ); if (!active) return; yield* settleActive( active, @@ -420,7 +504,7 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { ); if (actions) { const summary = yield* actions.getSummary; - yield* providerRegistry.setProviderManagedRuntimeSummary({ + yield* publishRuntime({ instanceId: candidate.instanceId, runtime: { ...summary, @@ -463,6 +547,8 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { input: ProviderRuntimeStartInput, background: ActiveRuntimeOperation["background"], outcome: Deferred.Deferred, + operationId: string, + registeredActiveRef: Ref.Ref, ) { const target = yield* readTarget(input.instanceId); const planned = yield* plan({ instanceId: input.instanceId, action: input.action }); @@ -474,7 +560,13 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { message: "The provider setup plan changed. Review it again before continuing.", }); } - const operationId = `runtime-${yield* random.uuidV4()}`; + if ( + background?.requestId && + !(yield* requests.list).some((request) => request.requestId === background.requestId) + ) { + yield* Deferred.succeed(outcome, "cancelled"); + return yield* Effect.interrupt; + } const reserved = yield* lifecycleCoordinator.reserve({ instanceId: input.instanceId, provider: target.provider, @@ -488,10 +580,18 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { message: "Another setup or connection operation is already running for this provider.", }); } + if (background === undefined) { + yield* requests.requested({ + requestId: operationId, + instanceId: input.instanceId, + driver: target.provider, + action: input.action, + catalogRevision: input.catalogRevision, + }); + } const startedAt = yield* nowIso; const baseSummary = yield* target.actions.getSummary.pipe( - // Nothing is published yet, so Cancel cannot reach a stalled read; the - // bound keeps it from holding the reservation indefinitely. + // Bound discovery even when no client cancels the pending request. Effect.timeoutOrElse({ duration: STARTUP_SUMMARY_TIMEOUT, orElse: () => @@ -515,6 +615,25 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { : lifecycleCoordinator.release({ operationId }).pipe(Effect.asVoid), ), ); + if (background?.requestId) { + const current = yield* readTarget(input.instanceId); + const requested = (yield* requests.list).find( + (request) => request.requestId === background.requestId, + ); + if ( + !requested || + !current.snapshot.enabled || + current.provider !== target.provider || + current.actions !== target.actions || + requested.driver !== target.provider || + requested.action !== input.action || + requested.catalogRevision !== input.catalogRevision + ) { + yield* requests.update(background.requestId, () => undefined); + yield* Deferred.succeed(outcome, "cancelled"); + return yield* Effect.interrupt; + } + } const fiberRef = yield* Ref.make | undefined>(undefined); const committedRef = yield* Ref.make(false); const transitionLock = yield* Semaphore.make(1); @@ -530,17 +649,22 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { fiberRef, transitionLock, background, + requestId: background === undefined ? operationId : background.requestId, outcome, downloadRef, }; - const runtimeReserved = yield* Ref.modify(activeRef, (current) => { - if ([...current.values()].some((candidate) => candidate.provider === target.provider)) { - return [false, current] as const; - } - const next = new Map(current); - next.set(input.instanceId, active); - return [true, next] as const; - }); + const runtimeReserved = yield* Effect.uninterruptible( + Ref.modify(activeRef, (current) => { + if ([...current.values()].some((candidate) => candidate.provider === target.provider)) { + return [false, current] as const; + } + const next = new Map(current); + next.set(input.instanceId, active); + return [true, next] as const; + }).pipe( + Effect.tap((reserved) => (reserved ? Ref.set(registeredActiveRef, true) : Effect.void)), + ), + ); if (!runtimeReserved) { yield* lifecycleCoordinator.release({ operationId }); return yield* makeError({ @@ -550,14 +674,6 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { message: "Another account is already changing this shared provider runtime.", }); } - if (background === undefined) { - yield* requests.requested({ - instanceId: input.instanceId, - driver: target.provider, - action: input.action, - catalogRevision: input.catalogRevision, - }); - } const cleanupStartupExit = (exit: Exit.Exit): Effect.Effect => Exit.isSuccess(exit) @@ -575,7 +691,7 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { if (progress.download) yield* Ref.set(downloadRef, progress.download); const current = (yield* Ref.get(activeRef)).get(input.instanceId); if (current?.operationId !== operationId) return; - yield* providerRegistry.setProviderManagedRuntimeSummary({ + yield* publishRuntime({ instanceId: input.instanceId, runtime: { ...baseSummary, @@ -619,26 +735,24 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { ), ); - const initialProviders = yield* providerRegistry - .setProviderManagedRuntimeSummary({ - instanceId: input.instanceId, - runtime: { - ...baseSummary, - operation: operation({ - operationId, - action: input.action, - status: "preparing", - startedAt, - message: "Preparing the provider runtime operation.", - background, - }), - }, - }) - .pipe(Effect.onExit(cleanupStartupExit)); + const initialProviders = yield* publishRuntime({ + instanceId: input.instanceId, + runtime: { + ...baseSummary, + operation: operation({ + operationId, + action: input.action, + status: "preparing", + startedAt, + message: "Preparing the provider runtime operation.", + background, + }), + }, + }).pipe(Effect.onExit(cleanupStartupExit)); const publishUnexpectedFailure = Effect.gen(function* () { const finishedAt = yield* nowIso; - yield* providerRegistry.setProviderManagedRuntimeSummary({ + yield* publishRuntime({ instanceId: input.instanceId, runtime: { ...baseSummary, @@ -698,6 +812,10 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { } const finishedAt = yield* nowIso; const download = yield* Ref.get(downloadRef); + const retryAfterMs = + completion._tag === "Failure" + ? managedRuntimeRetryAfterMs(completion.failure) + : undefined; const latestSummary = result._tag === "Success" ? ((yield* providerRegistry.getProviders).find( @@ -708,55 +826,52 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { Effect.gen(function* () { const current = (yield* Ref.get(activeRef)).get(input.instanceId); if (current?.operationId !== operationId) return; - yield* providerRegistry - .setProviderManagedRuntimeSummary({ - instanceId: input.instanceId, - runtime: { - ...latestSummary, - operation: operation({ - operationId, - action: input.action, - status: completion._tag === "Success" ? "succeeded" : "failed", - startedAt, - finishedAt, - message: - completion._tag === "Success" - ? runtimeSuccessMessage(input.action) - : completion.failure.message, - ...(completion._tag === "Success" - ? {} - : { - failureClass: managedRuntimeFailureClass(completion.failure, lastStage), - }), - background, - download, - }), - }, - }) - .pipe( - Effect.catchCause(() => publishUnexpectedFailure), - Effect.andThen( - requests.settled({ - instanceId: input.instanceId, - driver: target.provider, - outcome: - completion._tag === "Success" - ? "succeeded" - : managedRuntimeFailureRetryable(completion.failure, lastStage) - ? "retryable" - : "failed", - }), - ), - Effect.andThen( - Deferred.succeed( - outcome, - completion._tag === "Success" ? "succeeded" : "failed", - ), - ), - Effect.ensuring( - cleanupIfCurrent(input.instanceId, operationId, "complete").pipe(Effect.asVoid), - ), - ); + yield* publishRuntime({ + instanceId: input.instanceId, + runtime: { + ...latestSummary, + operation: operation({ + operationId, + action: input.action, + status: completion._tag === "Success" ? "succeeded" : "failed", + startedAt, + finishedAt, + message: + completion._tag === "Success" + ? runtimeSuccessMessage(input.action) + : completion.failure.message, + ...(completion._tag === "Success" + ? {} + : { + failureClass: managedRuntimeFailureClass(completion.failure, lastStage), + }), + background, + download, + }), + }, + }).pipe( + Effect.catchCause(() => publishUnexpectedFailure), + Effect.andThen( + requests.settled({ + instanceId: input.instanceId, + driver: target.provider, + ...(active.requestId ? { requestId: active.requestId } : {}), + ...(retryAfterMs === undefined ? {} : { retryAfterMs }), + outcome: + completion._tag === "Success" + ? "succeeded" + : managedRuntimeFailureRetryable(completion.failure, lastStage) + ? "retryable" + : "failed", + }), + ), + Effect.andThen( + Deferred.succeed(outcome, completion._tag === "Success" ? "succeeded" : "failed"), + ), + Effect.ensuring( + cleanupIfCurrent(input.instanceId, operationId, "complete").pipe(Effect.asVoid), + ), + ); }), ); }), @@ -791,64 +906,174 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { return { providers: initialProviders }; }); - // Once asked, a runtime change belongs to the manager: planning can stall for - // a long time on a loaded machine, and a client that reconnects meanwhile only - // detaches from the reply. Cancel and manager shutdown still stop it. - const start: ProviderRuntimeManagerShape["start"] = Effect.fn("ProviderRuntimeManager.start")( - function* (input) { - yield* yieldBackgroundWork(input.instanceId); - const outcome = yield* Deferred.make(); - const fiber = yield* startup(input, undefined, outcome).pipe(Effect.forkIn(managerScope)); - return yield* Fiber.join(fiber); - }, - ); + const startupAdmission = yield* Semaphore.make(1); - // The user's own change comes first: a background change of another provider stops - // and keeps its partial download for its next try. - const yieldBackgroundWork = Effect.fn("ProviderRuntimeManager.yieldBackgroundWork")(function* ( - instanceId: ProviderInstanceId, + const interruptStartup = Effect.fn("ProviderRuntimeManager.interruptStartup")(function* ( + operationId: string, + outcome: ProviderRuntimeOutcome, ) { - const background = [...(yield* Ref.get(activeRef))].filter( - ([candidate, active]) => candidate !== instanceId && active.background !== undefined, - ); - yield* Effect.forEach( - background, - ([candidate, active]) => - active.transitionLock - .withPermits(1)( + const starting = (yield* Ref.get(startupsRef)).get(operationId); + if (!starting) return; + yield* Deferred.succeed(starting.outcome, outcome); + yield* starting.interrupt; + }); + + const beginStartup = Effect.fn("ProviderRuntimeManager.beginStartup")(function* ( + input: ProviderRuntimeStartInput, + background: ActiveRuntimeOperation["background"], + outcome: Deferred.Deferred, + ) { + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const operationId = `runtime-${yield* random.uuidV4()}`; + const registered = yield* Deferred.make(); + const registeredActiveRef = yield* Ref.make(false); + const fiber = yield* restore( + Deferred.await(registered).pipe( + Effect.andThen(startup(input, background, outcome, operationId, registeredActiveRef)), + ), + ).pipe( + Effect.onExit((exit) => Effect.gen(function* () { - if (yield* Ref.get(active.committedRef)) return undefined; - return yield* takeIfCurrent(candidate, active.operationId); + if (Exit.isFailure(exit)) { + yield* cleanupInterruptedStart(input.instanceId, operationId); + // After registration, the active operation's claimant owns release through publication. + if (!(yield* Ref.get(registeredActiveRef))) { + const reservation = yield* lifecycleCoordinator.current(input.instanceId); + if (reservation?.operationId === operationId) { + yield* lifecycleCoordinator.release({ operationId }); + } + } + yield* Deferred.succeed(outcome, "interrupted"); + } + yield* Ref.update(startupsRef, (current) => { + const next = new Map(current); + next.delete(operationId); + return next; + }); }), - ) - .pipe( - Effect.flatMap((claim) => - claim - ? settleActive(claim, "interrupt", publishCancelled(candidate, claim), "yielded") - : Effect.void, - ), - Effect.ignore, ), - { discard: true }, + Effect.forkIn(managerScope), + ); + yield* Ref.update(startupsRef, (current) => + new Map(current).set(operationId, { + instanceId: input.instanceId, + background, + interrupt: Fiber.interrupt(fiber).pipe(Effect.asVoid), + outcome, + }), + ); + yield* Deferred.succeed(registered, undefined); + return fiber; + }), ); }); + const yieldBackgroundWork = Effect.fn("ProviderRuntimeManager.yieldBackgroundWork")(function* () { + const background = [...(yield* Ref.get(activeRef))].filter( + ([, active]) => active.background !== undefined, + ); + for (const [instanceId, active] of background) { + const claim = yield* active.transitionLock.withPermits(1)( + Effect.gen(function* () { + if (yield* Ref.get(active.committedRef)) return undefined; + return yield* takeIfCurrent(instanceId, active.operationId); + }), + ); + if (!claim) continue; + yield* interruptStartup(active.operationId, "yielded"); + yield* settleActive(claim, "interrupt", publishCancelled(instanceId, claim), "yielded").pipe( + Effect.ignore, + ); + } + for (const [operationId, starting] of yield* Ref.get(startupsRef)) { + if (!starting.background) continue; + // Committed changes must finish reconciliation; only unregistered startup is left to yield. + if ((yield* Ref.get(activeRef)).get(starting.instanceId)?.operationId === operationId) + continue; + yield* interruptStartup(operationId, "yielded"); + } + }); + + // A disconnected caller detaches from the reply; the manager still owns the startup. + const start: ProviderRuntimeManagerShape["start"] = Effect.fn("ProviderRuntimeManager.start")( + function* (input) { + const outcome = yield* Deferred.make(); + const fiber = yield* startupAdmission.withPermits(1)( + Effect.gen(function* () { + yield* yieldBackgroundWork(); + return yield* beginStartup(input, undefined, outcome); + }), + ); + return yield* Fiber.join(fiber); + }, + ); + const startInBackground: ProviderRuntimeManagerShape["startInBackground"] = Effect.fn( "ProviderRuntimeManager.startInBackground", )(function* (input) { const outcome = yield* Deferred.make(); - yield* startup(input, { attempt: input.attempt }, outcome).pipe( - Effect.forkIn(managerScope), - Effect.flatMap(Fiber.join), + const fiber = yield* startupAdmission.withPermits(1)( + Effect.gen(function* () { + if ([...(yield* Ref.get(startupsRef)).values()].some((starting) => !starting.background)) { + return yield* makeError({ + provider: ProviderDriverKind.make("unknown"), + instanceId: input.instanceId, + reason: "runtime_busy", + message: "A requested runtime change is starting.", + }); + } + return yield* beginStartup( + input, + { + attempt: input.attempt, + ...(input.requestId ? { requestId: input.requestId } : {}), + }, + outcome, + ); + }), ); + const exit = yield* Fiber.await(fiber); + if (Exit.isFailure(exit) && !Cause.hasInterruptsOnly(exit.cause)) { + return yield* Effect.failCause(exit.cause); + } return yield* Deferred.await(outcome); }); const cancel: ProviderRuntimeManagerShape["cancel"] = Effect.fn("ProviderRuntimeManager.cancel")( function* (input) { const target = yield* readTarget(input.instanceId); - const candidate = (yield* Ref.get(activeRef)).get(input.instanceId); - if (!candidate || candidate.operationId !== input.operationId) { + let candidate = (yield* Ref.get(activeRef)).get(input.instanceId); + const pending = + "requestId" in input + ? (yield* requests.list).find( + (request) => + request.instanceId === input.instanceId && request.requestId === input.requestId, + ) + : undefined; + let matchesActive = + candidate && + ("operationId" in input + ? candidate.operationId === input.operationId + : pending !== undefined && candidate.requestId === input.requestId); + if (!matchesActive && pending) { + yield* requests.update(pending.requestId, () => undefined); + for (const [operationId, starting] of yield* Ref.get(startupsRef)) { + if ( + operationId === pending.requestId || + starting.background?.requestId === pending.requestId + ) { + yield* interruptStartup(operationId, "cancelled"); + } + } + candidate = (yield* Ref.get(activeRef)).get(input.instanceId); + if (candidate?.requestId !== pending.requestId) { + yield* syncPendingRecovery(); + return { providers: yield* providerRegistry.getProviders }; + } + matchesActive = true; + } + if (!candidate || !matchesActive) { return yield* makeError({ provider: target.provider, instanceId: input.instanceId, @@ -856,12 +1081,13 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { message: "The provider runtime operation is no longer active.", }); } + const operationId = candidate.operationId; const claim = yield* candidate.transitionLock.withPermits(1)( Effect.gen(function* () { const current = (yield* Ref.get(activeRef)).get(input.instanceId); - if (current?.operationId !== input.operationId) return undefined; + if (current?.operationId !== operationId) return undefined; if (yield* Ref.get(current.committedRef)) return "committed" as const; - return yield* takeIfCurrent(input.instanceId, input.operationId); + return yield* takeIfCurrent(input.instanceId, operationId); }), ); if (claim === "committed") { @@ -880,17 +1106,19 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { message: "The provider runtime operation is no longer active.", }); } + yield* requests.settled({ + instanceId: input.instanceId, + driver: target.provider, + outcome: "cancelled", + ...(claim.requestId ? { requestId: claim.requestId } : {}), + }); + yield* interruptStartup(claim.operationId, "cancelled"); const providers = yield* settleActive( claim, "interrupt", publishCancelled(input.instanceId, claim), "cancelled", ); - yield* requests.settled({ - instanceId: input.instanceId, - driver: target.provider, - outcome: "cancelled", - }); return { providers }; }, ); @@ -983,6 +1211,16 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { ); }).pipe(Effect.delay(RESELECT_INTERVAL), Effect.forever, Effect.forkScoped); + yield* syncPendingRecovery(); + yield* requests.changes.pipe( + Stream.runForEach(() => syncPendingRecovery()), + Effect.forkScoped, + ); + yield* providerRegistry.streamChanges.pipe( + Stream.runForEach(() => syncPendingRecovery()), + Effect.forkScoped, + ); + return ProviderRuntimeManager.of({ plan, start, cancel, reselect, startInBackground }); }); diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts index dbd08b8f43..22078d31f8 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts @@ -1,6 +1,7 @@ import * as NodeOS from "node:os"; import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { assert, describe, it } from "@effect/vitest"; import { ProviderDriverKind, @@ -64,6 +65,7 @@ const plan = (catalogRevision: string) => ({ }); const request = { + requestId: "requested-install", instanceId: INSTANCE, driver: CODEX, action: "install" as const, @@ -85,11 +87,37 @@ describe("ProviderRuntimeRequests", () => { }).pipe( Effect.provide( ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-requests-" }).pipe( - Layer.provideMerge(NodeServices.layer), + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), ), ), ), ); + + it.effect.each([1_000, 2 * 60 * MINUTE])( + "persists the later of retry jitter and the source cooldown (%s ms)", + (retryAfterMs) => + Effect.gen(function* () { + const requests = yield* makeRequests(); + yield* requests.requested(request); + const now = yield* Clock.currentTimeMillis; + yield* requests.settled({ ...request, outcome: "retryable", retryAfterMs }); + const [kept] = yield* (yield* makeRequests()).list; + assert.deepInclude(kept, request); + assert.isAtLeast(kept!.dueAt, now + Math.max(retryAfterMs, 4 * MINUTE)); + assert.isAtMost(kept!.dueAt, now + Math.max(retryAfterMs, 6 * MINUTE)); + const replacement = { ...request, requestId: "replacement", catalogRevision: "reviewed:2" }; + yield* requests.requested(replacement); + yield* requests.settled({ ...request, outcome: "cancelled" }); + yield* requests.update(request.requestId, () => undefined); + assert.deepInclude((yield* requests.list)[0], replacement); + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-source-cooldown-" }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), + ); }); describe("ProviderRuntimeRecovery", () => { @@ -135,7 +163,7 @@ describe("ProviderRuntimeRecovery", () => { }).pipe( Effect.provide( ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-recovery-" }).pipe( - Layer.provideMerge(NodeServices.layer), + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), ), ), ); diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts index df759791a2..81882887af 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts @@ -47,15 +47,16 @@ export const make = Effect.fn("ProviderRuntimeRecovery.make")(function* () { .pipe(Effect.option) : Option.none(); if (Option.isNone(planned) || planned.value.catalogRevision !== request.catalogRevision) { - return yield* requests.update(request.instanceId, () => undefined); + return yield* requests.update(request.requestId, () => undefined); } const attempt = request.retries + 1; const lease = yield* nextTryAt(attempt); - yield* requests.update(request.instanceId, (current) => ({ + yield* requests.update(request.requestId, (current) => ({ ...current, retries: attempt, dueAt: lease, })); + if (!(yield* requests.list).some((current) => current.requestId === request.requestId)) return; // The manager settles the request when the change ends; only a try that never ran is undone. const outcome = yield* manager .startInBackground({ @@ -63,12 +64,13 @@ export const make = Effect.fn("ProviderRuntimeRecovery.make")(function* () { action: request.action, catalogRevision: request.catalogRevision, attempt, + requestId: request.requestId, }) .pipe(Effect.orElseSucceed(() => "busy" as const)); if (outcome === "interrupted") return yield* Effect.interrupt; if (outcome === "yielded" || outcome === "busy") { const later = (yield* Clock.currentTimeMillis) + BUSY_DELAY_MS; - yield* requests.update(request.instanceId, (current) => + yield* requests.update(request.requestId, (current) => current.retries === attempt ? { ...current, retries: request.retries, dueAt: later } : current, diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts index aa26f9cbc8..f6fe003cdc 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts @@ -7,14 +7,16 @@ import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Random from "effect/Random"; -import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import * as SubscriptionRef from "effect/SubscriptionRef"; import { ServerConfig } from "../../config.ts"; @@ -25,6 +27,7 @@ const ACP_REGISTRY = ProviderDriverKind.make("acpRegistry"); /** An install, update or repair the user asked for that has not succeeded yet. */ const RuntimeRequest = Schema.Struct({ + requestId: Schema.String, instanceId: ProviderInstanceId, driver: ProviderDriverKind, action: Schema.Literals(["install", "update", "repair"]), @@ -37,7 +40,13 @@ const RuntimeRequest = Schema.Struct({ }); export type RuntimeRequest = typeof RuntimeRequest.Type; const RequestsJson = Schema.fromJsonString(Schema.Array(RuntimeRequest)); -const decodeRequests = Schema.decodeEffect(RequestsJson); +const decodeRequests = Schema.decodeEffect( + Schema.fromJsonString( + Schema.Array( + Schema.Struct({ ...RuntimeRequest.fields, requestId: Schema.optionalKey(Schema.String) }), + ), + ), +); const encodeRequests = Schema.encodeEffect(RequestsJson); /** How a change of a runtime ended; `retryable` failed in a way a later try may not. */ @@ -67,6 +76,7 @@ export class ProviderRuntimeRequests extends Context.Service< { /** The user started a change: it replaces any request for the same runtime; Remove only clears. */ readonly requested: (input: { + readonly requestId: string; readonly instanceId: ProviderInstanceId; readonly driver: ProviderDriverKind; readonly action: ProviderManagedRuntimeAction; @@ -76,11 +86,14 @@ export class ProviderRuntimeRequests extends Context.Service< readonly instanceId: ProviderInstanceId; readonly driver: ProviderDriverKind; readonly outcome: RuntimeChangeOutcome; + readonly requestId?: string; + readonly retryAfterMs?: number; }) => Effect.Effect; readonly list: Effect.Effect>; - /** Changes the instance's request if there still is one; `undefined` drops it. */ + readonly changes: Stream.Stream>; + /** Changes this exact request if it still exists; `undefined` drops it. */ readonly update: ( - instanceId: ProviderInstanceId, + requestId: string, change: (request: RuntimeRequest) => RuntimeRequest | undefined, ) => Effect.Effect; } @@ -91,12 +104,18 @@ export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const platform = yield* HostProcess.Platform; + const random = yield* SecureRandom.SecureRandom; // Kept outside the runtime folders, which Remove deletes. const filePath = path.join(config.baseDir, "provider-runtimes", ".background", "requests.json"); - const requestsRef = yield* Ref.make>( - yield* fs.readFileString(filePath).pipe( - Effect.flatMap(decodeRequests), - Effect.orElseSucceed((): ReadonlyArray => []), + const stored = yield* fs.readFileString(filePath).pipe( + Effect.flatMap(decodeRequests), + Effect.orElseSucceed(() => []), + ); + const requestsRef = yield* SubscriptionRef.make>( + yield* Effect.forEach(stored, (request) => + Effect.gen(function* () { + return { ...request, requestId: request.requestId ?? `request-${yield* random.uuidV4()}` }; + }), ), ); const lock = yield* Semaphore.make(1); @@ -105,28 +124,36 @@ export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { next: (requests: ReadonlyArray) => Effect.Effect>, ) => lock.withPermits(1)( - Effect.gen(function* () { - const updated = yield* next(yield* Ref.get(requestsRef)); - yield* Ref.set(requestsRef, updated); - yield* encodeRequests(updated).pipe( - Effect.flatMap((contents) => - writeFileStringAtomically({ filePath, contents, mode: 0o600 }), - ), - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(Path.Path, path), - Effect.provideService(HostProcess.Platform, platform), - Effect.ignoreCause({ log: true }), - ); - }), + Effect.uninterruptible( + Effect.gen(function* () { + const current = yield* SubscriptionRef.get(requestsRef); + const updated = yield* next(current); + if (updated === current) return; + yield* encodeRequests(updated).pipe( + Effect.flatMap((contents) => + writeFileStringAtomically({ filePath, contents, mode: 0o600 }), + ), + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(HostProcess.Platform, platform), + Effect.ignoreCause({ log: true }), + ); + yield* SubscriptionRef.set(requestsRef, updated); + }), + ), ); + if (stored.some((request) => request.requestId === undefined)) { + yield* change((requests) => Effect.succeed([...requests])); + } + const updateEach = ( requests: ReadonlyArray, - instanceId: ProviderInstanceId, + requestId: string, next: (request: RuntimeRequest) => RuntimeRequest | undefined, ) => requests.flatMap((request) => { - if (request.instanceId !== instanceId) return [request]; + if (request.requestId !== requestId) return [request]; const updated = next(request); return updated ? [updated] : []; }); @@ -138,10 +165,10 @@ export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { yield* change((requests) => { const others = requests.filter((request) => !sameRuntime(request, input)); if (input.action === "remove") return Effect.succeed(others); - const { instanceId, driver, action, catalogRevision } = input; + const { requestId, instanceId, driver, action, catalogRevision } = input; return Effect.succeed([ ...others, - { instanceId, driver, action, catalogRevision, retries: 0, dueAt }, + { requestId, instanceId, driver, action, catalogRevision, retries: 0, dueAt }, ]); }); }); @@ -151,17 +178,24 @@ export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { ) { yield* change((requests) => Effect.gen(function* () { + const request = requests.find((candidate) => + input.requestId === undefined + ? candidate.instanceId === input.instanceId && sameRuntime(candidate, input) + : candidate.requestId === input.requestId, + ); + if (!request) return requests; switch (input.outcome) { case "succeeded": return requests.filter((request) => !sameRuntime(request, input)); case "cancelled": case "failed": - return updateEach(requests, input.instanceId, () => undefined); + return updateEach(requests, request.requestId, () => undefined); case "retryable": { - const request = requests.find((candidate) => candidate.instanceId === input.instanceId); - if (!request) return requests; - const dueAt = yield* nextTryAt(request.retries); - return updateEach(requests, input.instanceId, (current) => ({ ...current, dueAt })); + const dueAt = Math.max( + yield* nextTryAt(request.retries), + (yield* Clock.currentTimeMillis) + (input.retryAfterMs ?? 0), + ); + return updateEach(requests, request.requestId, (current) => ({ ...current, dueAt })); } } }), @@ -169,13 +203,25 @@ export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { }); const update = Effect.fn("ProviderRuntimeRequests.update")(function* ( - instanceId: ProviderInstanceId, + requestId: string, next: (request: RuntimeRequest) => RuntimeRequest | undefined, ) { - yield* change((requests) => Effect.succeed(updateEach(requests, instanceId, next))); + yield* change((requests) => + Effect.succeed( + requests.some((request) => request.requestId === requestId) + ? updateEach(requests, requestId, next) + : requests, + ), + ); }); - return ProviderRuntimeRequests.of({ requested, settled, list: Ref.get(requestsRef), update }); + return ProviderRuntimeRequests.of({ + requested, + settled, + list: SubscriptionRef.get(requestsRef), + changes: SubscriptionRef.changes(requestsRef), + update, + }); }); export const layer = Layer.effect(ProviderRuntimeRequests, make()); diff --git a/apps/web/src/scient/providerConnection/ProviderRuntimeSection.browser.test.tsx b/apps/web/src/scient/providerConnection/ProviderRuntimeSection.browser.test.tsx index cde7c6ef52..afac648c95 100644 --- a/apps/web/src/scient/providerConnection/ProviderRuntimeSection.browser.test.tsx +++ b/apps/web/src/scient/providerConnection/ProviderRuntimeSection.browser.test.tsx @@ -1836,6 +1836,7 @@ describe("ProviderRuntimeSection", () => { const cancelledProvider: ServerProvider = { ...provider, + driver: ProviderDriverKind.make("claudeAgent"), installed: true, version: "2.1.170", status: "ready", @@ -1877,6 +1878,51 @@ describe("ProviderRuntimeSection", () => { expect(markup).toContain('aria-label="Use Scient-managed Claude"'); expect(markup).not.toContain("Provider runtime setup cancelled"); expect(markup).not.toContain("previous working runtime"); + + const restartedProvider: ServerProvider = { + ...cancelledProvider, + connection: { + ...cancelledProvider.connection!, + runtime: { + ...cancelledProvider.connection!.runtime!, + operation: null, + pendingRecovery: { + requestId: "requested-before-restart", + action: "install", + nextRetryAt: "2026-11-16T12:05:00.000Z", + }, + }, + }, + }; + const clearedProvider: ServerProvider = { + ...restartedProvider, + connection: { + ...restartedProvider.connection!, + runtime: { ...restartedProvider.connection!.runtime!, pendingRecovery: null }, + }, + }; + commands.cancel.mockResolvedValue({ _tag: "Success", value: { providers: [clearedProvider] } }); + const host = await renderSection(_fixture, { + compact: true, + environmentId, + provider: restartedProvider, + displayName: "Claude", + }); + expect(host.textContent).toContain("Scient will retry this requested install after"); + expect(host.querySelector('[role="progressbar"]')).toBeNull(); + expect(host.querySelector('[aria-label="Cancel Claude install retry"]')).not.toBeNull(); + const cancelRetry = findActionButton(host, "Cancel retry"); + expect(cancelRetry).toBeDefined(); + await act(() => cancelRetry!.click()); + await vi.waitFor(() => { + expect(commands.cancel).toHaveBeenCalledWith({ + environmentId, + input: { instanceId, requestId: "requested-before-restart" }, + }); + expect(host.textContent).not.toContain("Cancel retry"); + }); + expect(host.textContent).toContain("System installation"); + expect(commands.start).not.toHaveBeenCalled(); }); it("shows the current missing-runtime state after removal instead of a stale success row", async () => { diff --git a/apps/web/src/scient/providerConnection/ProviderRuntimeSection.tsx b/apps/web/src/scient/providerConnection/ProviderRuntimeSection.tsx index e166329380..c17158ab3b 100644 --- a/apps/web/src/scient/providerConnection/ProviderRuntimeSection.tsx +++ b/apps/web/src/scient/providerConnection/ProviderRuntimeSection.tsx @@ -75,6 +75,11 @@ type LocalRuntimeFailure = readonly kind: "operation"; readonly operationId: ProviderRuntimeOperation["operationId"]; readonly message: string; + } + | { + readonly kind: "recovery"; + readonly requestId: string; + readonly message: string; }; function runtimeFromResult( @@ -101,6 +106,9 @@ function localRuntimeFailureMessage( ? failure.message : null; } + if (failure.kind === "recovery") { + return runtime.pendingRecovery?.requestId === failure.requestId ? failure.message : null; + } const operation = runtime.operation; return operation?.operationId === failure.operationId && isActiveProviderRuntimeOperation(operation) @@ -114,6 +122,12 @@ export function resolveProviderRuntimeForPresentation( ): ProviderRuntimeSummary | null | undefined { if (!serverRuntime) return localRuntime; if (!localRuntime) return serverRuntime; + if ( + serverRuntime.pendingRecovery && + localRuntime.pendingRecovery === null && + !isActiveProviderRuntimeOperation(localRuntime.operation) + ) + return localRuntime; if (serverRuntime.operation?.operationId === localRuntime.operation?.operationId) { return serverRuntime; } @@ -258,6 +272,7 @@ export function ProviderRuntimeSection(props: { const operation = runtime?.operation ?? null; const runtimeActions = runtime?.actions; const activeOperation = isActiveProviderRuntimeOperation(operation) ? operation : null; + const pendingRecovery = runtime?.pendingRecovery; const plan = !activeOperation && preparedPlan?.action === "remove" && runtimeActions?.includes("remove") ? preparedPlan @@ -444,27 +459,30 @@ export function ProviderRuntimeSection(props: { }; const cancel = async () => { - if (!activeOperation) return; + if (!activeOperation && !pendingRecovery) return; setLocalFailure(null); setPendingAction("cancel"); const result = await cancelRuntime({ environmentId, input: { instanceId: provider.instanceId, - operationId: activeOperation.operationId, + ...(activeOperation + ? { operationId: activeOperation.operationId } + : { requestId: pendingRecovery!.requestId }), }, }); setPendingAction(null); if (result._tag === "Failure") { if (!isAtomCommandInterrupted(result)) { - setLocalFailure({ - kind: "operation", - operationId: activeOperation.operationId, - message: providerLifecycleFailureMessage( - squashAtomCommandFailure(result), - `Scient could not cancel the ${runtimeDisplayName} runtime operation.`, - ), - }); + const message = providerLifecycleFailureMessage( + squashAtomCommandFailure(result), + `Scient could not cancel the ${runtimeDisplayName} runtime operation.`, + ); + setLocalFailure( + activeOperation + ? { kind: "operation", operationId: activeOperation.operationId, message } + : { kind: "recovery", requestId: pendingRecovery!.requestId, message }, + ); } return; } @@ -650,6 +668,28 @@ export function ProviderRuntimeSection(props: { : "space-y-3 rounded-lg border p-3" } > + {pendingRecovery ? ( +
+

+ Scient will retry this requested {pendingRecovery.action} after{" "} + + . +

+ +
+ ) : null}
(); const fetchVerified = async (copy: { readonly url: string; readonly mirrorUrl?: string | undefined; @@ -728,6 +733,7 @@ export class ManagedProviderRuntime { allowedHosts: [...artifact.allowedHosts, new URL(copy.mirrorUrl).hostname], } : { url: copy.url, allowedHosts: artifact.allowedHosts }), + sourceCooldowns, checksum: copy.checksum, destination: copy.destination, partialPath: this.#partialDownloadPath(copy.checksum), @@ -765,21 +771,49 @@ export class ManagedProviderRuntime { } catch (cause) { if (signal.aborted) throw cause; await NodeFSP.rm(archivePath, { force: true }); + compressedCause = cause; compressedFailure = managedRuntimeFailureClass(cause); input.onCompressedFallback?.(cause); } finally { await NodeFSP.rm(compressedPath, { force: true }); } } - await fetchVerified({ - url: artifact.url, - mirrorUrl: managedRuntimeMirrorUrl(artifact), - checksum: artifact.checksum, - size: artifact.size, - destination: archivePath, - compressed: false, - }); - await this.#dependencies.verify(archivePath, artifact.checksum); + try { + await fetchVerified({ + url: artifact.url, + mirrorUrl: managedRuntimeMirrorUrl(artifact), + checksum: artifact.checksum, + size: artifact.size, + destination: archivePath, + compressed: false, + }); + await this.#dependencies.verify(archivePath, artifact.checksum); + } catch (cause) { + if (signal.aborted || compressedCause === undefined) throw cause; + const failureClass = managedRuntimeFailureClass(cause); + const sourceFailure = + failureClass === "http_4xx" || + failureClass === "http_5xx" || + failureClass === "network" || + failureClass === "timeout" || + failureClass === "checksum" || + failureClass === "size_mismatch" || + failureClass === "certificate"; + throw new ManagedRuntimeFileError( + cause instanceof Error ? cause.message : "Managed runtime download failed.", + { + cause, + failureClass, + retryable: + managedRuntimeFailureRetryable(cause) || + (sourceFailure && managedRuntimeFailureRetryable(compressedCause)), + retryAfterMs: Math.max( + managedRuntimeRetryAfterMs(cause) ?? 0, + managedRuntimeRetryAfterMs(compressedCause) ?? 0, + ), + }, + ); + } } async #installOwned( diff --git a/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts b/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts index 9d3143ce74..d5a4174320 100644 --- a/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts +++ b/packages/scient-provider-runtime/src/managedScientAgentDownload.test.ts @@ -12,6 +12,7 @@ import type { ManagedProviderRuntimeProgress } from "./managedProviderRuntime.ts import { hydrateManagedRuntimeArtifact } from "./managedRuntimeArtifact.ts"; import { ManagedScientAgentRuntime } from "./managedScientAgentRuntime.ts"; import { resolveScientAgentArtifactPolicy } from "./scientAgentManifest.ts"; +import { managedRuntimeFailureRetryable, managedRuntimeRetryAfterMs } from "./runtimeFiles.ts"; vi.mock("node:timers/promises", { spy: true }); // The transient-response backoff resolves at once but still honours cancellation. @@ -63,6 +64,7 @@ async function installedFixture() { } afterEach(async () => { + vi.useRealTimers(); vi.unstubAllGlobals(); await Promise.all( roots.splice(0).map((root) => NodeFSP.rm(root, { recursive: true, force: true })), @@ -335,6 +337,49 @@ describe("installing from the compressed copy", () => { }, ); + it("carries a compressed source cooldown through raw fallback and the terminal failure", async () => { + const { root, previousState } = await installedFixture(); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(0); + using wait = vi.spyOn(NodeTimersPromises, "setTimeout"); + wait.mockClear(); + const urls: string[] = []; + const fetch = vi.fn(async (url: URL) => { + urls.push(String(url)); + return url.hostname === "downloads.scientfactory.com" + ? new Response("busy", { status: 429, headers: { "retry-after": "3600" } }) + : new Response("missing", { status: 404 }); + }); + vi.stubGlobal("fetch", fetch); + const runtime = new ManagedScientAgentRuntime(root, { smoke: async () => undefined }); + const compressedFailures: unknown[] = []; + const failure = await runtime + .install({ + artifact: compressedArtifact("0.2.0"), + signal: new AbortController().signal, + onCompressedFallback: (cause) => { + compressedFailures.push(cause); + vi.advanceTimersByTime(10_000); + }, + }) + .catch((cause: unknown) => cause); + expect(compressedFailures).toHaveLength(1); + expect(managedRuntimeRetryAfterMs(compressedFailures[0])).toBe(3_590_000); + expect(managedRuntimeRetryAfterMs(new Error("wrapped", { cause: failure }))).toBe(3_590_000); + expect(managedRuntimeFailureRetryable(failure)).toBe(true); + expect(String(failure)).toContain("github.com Managed runtime download failed with HTTP 404"); + expect(urls).toEqual([ + "https://downloads.scientfactory.com/scient-agent/v0.2.0/scient-agent-darwin-arm64.zst", + "https://github.com/ScientFactory/scient-agent/releases/download/v0.2.0/scient-agent-darwin-arm64.zst", + "https://github.com/ScientFactory/scient-agent/releases/download/v0.2.0/scient-agent-darwin-arm64", + ]); + expect(wait).not.toHaveBeenCalled(); + expect(await runtime.readState()).toEqual(previousState); + expect( + await NodeFSP.readdir(NodePath.join(root, "provider-runtimes", "scient-agent", "staging")), + ).toEqual([]); + }); + it("stops without the binary when told not to fall back, or when cancelled", async () => { const { root, previousState } = await installedFixture(); const runtime = new ManagedScientAgentRuntime(root, { smoke: async () => undefined }); diff --git a/packages/scient-provider-runtime/src/runtimeDownload.test.ts b/packages/scient-provider-runtime/src/runtimeDownload.test.ts index 047ed30351..6ec7522253 100644 --- a/packages/scient-provider-runtime/src/runtimeDownload.test.ts +++ b/packages/scient-provider-runtime/src/runtimeDownload.test.ts @@ -12,6 +12,7 @@ import { ManagedRuntimeFileError, managedRuntimeFailureClass, managedRuntimeFailureRetryable, + managedRuntimeRetryAfterMs, verifyManagedRuntimeChecksum, } from "./runtimeFiles.ts"; @@ -19,12 +20,14 @@ vi.mock("node:timers/promises", { spy: true }); // The retry backoff resolves at once but still honours cancellation; the // "cancels the retry delay promptly" test drives the real timer. beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); vi.mocked(NodeTimersPromises.setTimeout).mockImplementation((async ( - _delay: number, + delay: number, value: unknown, options?: { signal?: AbortSignal }, ) => { options?.signal?.throwIfAborted(); + vi.advanceTimersByTime(delay); return value; }) as never); }); @@ -111,41 +114,60 @@ describe("transient managed runtime downloads", () => { await expect(NodeFSP.stat(request.destination)).rejects.toMatchObject({ code: "ENOENT" }); }); - it("waits as long as the server asks, up to a minute", async () => { + it("honours a short server cooldown inside the local retry budget", async () => { const request = await input(); const fetch = vi .fn() .mockResolvedValueOnce(new Response("busy", { status: 503, headers: { "retry-after": "7" } })) - .mockResolvedValueOnce( - new Response("busy", { status: 429, headers: { "retry-after": "3600" } }), - ) .mockResolvedValueOnce(new Response("ok", { headers: { "content-length": "2" } })); vi.stubGlobal("fetch", fetch); vi.mocked(NodeTimersPromises.setTimeout).mockClear(); await downloadManagedRuntime(request); expect(vi.mocked(NodeTimersPromises.setTimeout).mock.calls.map(([delay]) => delay)).toEqual([ - 7_000, 60_000, + 7_000, ]); + expect(fetch).toHaveBeenCalledTimes(2); }); - it("cancels recovery without sending another request", async () => { - const controller = new AbortController(); - const request = await input(controller.signal); - const fetch = vi.fn().mockImplementation(() => - Promise.resolve( - new Response( - new ReadableStream({ - cancel: () => controller.abort(), - }), - { status: 503 }, + it.each(["3600", "99999999999999999999999999999999999999999999999999999999999999999999999"])( + "leaves long Retry-After %s to durable recovery without waiting locally", + async (header) => { + const request = await input(); + const fetch = vi.fn( + async () => new Response("busy", { status: 429, headers: { "retry-after": header } }), + ); + vi.stubGlobal("fetch", fetch); + vi.mocked(NodeTimersPromises.setTimeout).mockClear(); + const failure = await downloadManagedRuntime(request).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ManagedRuntimeFileError); + expect(managedRuntimeFailureRetryable(failure)).toBe(true); + expect(managedRuntimeRetryAfterMs(failure)).toBe(header === "3600" ? 3_600_000 : 14_400_000); + expect(fetch).toHaveBeenCalledOnce(); + expect(NodeTimersPromises.setTimeout).not.toHaveBeenCalled(); + }, + ); + + it.each([undefined, "3600"])( + "cancels recovery without another request (Retry-After %s)", + async (header) => { + const controller = new AbortController(); + const request = await input(controller.signal); + const fetch = vi.fn().mockImplementation(() => + Promise.resolve( + new Response( + new ReadableStream({ + cancel: () => controller.abort(), + }), + { status: 503, ...(header ? { headers: { "retry-after": header } } : {}) }, + ), ), - ), - ); - vi.stubGlobal("fetch", fetch); - await expect(downloadManagedRuntime(request)).rejects.toMatchObject({ name: "AbortError" }); - expect(fetch).toHaveBeenCalledOnce(); - await expect(NodeFSP.stat(request.destination)).rejects.toMatchObject({ code: "ENOENT" }); - }); + ); + vi.stubGlobal("fetch", fetch); + await expect(downloadManagedRuntime(request)).rejects.toMatchObject({ name: "AbortError" }); + expect(fetch).toHaveBeenCalledOnce(); + await expect(NodeFSP.stat(request.destination)).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); it("retains host admission when a retried response redirects", async () => { const request = await input(); @@ -209,37 +231,46 @@ describe("transient managed runtime downloads", () => { await expect(NodeFSP.stat(request.destination)).rejects.toMatchObject({ code: "ENOENT" }); }); - it("cancels the retry delay promptly", async () => { - const controller = new AbortController(); - const request = await input(controller.signal); - const bodyReleased = Promise.withResolvers(); - const fetch = vi.fn().mockImplementation( - async () => - new Response(new ReadableStream({ cancel: () => bodyReleased.resolve() }), { - status: 503, - }), - ); - vi.stubGlobal("fetch", fetch); - const retryStarted = Promise.withResolvers(); - const { setTimeout: waitForRetry } = - await vi.importActual("node:timers/promises"); - vi.mocked(NodeTimersPromises.setTimeout).mockClear(); - using retryTimer = vi.spyOn(NodeTimersPromises, "setTimeout").mockImplementation((...args) => { - const waiting = waitForRetry(...args); - retryStarted.resolve(); - return waiting; - }); - const download = downloadManagedRuntime(request); - await bodyReleased.promise; - await retryStarted.promise; - const [delay] = retryTimer.mock.calls[0]!; - // The first wait is about two seconds, spread by up to a fifth either way. - expect(delay).toBeGreaterThanOrEqual(1_600); - expect(delay).toBeLessThanOrEqual(2_400); - controller.abort(); - await expect(download).rejects.toMatchObject({ name: "AbortError" }); - expect(fetch).toHaveBeenCalledOnce(); - }); + it.each([undefined, "60"])( + "cancels the retry delay promptly (Retry-After %s)", + async (header) => { + const controller = new AbortController(); + const request = await input(controller.signal); + const bodyReleased = Promise.withResolvers(); + const fetch = vi.fn().mockImplementation( + async () => + new Response(new ReadableStream({ cancel: () => bodyReleased.resolve() }), { + status: 503, + ...(header ? { headers: { "retry-after": header } } : {}), + }), + ); + vi.stubGlobal("fetch", fetch); + const retryStarted = Promise.withResolvers(); + const { setTimeout: waitForRetry } = + await vi.importActual("node:timers/promises"); + vi.mocked(NodeTimersPromises.setTimeout).mockClear(); + using retryTimer = vi + .spyOn(NodeTimersPromises, "setTimeout") + .mockImplementation((...args) => { + const waiting = waitForRetry(...args); + retryStarted.resolve(); + return waiting; + }); + const download = downloadManagedRuntime(request); + await bodyReleased.promise; + await retryStarted.promise; + const [delay] = retryTimer.mock.calls[0]!; + if (header) expect(delay).toBe(60_000); + else { + // The first wait is about two seconds, spread by up to a fifth either way. + expect(delay).toBeGreaterThanOrEqual(1_600); + expect(delay).toBeLessThanOrEqual(2_400); + } + controller.abort(); + await expect(download).rejects.toMatchObject({ name: "AbortError" }); + expect(fetch).toHaveBeenCalledOnce(); + }, + ); it("retries a failed connection and names it when every try fails", async () => { const request = await input(); @@ -350,6 +381,90 @@ describe("downloads with Scient's own copy first", () => { expect(urls(fetch)).toEqual([mirror, request.fallbackUrl, request.fallbackUrl]); }); + it.each([ + { mirrorFirst: true, fallbackElapsedMs: 0 }, + { mirrorFirst: false, fallbackElapsedMs: 0 }, + { mirrorFirst: true, fallbackElapsedMs: 10_000 }, + ])( + "preserves the mirror cooldown after GitHub fails ($mirrorFirst, $fallbackElapsedMs)", + async ({ mirrorFirst, fallbackElapsedMs }) => { + vi.setSystemTime(0); + using _random = vi.spyOn(Math, "random").mockReturnValue(0.5); + const base = await twoSources(); + const request = mirrorFirst + ? base + : { ...base, url: base.fallbackUrl, fallbackUrl: base.url }; + let fallbackTries = 0; + const fetch = vi.fn(async (url: URL) => { + if (url.hostname === "downloads.scientfactory.com") + return new Response("busy", { status: 429, headers: { "retry-after": "3600" } }); + if (fallbackTries++ === 0) vi.advanceTimersByTime(fallbackElapsedMs); + return new Response("busy", { status: 503 }); + }); + vi.stubGlobal("fetch", fetch); + const failure = await downloadManagedRuntime(request).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ManagedRuntimeFileError); + expect(managedRuntimeFailureRetryable(failure)).toBe(true); + expect(failure).toMatchObject({ retryAfterMs: 3_548_000 - fallbackElapsedMs }); + expect(managedRuntimeRetryAfterMs(new Error("wrapped", { cause: failure }))).toBe( + 3_548_000 - fallbackElapsedMs, + ); + expect(urls(fetch).filter((url) => url === mirror)).toHaveLength(1); + expect(fallbackTries).toBe(5); + expect(fetch).toHaveBeenCalledTimes(6); + expect(String(failure)).toContain("downloads.scientfactory.com returned HTTP 429"); + expect(String(failure)).toContain("github.com returned HTTP 503"); + vi.advanceTimersByTime(3_600_000); + expect(managedRuntimeRetryAfterMs(failure)).toBeUndefined(); + }, + ); + + it.each([false, true])( + "exposes the longest outstanding source cooldown (reverse %s)", + async (reverse) => { + vi.setSystemTime(0); + const base = await twoSources(); + const request = reverse ? { ...base, url: base.fallbackUrl, fallbackUrl: base.url } : base; + const fetch = vi.fn( + async (url: URL) => + new Response("busy", { + status: 429, + headers: { + "retry-after": url.hostname === "downloads.scientfactory.com" ? "7200" : "3600", + }, + }), + ); + vi.stubGlobal("fetch", fetch); + vi.mocked(NodeTimersPromises.setTimeout).mockClear(); + const failure = await downloadManagedRuntime(request).catch((cause: unknown) => cause); + expect(managedRuntimeRetryAfterMs(failure)).toBe(7_200_000); + expect(fetch).toHaveBeenCalledTimes(2); + expect(NodeTimersPromises.setTimeout).not.toHaveBeenCalled(); + }, + ); + + it("revisits a cooled source only once its deadline has elapsed", async () => { + vi.setSystemTime(0); + using _random = vi.spyOn(Math, "random").mockReturnValue(0.5); + const request = await twoSources(); + const fetch = vi + .fn() + .mockResolvedValueOnce(new Response("busy", { status: 429, headers: { "retry-after": "7" } })) + .mockResolvedValueOnce(new Response("busy", { status: 503 })) + .mockResolvedValueOnce(new Response("busy", { status: 503 })) + .mockResolvedValueOnce(ok()); + vi.stubGlobal("fetch", fetch); + vi.mocked(NodeTimersPromises.setTimeout).mockClear(); + await expect(downloadManagedRuntime(request)).resolves.toMatchObject({ + tries: 4, + source: "primary", + }); + expect(urls(fetch)).toEqual([mirror, request.fallbackUrl, request.fallbackUrl, mirror]); + expect(vi.mocked(NodeTimersPromises.setTimeout).mock.calls.map(([delay]) => delay)).toEqual([ + 2_000, 5_000, + ]); + }); + it("fails only when both copies fail, and names both", async () => { const request = await twoSources(); const fetch = vi.fn().mockImplementation(async () => new Response("busy", { status: 504 })); @@ -550,6 +665,25 @@ describe("resumable downloads", () => { // Nine tries, more than the five a failing server gets. expect(fetch).toHaveBeenCalledTimes(9); }); + + it("bounds retries when a server repeatedly replaces the same retained prefix", async () => { + const request = await resumable(); + const content = Buffer.alloc(2 * 1024 * 1024, 7); + const fetch = vi.fn(async () => new Response(content.subarray(0, 1024 * 1024))); + vi.stubGlobal("fetch", fetch); + await expect( + downloadManagedRuntime({ + ...request, + expectedSize: content.length, + checksum: { + algorithm: "sha256", + digest: NodeCrypto.createHash("sha256").update(content).digest("hex"), + }, + }), + ).rejects.toThrow("cut off on all 5 tries"); + expect(fetch).toHaveBeenCalledTimes(5); + expect((await NodeFSP.stat(request.partialPath)).size).toBe(1024 * 1024); + }); }); describe("downloads without an overall time limit", () => { @@ -649,6 +783,20 @@ describe("managed runtime failure classes", () => { expect(managedRuntimeFailureRetryable(failure)).toBe(retryable); }); + it("reads the remaining bounded cooldown through wrappers without hiding a longer inner one", () => { + vi.setSystemTime(0); + const inner = new ManagedRuntimeFileError("busy", { retryAfterMs: 99_999_999 }); + const wrapped = new ManagedRuntimeFileError("outer", { cause: inner, retryAfterMs: 1_000 }); + expect(managedRuntimeRetryAfterMs(new Error("wrapped", { cause: wrapped }))).toBe(14_400_000); + vi.advanceTimersByTime(60_000); + expect(inner.retryAfterMs).toBe(14_340_000); + expect(wrapped.retryAfterMs).toBeUndefined(); + expect(managedRuntimeRetryAfterMs(wrapped)).toBe(14_340_000); + vi.advanceTimersByTime(14_340_000); + expect(managedRuntimeRetryAfterMs(wrapped)).toBeUndefined(); + expect(managedRuntimeRetryAfterMs(new Error("Retry-After: 3600"))).toBeUndefined(); + }); + it("classifies a damaged download, cancellation and local disk errors without reading messages", async () => { const request = await input(); await NodeFSP.writeFile(request.destination, "ko"); diff --git a/packages/scient-provider-runtime/src/runtimeFiles.ts b/packages/scient-provider-runtime/src/runtimeFiles.ts index b35dfd4e57..bfb3a7018a 100644 --- a/packages/scient-provider-runtime/src/runtimeFiles.ts +++ b/packages/scient-provider-runtime/src/runtimeFiles.ts @@ -27,7 +27,9 @@ function isTransientDownloadStatus(status: number): boolean { * minute in all, spread so many installs don't retry at the same moment. */ const RETRY_DELAYS_MS = [2_000, 5_000, 15_000, 30_000] as const; -const MAX_RETRY_AFTER_MS = 60_000; +const MAX_RETRY_AFTER_MS = 4 * 60 * 60 * 1_000; +// Covers all four jittered backoffs; longer server waits belong to durable recovery. +const LOCAL_RETRY_WAIT_BUDGET_MS = 65_000; /** Largest artifact the installer downloads; release discovery rejects larger ones. */ export const MANAGED_RUNTIME_MAX_DOWNLOAD_BYTES = 512 * 1024 * 1024; const MAX_DOWNLOAD_BYTES = MANAGED_RUNTIME_MAX_DOWNLOAD_BYTES; @@ -63,17 +65,30 @@ export class ManagedRuntimeFileError extends Error { readonly failureClass: ManagedRuntimeFailureClass | undefined; /** Whether trying the same change again later may succeed, when the class alone can't say. */ readonly retryable: boolean | undefined; + readonly #retryNotBefore: number | undefined; + /** Remaining server cooldown, bounded to four hours; absent once it has elapsed. */ + get retryAfterMs(): number | undefined { + if (this.#retryNotBefore === undefined) return undefined; + const remaining = Math.min(MAX_RETRY_AFTER_MS, this.#retryNotBefore - Date.now()); + return remaining > 0 ? remaining : undefined; + } constructor( message: string, options?: ErrorOptions & { readonly failureClass?: ManagedRuntimeFailureClass; readonly retryable?: boolean; + readonly retryAfterMs?: number; }, ) { super(message, options); this.name = "ManagedRuntimeFileError"; this.failureClass = options?.failureClass; this.retryable = options?.retryable; + const delay = options?.retryAfterMs; + this.#retryNotBefore = + delay !== undefined && Number.isFinite(delay) && delay > 0 + ? Date.now() + Math.min(delay, MAX_RETRY_AFTER_MS) + : undefined; } } @@ -245,10 +260,22 @@ export function managedRuntimeFailureRetryable(cause: unknown, stage?: string): return RETRYABLE_FAILURE_CLASSES.has(managedRuntimeFailureClass(cause, stage)); } +/** Remaining server cooldown through wrapped failures; contains no transport details. */ +export function managedRuntimeRetryAfterMs(cause: unknown): number | undefined { + let remaining = 0; + for (const error of causeChain(cause)) { + if (error instanceof ManagedRuntimeFileError) + remaining = Math.max(remaining, error.retryAfterMs ?? 0); + } + return remaining > 0 ? remaining : undefined; +} + function retryAfterMs(header: string | null): number | null { if (!header) return null; const seconds = Number(header); - const ms = Number.isFinite(seconds) ? seconds * 1_000 : Date.parse(header) - Date.now(); + const ms = Number.isFinite(seconds) + ? Math.min(seconds, MAX_RETRY_AFTER_MS / 1_000) * 1_000 + : Date.parse(header) - Date.now(); return Number.isFinite(ms) ? Math.min(Math.max(ms, 0), MAX_RETRY_AFTER_MS) : null; } @@ -344,6 +371,8 @@ interface ManagedRuntimeDownloadInput { * before going back to a source that already failed. */ readonly fallbackUrl?: string | undefined; + /** Absolute not-before times by HTTPS origin, shared across compressed and raw copies. */ + readonly sourceCooldowns?: Map | undefined; /** Checked after every try, so a damaged copy counts as that source failing. */ readonly checksum?: ManagedRuntimeChecksum | undefined; readonly destination: string; @@ -421,7 +450,13 @@ export async function downloadManagedRuntime( }; const lastFailure = new Map(); let usable = [...sources]; - let round = [...usable]; + const sourceCooldowns = input.sourceCooldowns ?? new Map(); + const notBefore = (source: string) => sourceCooldowns.get(new URL(source).origin) ?? 0; + const remainingCooldown = () => + Math.max(0, ...sources.map((source) => notBefore(source) - Date.now())); + let waitBudgetMs = LOCAL_RETRY_WAIT_BUDGET_MS; + let retryProgressBytes = 0; + let cause: unknown; let tries = 0; const facts = (source: string, cause?: unknown): ManagedRuntimeDownloadResult => { const networkKind = cause instanceof TransientDownloadFailure ? cause.networkKind : undefined; @@ -439,80 +474,96 @@ export async function downloadManagedRuntime( ...(networkKind ? { networkKind } : {}), }; }; + const exhausted = async (): Promise => { + input.signal.throwIfAborted(); + const kept = + partialPath && (await partialDownloadSize(partialPath)) > 0 + ? " Scient kept what it downloaded; trying again continues from there." + : ""; + const retryAfterMs = remainingCooldown(); + const retryable = + retryAfterMs > 0 || + [...lastFailure.values()].some((failure) => failure instanceof TransientDownloadFailure); + if (sources.length === 1 && cause instanceof TransientDownloadFailure) { + throw new ManagedRuntimeFileError(`${exhaustedMessage(cause, tries)}${kept}`, { + cause, + failureClass: transientFailureClass(cause), + retryable, + retryAfterMs, + }); + } + const details = sources + .flatMap((candidate) => { + const failure = lastFailure.get(candidate); + return failure ? [`${new URL(candidate).hostname} ${describeSourceFailure(failure)}`] : []; + }) + .join(" "); + throw new ManagedRuntimeFileError( + `Scient couldn't download it from ${sources.length > 1 ? "either source" : "the source"} after ${tries} tries. ${details}${kept}`, + { + cause, + failureClass: managedRuntimeFailureClass(cause), + retryable, + retryAfterMs, + }, + ); + }; for (let retry = 0; ;) { - const source = round.shift()!; - tries += 1; - const receivedBefore = transfer.receivedBytes; - let cause: unknown; - try { - const attempt = await downloadOnce(input, source, allowedHosts, partialPath, transfer); - if (expected !== undefined && attempt.digest !== expected) { - await removePartialDownload(partialPath ?? input.destination); - // Bytes kept from an earlier try may be what is wrong, so fetch the whole file once. - if (attempt.resumedBytes === 0) throw checksumMismatch(); - const fresh = await downloadOnce(input, source, allowedHosts, partialPath, transfer); - if (fresh.digest !== expected) { + for (const source of usable) { + input.signal.throwIfAborted(); + if (notBefore(source) > Date.now()) continue; + tries += 1; + try { + const attempt = await downloadOnce(input, source, allowedHosts, partialPath, transfer); + if (expected !== undefined && attempt.digest !== expected) { await removePartialDownload(partialPath ?? input.destination); - throw checksumMismatch(); + // Bytes kept from an earlier try may be what is wrong, so fetch the whole file once. + if (attempt.resumedBytes === 0) throw checksumMismatch(); + const fresh = await downloadOnce(input, source, allowedHosts, partialPath, transfer); + if (fresh.digest !== expected) { + await removePartialDownload(partialPath ?? input.destination); + throw checksumMismatch(); + } } + if (partialPath) { + await NodeFSP.rename(partialPath, input.destination); + await NodeFSP.rm(partialValidatorPath(partialPath), { force: true }).catch( + () => undefined, + ); + } + return facts(source); + } catch (caught) { + cause = caught; } - if (partialPath) { - await NodeFSP.rename(partialPath, input.destination); - await NodeFSP.rm(partialValidatorPath(partialPath), { force: true }).catch(() => undefined); + input.signal.throwIfAborted(); + if (cause instanceof TransientDownloadFailure && cause.retryAfterMs !== null) { + sourceCooldowns.set( + new URL(source).origin, + Math.max(notBefore(source), Date.now() + cause.retryAfterMs), + ); } - return facts(source); - } catch (caught) { - cause = caught; + input.onFailedTry?.(facts(source, cause)); + // With a second copy, a problem with this one only rules this copy out. + const ruledOut = sources.length > 1 && cause instanceof ManagedRuntimeFileError; + if (!(cause instanceof TransientDownloadFailure) && !ruledOut) throw cause; + lastFailure.set(source, cause as Error); + if (ruledOut) usable = usable.filter((candidate) => candidate !== source); } - input.onFailedTry?.(facts(source, cause)); - // With a second copy, a problem with this one only rules this copy out. - // Cancellation still ends everything. - const ruledOut = sources.length > 1 && cause instanceof ManagedRuntimeFileError; - if (!(cause instanceof TransientDownloadFailure) && !ruledOut) throw cause; - lastFailure.set(source, cause as Error); - if (ruledOut) { - usable = usable.filter((candidate) => candidate !== source); - round = round.filter((candidate) => candidate !== source); + // Only a growing retained prefix replenishes retries; a server may ignore Range. + if (partialPath && transfer.downloadedBytes - retryProgressBytes >= RESUME_PROGRESS_BYTES) { + retry = 0; + retryProgressBytes = transfer.downloadedBytes; + waitBudgetMs = LOCAL_RETRY_WAIT_BUDGET_MS; } - if (usable.length > 0 && round.length > 0) continue; - // Steady progress on a slow line is not a failing server; keep going while it lasts. - if (transfer.receivedBytes - receivedBefore >= RESUME_PROGRESS_BYTES) retry = 0; const baseDelay = RETRY_DELAYS_MS[retry]; - if (usable.length === 0 || baseDelay === undefined) { - const kept = - partialPath && (await partialDownloadSize(partialPath)) > 0 - ? " Scient kept what it downloaded; trying again continues from there." - : ""; - if (sources.length === 1) { - throw new ManagedRuntimeFileError( - `${exhaustedMessage(cause as TransientDownloadFailure, tries)}${kept}`, - { - cause, - failureClass: transientFailureClass(cause as TransientDownloadFailure), - retryable: true, - }, - ); - } - const details = sources - .map( - (candidate) => - `${new URL(candidate).hostname} ${describeSourceFailure(lastFailure.get(candidate)!)}`, - ) - .join(" "); - throw new ManagedRuntimeFileError( - `Scient couldn't download it from either source after ${tries} tries. ${details}${kept}`, - { - cause, - failureClass: managedRuntimeFailureClass(cause), - retryable: [...lastFailure.values()].some( - (failure) => failure instanceof TransientDownloadFailure, - ), - }, - ); - } - const delayMs = - (cause instanceof TransientDownloadFailure ? cause.retryAfterMs : null) ?? - Math.round(baseDelay * (0.8 + Math.random() * 0.4)); + if (usable.length === 0 || baseDelay === undefined) return exhausted(); + const delayMs = Math.max( + Math.round(baseDelay * (0.8 + Math.random() * 0.4)), + Math.min(...usable.map((source) => Math.max(0, notBefore(source) - Date.now()))), + ); + // Long server cooldowns belong to durable recovery, not an open install dialog. + if (delayMs > waitBudgetMs) return exhausted(); + waitBudgetMs -= delayMs; retry += 1; input.onRetry?.({ retry, @@ -521,7 +572,6 @@ export async function downloadManagedRuntime( delayMs, }); await NodeTimersPromises.setTimeout(delayMs, undefined, { signal: input.signal }); - round = [...usable]; } } From 15878f859abfc533011474f38aa4bd89f564f502 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Mon, 16 Nov 2026 23:40:54 +0200 Subject: [PATCH 06/12] fix(updates): report check starts and accurate update availability --- .../updateOutcomes/DesktopUpdateFacts.test.ts | 2 +- .../updateOutcomes/DesktopUpdateOutcomes.ts | 18 +++++-- .../src/updates/DesktopUpdates.scient.test.ts | 49 ++++++++++++++++++- .../src/telemetry/DesktopUpdateFacts.test.ts | 34 ------------- docs/internals/product-analytics.md | 11 +++-- packages/scient-analytics/src/contract.ts | 2 +- 6 files changed, 69 insertions(+), 47 deletions(-) delete mode 100644 apps/server/src/telemetry/DesktopUpdateFacts.test.ts diff --git a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts index fe717f8d92..7943f96c13 100644 --- a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts +++ b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts @@ -170,7 +170,7 @@ describe("DesktopUpdateFacts", () => { const journal = yield* inLaunch(home, "true", (facts) => Effect.gen(function* () { for (let index = 0; index < 66; index += 1) { - yield* facts.record("app.update.outcome", { step: "check", outcome: "up-to-date" }); + yield* facts.record("app.update.outcome", { step: "check", outcome: "completed" }); } yield* facts.record("app.update.summary", { reason: "daily", status: "up-to-date" }); return yield* nextOffers(facts, facts.sourceReady(PRIMARY_INSTANCE_ID)); diff --git a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts index cd45a43e6d..f9387abea5 100644 --- a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts +++ b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts @@ -6,6 +6,7 @@ * @module DesktopUpdateOutcomes */ import type { DesktopUpdateState } from "@t3tools/contracts"; +import { compareSemverVersions } from "@t3tools/shared/semver"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; @@ -199,12 +200,21 @@ export const make = Effect.fn("DesktopUpdateOutcomes.make")(function* (options: const now = yield* Clock.currentTimeMillis; const lastCheck = yield* Ref.get(lastCheckRef); const lastSuccessfulCheckAt = yield* Ref.get(lastSuccessfulCheckAtRef); - const offered = state.availableVersion !== null || state.downloadedVersion !== null; + const targetVersion = state.downloadedVersion ?? state.availableVersion; yield* record("app.update.summary", { reason, status: state.status, channel: state.channel, - behind: offered ? "yes" : Option.isSome(lastCheck) && lastCheck.value.ok ? "no" : "unknown", + behind: + Option.isSome(lastCheck) && !lastCheck.value.ok + ? "unknown" + : targetVersion !== null + ? compareSemverVersions(targetVersion, state.currentVersion) > 0 + ? "yes" + : "no" + : Option.isSome(lastCheck) + ? "no" + : "unknown", stageAgeMs: Math.max(0, now - times.statusSince), ...Option.match(lastSuccessfulCheckAt, { onNone: () => ({}), @@ -232,10 +242,10 @@ export const make = Effect.fn("DesktopUpdateOutcomes.make")(function* (options: checkStarted: (reason: string) => begin("check", null).pipe( Effect.provideService(CurrentUpdateTrigger, checkTrigger(reason)), - Effect.asVoid, + Effect.flatMap((attempt) => outcome("check", attempt, "started")), ), checkFinished: (state: DesktopUpdateState) => - finish("check", state.availableVersion === null ? "up-to-date" : "available", { + finish("check", "completed", { targetVersion: state.availableVersion, }), checkFailed: (cause: unknown) => finish("check", "failed", { cause }), diff --git a/apps/desktop/src/updates/DesktopUpdates.scient.test.ts b/apps/desktop/src/updates/DesktopUpdates.scient.test.ts index 6f0ca2580d..307c5b12fd 100644 --- a/apps/desktop/src/updates/DesktopUpdates.scient.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.scient.test.ts @@ -1,15 +1,19 @@ import * as NodeModule from "node:module"; import { assert, describe, it } from "@effect/vitest"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import * as ElectronUpdater from "../electron/ElectronUpdater.ts"; import * as DesktopUpdateFacts from "../scient/updateOutcomes/DesktopUpdateFacts.ts"; +import * as DesktopUpdateOutcomes from "../scient/updateOutcomes/DesktopUpdateOutcomes.ts"; import * as DesktopUpdates from "./DesktopUpdates.ts"; import { flushCallbacks, makeHarness, type UpdatesHarnessOptions } from "./updatesTestHarness.ts"; +import { createInitialDesktopUpdateState } from "./updateMachine.ts"; const requireFromUpdater = NodeModule.createRequire( NodeModule.createRequire(import.meta.url).resolve("electron-updater"), @@ -41,6 +45,8 @@ function recordingHarness(options: UpdatesHarnessOptions = {}) { return { harness, outcomes, + facts, + recorded, layer: Layer.merge(TestClock.layer(), harness.layer.pipe(Layer.provide(facts))), }; } @@ -71,12 +77,51 @@ describe("DesktopUpdates outcomes", () => { yield* updates.check("menu"); yield* flushCallbacks; const checks = outcomes("check"); - assert.lengthOf(checks, 1); - assert.include(checks[0], { outcome: "failed", failureClass: "network", trigger: "user" }); + assert.deepEqual( + checks.map((check) => check.outcome), + ["started", "failed"], + ); + assert.include(checks[1], { + failureClass: "network", + trigger: "user", + attemptRef: checks[0]?.attemptRef, + }); }), ).pipe(Effect.provide(layer)); }); + it.effect("does not label an approved downgrade or an unreachable feed as behind", () => { + const { facts, recorded, layer } = recordingHarness(); + return Effect.gen(function* () { + const outcomes = yield* DesktopUpdateOutcomes.make({ + facts: Option.some(yield* DesktopUpdateFacts.DesktopUpdateFacts), + packageKind: "zip", + sourceKind: "github", + policyOff: false, + }); + const state = { + ...createInitialDesktopUpdateState( + "1.2.4-nightly.20261027.1", + { appArch: "x64", hostArch: "x64", runningUnderArm64Translation: false }, + "latest", + ), + status: "available" as const, + availableVersion: "1.2.3", + }; + yield* outcomes.checkStarted("menu"); + yield* outcomes.checkFinished(state); + yield* outcomes.summaries(state, Stream.empty); + assert.equal(recorded.at(-1)?.properties.behind, "no"); + yield* outcomes.checkStarted("menu"); + yield* outcomes.checkFailed(Object.assign(new Error("offline"), { code: "ENETUNREACH" })); + yield* outcomes.summaries( + { ...state, status: "error", availableVersion: "1.2.5" }, + Stream.empty, + ); + assert.equal(recorded.at(-1)?.properties.behind, "unknown"); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(layer, facts, NodeCrypto.layer))); + }); + it.effect("records one failed install after the handoff, however many paths report it", () => { const refused = Object.assign(new Error("spawn EACCES"), { code: "EACCES" }); const { harness, outcomes, layer } = recordingHarness({ diff --git a/apps/server/src/telemetry/DesktopUpdateFacts.test.ts b/apps/server/src/telemetry/DesktopUpdateFacts.test.ts deleted file mode 100644 index f9813071c2..0000000000 --- a/apps/server/src/telemetry/DesktopUpdateFacts.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { DesktopUpdateFact } from "@t3tools/contracts"; -import { assert, describe, it } from "@effect/vitest"; -import * as Effect from "effect/Effect"; -import * as Option from "effect/Option"; - -import { ingestDesktopUpdateFacts } from "./DesktopUpdateFacts.ts"; -import { disabledAnalyticsService } from "./ScientAnalytics.ts"; - -const fact: DesktopUpdateFact = { - id: "6f1c2a9e-4b7d-4e21-9a3f-0c5d8e7b1a24", - name: "app.update.outcome", - occurredAt: "2026-11-16T10:00:00.000Z", - appVersion: "0.6.29", - buildChannel: "stable", - properties: { step: "install", outcome: "handoff-attempted" }, -}; - -describe("ingestDesktopUpdateFacts", () => { - it.effect("acknowledges facts only once the outbox settled them", () => - Effect.gen(function* () { - const unconfirmed = yield* ingestDesktopUpdateFacts( - Option.some({ ...disabledAnalyticsService, ingestRecorded: () => Effect.succeed(false) }), - [fact], - ); - assert.deepEqual(unconfirmed, []); - - const persisted = yield* ingestDesktopUpdateFacts( - Option.some({ ...disabledAnalyticsService, ingestRecorded: () => Effect.succeed(true) }), - [fact], - ); - assert.deepEqual(persisted, [fact.id]); - }), - ); -}); diff --git a/docs/internals/product-analytics.md b/docs/internals/product-analytics.md index b091ed5e37..97050283a9 100644 --- a/docs/internals/product-analytics.md +++ b/docs/internals/product-analytics.md @@ -760,9 +760,9 @@ The desktop updater records two Essential events, both fully closed: their norma field, so nothing passes through as recorded. Versions are public release numbers or omitted; URLs, hosts, paths, raw errors and the updater's `message` never enter them. -`app.update.outcome` is one outcome per step attempt, with `step` and `outcome`: `check` -(`available`, `up-to-date`, `failed`), `download` (`started`, `completed`, `failed`), `install` -(`handoff-attempted`, `failed`) and `apply` (`completed`, `not-applied`, `superseded`). `trigger` +`app.update.outcome` records a start and one terminal outcome per check/download attempt: +`check` and `download` (`started`, `completed`, `failed`), `install` (`handoff-attempted`, +`failed`) and `apply` (`completed`, `not-applied`, `superseded`). `trigger` is `user` or `background` (startup and four-hourly checks, the background download). `failureClass` (failed only) comes from electron-updater's codes, HTTP statuses and Node/Chromium network errors: `network`, `timeout`, `certificate`, `http_4xx`, `http_5xx`, `feed` (the feed lacks @@ -780,8 +780,9 @@ no timer that restarts servers after the handoff. `app.update.summary` is sent at launch, when the status settles somewhere new (a check in progress is not a change) and daily while running: `reason`, `status`, `channel`, `behind` -(`yes` when a newer offer is known, `no` after a successful check found none, otherwise -`unknown`), `stageAgeBucket`, `lastCheckAgeBucket` (last successful check, or `never`), +(`yes` when a newer offer is known, `no` for an approved older/equal target or a +successful check that found none, and `unknown` after a failed check even if an older +offer remains retained), `stageAgeBucket`, `lastCheckAgeBucket` (last successful check, or `never`), `errorContext` on errors, `progressBucket` and `sinceProgressBucket` while downloading, `readyAgeBucket` when downloaded, `disabledClass` when off and `droppedFactsBucket`. A ready update waiting for days is not a failed install. diff --git a/packages/scient-analytics/src/contract.ts b/packages/scient-analytics/src/contract.ts index 2253e7b794..6ad2de4eb4 100644 --- a/packages/scient-analytics/src/contract.ts +++ b/packages/scient-analytics/src/contract.ts @@ -340,7 +340,7 @@ const HEALTH_OPERATIONS = new Set([ const HEALTH_OUTCOMES = new Set(["started", "completed", "failed", "abnormal"]); // Desktop app updates: each step's outcomes, and why a step failed. const UPDATE_STEP_OUTCOMES: Readonly>> = { - check: ["available", "up-to-date", "failed"], + check: ["started", "completed", "failed"], download: ["started", "completed", "failed"], install: ["handoff-attempted", "failed"], apply: ["completed", "not-applied", "superseded"], From 45401bea351eb6193fea48d02d5aec2197a953ec Mon Sep 17 00:00:00 2001 From: Yaacov Date: Mon, 16 Nov 2026 23:45:46 +0200 Subject: [PATCH 07/12] fix(analytics): never retain desktop update facts collected without consent --- .../updateOutcomes/DesktopUpdateFacts.test.ts | 20 +++++++++++++++---- .../updateOutcomes/DesktopUpdateFacts.ts | 6 +++++- docs/internals/product-analytics.md | 5 +++-- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts index 7943f96c13..6e40308057 100644 --- a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts +++ b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.test.ts @@ -54,6 +54,7 @@ function inLaunch( program: ( facts: DesktopUpdateFacts.DesktopUpdateFacts["Service"], ) => Effect.Effect, + consent?: string, ) { return Effect.scoped( Effect.gen(function* () { @@ -61,7 +62,10 @@ function inLaunch( }), ).pipe( Effect.provide(journalLayer(home)), - Effect.provideService(HostProcess.Environment, { SCIENT_ANALYTICS_ENABLED: analytics }), + Effect.provideService(HostProcess.Environment, { + SCIENT_ANALYTICS_ENABLED: analytics, + ...(consent === undefined ? {} : { SCIENT_ANALYTICS_CONSENT: consent }), + }), ); } @@ -132,7 +136,11 @@ describe("DesktopUpdateFacts", () => { }), ); - it.effect("never keeps or replays facts while analytics is off", () => + it.effect.each([ + { reason: "availability is off", analytics: "false" as const, consent: "diagnostic" }, + { reason: "consent is off", analytics: "true" as const, consent: "off" }, + { reason: "consent is invalid", analytics: "true" as const, consent: "invalid" }, + ])("never keeps or replays facts when $reason", ({ analytics, consent }) => Effect.gen(function* () { const home = temporaryHome(); yield* inLaunch(home, "true", (facts) => @@ -140,8 +148,12 @@ describe("DesktopUpdateFacts", () => { ); assert.isTrue(NodeFS.existsSync(journalFile(home))); - yield* inLaunch(home, "false", (facts) => - facts.record("app.update.summary", { reason: "launch", status: "idle" }), + yield* inLaunch( + home, + analytics, + (facts) => + facts.record("app.update.outcome", { step: "install", outcome: "handoff-attempted" }), + consent, ); assert.isFalse(NodeFS.existsSync(journalFile(home))); diff --git a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.ts b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.ts index 4fca357f3f..1d4879e0c2 100644 --- a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.ts +++ b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.ts @@ -97,7 +97,11 @@ const make = Effect.gen(function* () { Effect.map((found) => found?.document?.limits?.telemetry !== false), Effect.orElseSucceed(() => false), ); - const enabled = metadata.SCIENT_ANALYTICS_ENABLED === "true" && telemetryAllowed; + const consent = env.SCIENT_ANALYTICS_CONSENT ?? "diagnostic"; + const enabled = + metadata.SCIENT_ANALYTICS_ENABLED === "true" && + telemetryAllowed && + ["essential", "product", "diagnostic"].includes(consent); const buildChannel = metadata.SCIENT_ANALYTICS_BUILD_CHANNEL as DesktopUpdateFactValue["buildChannel"]; diff --git a/docs/internals/product-analytics.md b/docs/internals/product-analytics.md index 97050283a9..1797dc124e 100644 --- a/docs/internals/product-analytics.md +++ b/docs/internals/product-analytics.md @@ -791,8 +791,9 @@ Installs run with every backend stopped and an apply outcome is known before any so both events wait in a persisted journal in the desktop process ([`DesktopUpdateFacts.ts`](../../apps/desktop/src/scient/updateOutcomes/DesktopUpdateFacts.ts): at most 64 facts and 14 days; drops are counted into the next summary). It records nothing while -analytics is off for that launch (release metadata, `SCIENT_ANALYTICS_ENABLED`, or a managed -policy without telemetry) and deletes what an earlier launch kept. Facts travel on the telemetry +analytics is off for that launch (release metadata, `SCIENT_ANALYTICS_ENABLED`, an off or invalid +`SCIENT_ANALYTICS_CONSENT`, or a managed policy without telemetry) and deletes what an earlier +launch kept. Facts travel on the telemetry pipe's `desktopUpdateStatus` report to one backend (the primary when ready, else the first ready one; two backends have two installation identities). That backend's `ingestRecorded` queues them with their own event ids and times, through the normal consent and shape checks, and the desktop From 56a59a0fc3014a4e68b1477bfe3e1808d019d35c Mon Sep 17 00:00:00 2001 From: Yaacov Date: Tue, 17 Nov 2026 00:04:03 +0200 Subject: [PATCH 08/12] fix(updates): validate publisher pins at promotion and avoid guessed apply sources --- .github/workflows/release.yml | 41 +-- .../updateOutcomes/DesktopUpdateOutcomes.ts | 3 +- .../src/updates/DesktopUpdates.scient.test.ts | 12 +- docs/internals/product-analytics.md | 3 + docs/operations/beta-releases.md | 17 +- docs/operations/release.md | 20 +- scripts/scient-release-machinery.test.ts | 17 -- scripts/windows-publication-signing.test.ts | 273 ++++++++++++++---- scripts/windows-publication-signing.ts | 95 +++++- 9 files changed, 365 insertions(+), 116 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a970b25e5d..e8f3d53fcd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1207,26 +1207,11 @@ jobs: exit 1 } done - # Clients on either channel can switch to the other, so a publisher pinned by the newest - # Stable or Beta release binds this publication, whatever its channel or exception. - published="$RUNNER_TEMP/published-handoffs" - mkdir -p "$published" - handoffs=() - stable_tag="$(gh api "repos/$SCIENT_DESKTOP_CANONICAL_REPOSITORY/releases/latest" --jq .tag_name)" - gh release download "$stable_tag" --repo "$SCIENT_DESKTOP_CANONICAL_REPOSITORY" \ - --pattern scient-release-handoff.json --output "$published/stable.json" - handoffs+=(--published-handoff "$published/stable.json") - beta_tag="$(gh api "repos/$SCIENT_DESKTOP_BETA_REPOSITORY/releases?per_page=100" \ - --jq '[.[] | select(.draft == false and (.tag_name | contains("-beta.")))][0].tag_name // empty')" - if [[ -n "$beta_tag" ]]; then - gh release download "$beta_tag" --repo "$SCIENT_DESKTOP_BETA_REPOSITORY" \ - --pattern scient-release-handoff.json --output "$published/beta.json" - handoffs+=(--published-handoff "$published/beta.json") - fi node scripts/windows-publication-signing.ts \ --attestation release-assets/signing-win-x64.json \ --allow-unsigned "$ALLOW_UNSIGNED_WINDOWS" \ - "${handoffs[@]}" + --stable-repository "$SCIENT_DESKTOP_CANONICAL_REPOSITORY" \ + --beta-repository "$SCIENT_DESKTOP_BETA_REPOSITORY" - name: Remove internal build attestations shell: bash run: | @@ -1310,6 +1295,10 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 10 environment: ${{ needs.preflight.outputs.channel == 'beta' && 'beta' || 'production' }} + # Both channels share installed Windows publisher pins; hold through public promotion. + concurrency: + group: scient-desktop-stable-beta-publication + cancel-in-progress: false permissions: # SCIENT-FORK:START — write lets the stable publish dispatch the hosted app's deploy. actions: write @@ -1332,6 +1321,12 @@ jobs: run: | set -euo pipefail [[ -n "$BETA_TOKEN" ]] || { echo "SCIENT_BETA_RELEASE_TOKEN is required for the isolated artifact repository." >&2; exit 1; } + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + ref: ${{ needs.preflight.outputs.source_sha }} + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version-file: package.json - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: scient-release-v${{ needs.preflight.outputs.version }} @@ -1393,6 +1388,7 @@ jobs: shell: bash env: GH_TOKEN: ${{ needs.preflight.outputs.channel == 'beta' && secrets.SCIENT_BETA_RELEASE_TOKEN || github.token }} + PIN_READ_TOKEN: ${{ github.token }} DISTRIBUTION_REPOSITORY: ${{ needs.preflight.outputs.repository }} RELEASE_TAG: ${{ needs.preflight.outputs.tag }} RELEASE_VERSION: ${{ needs.preflight.outputs.version }} @@ -1411,7 +1407,7 @@ jobs: release_args=(--draft --latest=false) if [[ "$RELEASE_CHANNEL" == "beta" ]]; then [[ "$DISTRIBUTION_REPOSITORY" == "$SCIENT_DESKTOP_BETA_REPOSITORY" ]] || exit 1 - stable_latest_before="$(gh api "repos/$SCIENT_DESKTOP_CANONICAL_REPOSITORY/releases/latest" --jq .tag_name)" + stable_latest_before="$(GH_TOKEN="$PIN_READ_TOKEN" gh api "repos/$SCIENT_DESKTOP_CANONICAL_REPOSITORY/releases/latest" --jq .tag_name)" jq -ne --arg beta "$RELEASE_VERSION" --arg stable "$stable_latest_before" ' ($beta | split("-")[0] | split(".") | map(tonumber)) > ($stable | ltrimstr("v") | split(".") | map(tonumber)) @@ -1434,10 +1430,17 @@ jobs: (cd published-assets && sha256sum -- * | sort) > "$RUNNER_TEMP/published-release-sha256.txt" diff -u "$RUNNER_TEMP/expected-release-sha256.txt" "$RUNNER_TEMP/published-release-sha256.txt" + # Read both public channels with the source token, not Beta's repository-only write token. + GH_TOKEN="$PIN_READ_TOKEN" node scripts/windows-publication-signing.ts \ + --candidate-handoff release-assets/scient-release-handoff.json \ + --allow-unsigned "$ALLOW_UNSIGNED_WINDOWS" \ + --stable-repository "$SCIENT_DESKTOP_CANONICAL_REPOSITORY" \ + --beta-repository "$SCIENT_DESKTOP_BETA_REPOSITORY" + if [[ "$RELEASE_CHANNEL" == "beta" ]]; then gh release edit "$RELEASE_TAG" --repo "$DISTRIBUTION_REPOSITORY" --draft=false --prerelease --latest=false gh release view "$RELEASE_TAG" --repo "$DISTRIBUTION_REPOSITORY" --json isDraft,isPrerelease | jq -e '.isDraft == false and .isPrerelease == true' >/dev/null - stable_latest_after="$(gh api "repos/$SCIENT_DESKTOP_CANONICAL_REPOSITORY/releases/latest" --jq .tag_name)" + stable_latest_after="$(GH_TOKEN="$PIN_READ_TOKEN" gh api "repos/$SCIENT_DESKTOP_CANONICAL_REPOSITORY/releases/latest" --jq .tag_name)" [[ "$stable_latest_before" == "$stable_latest_after" ]] || { echo "Stable Latest changed during Beta publication; investigate before accepting the release." >&2 exit 1 diff --git a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts index f9387abea5..27cf3ad45a 100644 --- a/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts +++ b/apps/desktop/src/scient/updateOutcomes/DesktopUpdateOutcomes.ts @@ -280,7 +280,8 @@ export const make = Effect.fn("DesktopUpdateOutcomes.make")(function* (options: step: "apply", outcome: result, targetVersion: install.targetVersion, - ...context, + // The prior process's download source is not stored in the install record. + ...(options.packageKind === undefined ? {} : { packageKind: options.packageKind }), ...(install.attemptRef === undefined ? {} : { attemptRef: install.attemptRef }), ...(install.startedAt === undefined ? {} diff --git a/apps/desktop/src/updates/DesktopUpdates.scient.test.ts b/apps/desktop/src/updates/DesktopUpdates.scient.test.ts index 307c5b12fd..75817f7c19 100644 --- a/apps/desktop/src/updates/DesktopUpdates.scient.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.scient.test.ts @@ -168,7 +168,11 @@ describe("DesktopUpdates outcomes", () => { startedAt: 0, attemptRef: "0123456789abcdef", }); - const first = recordingHarness({ installRecord: record }); + const first = recordingHarness({ + installRecord: record, + env: { T3CODE_DESKTOP_MOCK_UPDATES: "false" }, + appUpdateYml: "provider: github\nowner: ScientFactory\nrepo: scient-desktop\n", + }); yield* Effect.scoped( DesktopUpdates.DesktopUpdates.pipe(Effect.flatMap((updates) => updates.configure)), ).pipe(Effect.provide(first.layer)); @@ -185,7 +189,11 @@ describe("DesktopUpdates outcomes", () => { const kept = [...first.harness.installRecords.values()][0]; assert.isDefined(kept); - const second = recordingHarness({ installRecord: kept }); + const second = recordingHarness({ + installRecord: kept, + env: { T3CODE_DESKTOP_MOCK_UPDATES: "false" }, + appUpdateYml: "provider: generic\nurl: https://downloads.scientfactory.com/updates/\n", + }); yield* Effect.scoped( DesktopUpdates.DesktopUpdates.pipe(Effect.flatMap((updates) => updates.configure)), ).pipe(Effect.provide(second.layer)); diff --git a/docs/internals/product-analytics.md b/docs/internals/product-analytics.md index 1797dc124e..a0278c013b 100644 --- a/docs/internals/product-analytics.md +++ b/docs/internals/product-analytics.md @@ -777,6 +777,9 @@ over; the next launch reports `apply` with the same `attemptRef`: `completed` wh runs, `not-applied` when the old version runs again (once, not at every launch), `superseded` when another build runs. A handoff without a later outcome is unresolved, not a failure; there is no timer that restarts servers after the handoff. +An `apply` outcome omits `sourceKind`: the install record does not retain the previous +process's feed. Its `attemptRef` links to the original handoff instead of guessing from +the new process's configured feed. `app.update.summary` is sent at launch, when the status settles somewhere new (a check in progress is not a change) and daily while running: `reason`, `status`, `channel`, `behind` diff --git a/docs/operations/beta-releases.md b/docs/operations/beta-releases.md index 8a32841b6a..ccecbc3a91 100644 --- a/docs/operations/beta-releases.md +++ b/docs/operations/beta-releases.md @@ -46,10 +46,12 @@ and every existing Beta, and the SHA must equal the dispatched commit. Beta always publishes, even when `publish_release` is left at its default `false`; that checkbox controls Stable only. Beta has no What's New catalog requirement and omits release notes. `allow_note_free` also controls Stable only. -The owner-approved unsigned-Windows exception applies automatically to Beta, -with the Windows signing notice retained in its GitHub release body. macOS -signing and notarization remain mandatory. Stable's publication, notes, signing -exception inputs and protected `production` approval remain unchanged. +The owner-approved unsigned-Windows exception applies automatically to Beta +until either channel has shipped a Windows app that pins a publisher. After that, +both channels require that publisher's signature, including candidates assembled +before the pin appeared. The Windows signing notice is retained in the workflow +summary. macOS signing and notarization remain mandatory. Stable's publication, +notes, signing exception inputs and protected `production` approval remain unchanged. Beta is manual rather than scheduled. Release the stable bridge containing the selector before the first Beta. A Beta must target a core ahead of current @@ -87,6 +89,13 @@ checked separately during packaging; the approved Windows exception remains. The run retains an immutable candidate for 30 days. Publication verifies its run, artifact digest, source/tree, distribution identity and all asset hashes. +Stable and Beta publication jobs share concurrency group +`scient-desktop-stable-beta-publication`. Inside that boundary, after environment +protection, each job fetches the newest public handoffs again and checks the +downloaded candidate's Windows signing record after draft-byte verification, just +before making it public. Lookup or signing-record failures stop publication. +Builds keep their separate channel concurrency; Stable still needs `production` +approval. It stages a draft in the isolated Beta repository, downloads and compares every uploaded byte, then automatically makes it public as a prerelease with `latest=false`. It verifies that canonical Stable Latest did not change. diff --git a/docs/operations/release.md b/docs/operations/release.md index 61be74d8ab..c0718fbb99 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -223,6 +223,10 @@ tag or release already exists. summary, not the release body, because the app shows the release body as release notes. Unsigned installers may trigger an Unknown Publisher or SmartScreen warning, which `docs/user/install.md` explains. The workflow + rechecks public Windows publisher pins after approval; a candidate that passed + assembly can still be refused if either channel has since shipped a signed app. + A failed final check leaves a draft; keep it private and build and accept a + signed replacement with a new release identity. The workflow publishes the already accepted assembled artifact, downloads the draft again, verifies every uploaded byte, and only then makes the release public and Latest. Existing tags or releases are never rebuilt, overwritten, or @@ -289,8 +293,20 @@ only if its installer's signer CN matches the `publisherName` in the installed be Valid, its signer CN to equal `AZURE_TRUSTED_SIGNING_PUBLISHER_NAME`, and the packaged `app-update.yml` to pin exactly that name. An unsigned build must pin none. - Each release handoff records `windows: { signed, publisherName }`. -- Before publishing on either channel, the gate reads the newest published Stable - and Beta handoffs. If either pins a publisher, the candidate must be signed by +- Assembly checks the newest published Stable and Beta handoffs. After environment + approval and draft-byte verification, publication reads them again just before + making the release public and checks the Windows record in the exact downloaded, + identity-verified candidate handoff. It never rebuilds the installer + or substitutes a new signing attestation. Failed lookups, missing handoff assets + and invalid signing records stop publication; only a successful Beta inventory + with no published Beta means that channel pins nothing. +- Stable and Beta publication jobs share concurrency group + `scient-desktop-stable-beta-publication`, covering the fresh pin check, draft + staging, uploaded-byte verification and public promotion. Builds retain their + separate channel concurrency, and `production`/`beta` protections still apply. + Scheduled Stable candidates, manual releases and Stable releases promoted from + accepted Beta source all use this same publication job. +- If either channel pins a publisher, the candidate must be signed by that publisher, whatever `allow_unsigned_windows` says: users switch between Stable and Beta, so a pin on one binds both. Handoffs from before this record pin nothing, so today's unsigned releases are unaffected until the first signed diff --git a/scripts/scient-release-machinery.test.ts b/scripts/scient-release-machinery.test.ts index 2ddea191ab..11dde615c1 100644 --- a/scripts/scient-release-machinery.test.ts +++ b/scripts/scient-release-machinery.test.ts @@ -143,23 +143,6 @@ describe("Scient release machinery", () => { } }); - it("keeps unsigned Windows publication an explicit exception", () => { - const workflow = NodeFS.readFileSync( - NodePath.join(import.meta.dirname, "../.github/workflows/release.yml"), - "utf8", - ); - assert.include(workflow, "allow_unsigned_windows:"); - assert.include( - workflow, - "allow_unsigned_windows is only valid for an explicitly publishing run.", - ); - assert.include( - workflow, - "Windows publication is unsigned; rerun only with explicit allow_unsigned_windows=true.", - ); - assert.include(workflow, "Publication requires signed macOS artifacts:"); - }); - // regression: 20117b09c1 fix(release): use Bash for version alignment on all platforms (#44) it("pins Bash for every package-version alignment step", () => { const workflow = NodeFS.readFileSync( diff --git a/scripts/windows-publication-signing.test.ts b/scripts/windows-publication-signing.test.ts index 7616a1cf00..88a63f269e 100644 --- a/scripts/windows-publication-signing.test.ts +++ b/scripts/windows-publication-signing.test.ts @@ -5,7 +5,6 @@ import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import { assert, describe, it } from "@effect/vitest"; -import { parse } from "yaml"; import { createScientReleaseHandoff } from "./create-scient-release-handoff.ts"; import { @@ -58,6 +57,21 @@ describe("Windows publication signing gate", () => { assert.isNull( decideWindowsPublication({ candidate: signed, pinned, allowUnsigned: false }).warning, ); + const conflictingPins = pinnedWindowsPublishers([ + { windows: { signed: true, publisherName: PUBLISHER } }, + { windows: { signed: true, publisherName: "Different Channel Publisher" } }, + ]); + for (const publisherName of conflictingPins) { + assert.throws( + () => + decideWindowsPublication({ + candidate: { signed: true, publisherName }, + pinned: conflictingPins, + allowUnsigned: true, + }), + /would fail their update signature check/, + ); + } }); it("fails closed on attestations and records it cannot read", () => { @@ -122,75 +136,214 @@ describe("Windows publication signing gate", () => { }); }); -const workflow = parse( - NodeFS.readFileSync( - NodePath.join(import.meta.dirname, "../.github/workflows/release.yml"), - "utf8", - ), -) as { jobs: { assemble: { steps: Array<{ name?: string; run?: string }> } } }; -const gateStep = workflow.jobs.assemble.steps.find( - (entry) => entry.name === "Require signed macOS and explicitly approved Windows publication", -)?.run; - -// oxlint-disable-next-line t3code/no-global-process-runtime -- This synchronous platform gate skips the Linux Bash step on native Windows. -describe.skipIf(process.platform === "win32")("Windows publication gate step", () => { - it("reads the newest Stable and Beta handoffs before deciding", () => { - if (!gateStep) throw new Error("Missing the publication signing step."); - const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "scient-windows-gate-")); - try { - const bin = NodePath.join(root, "bin"); - NodeFS.mkdirSync(bin); - NodeFS.mkdirSync(NodePath.join(root, "release-assets")); - NodeFS.symlinkSync(import.meta.dirname, NodePath.join(root, "scripts")); - for (const arch of ["arm64", "x64"]) { - NodeFS.writeFileSync( - NodePath.join(root, `release-assets/signing-mac-${arch}.json`), - JSON.stringify({ platform: "mac", arch, signed: true }), - ); - } - NodeFS.writeFileSync( - NodePath.join(root, "release-assets/signing-win-x64.json"), - JSON.stringify({ platform: "win", arch: "x64", signed: false }), - ); - // Only the newest Beta is signed: its clients can switch to Stable, so Stable is bound too. - NodeFS.writeFileSync( - NodePath.join(bin, "gh"), - `#!/usr/bin/env node +const stableRepository = "ScientFactory/scient-desktop"; +const betaRepository = "ScientFactory/scient-desktop-beta"; +const stableTag = "v0.6.30"; +const betaTag = "v0.6.31-beta.20261116.1"; +const betaRelease = { tag_name: betaTag, draft: false }; + +function publicationFixture() { + const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "scient-windows-gate-")); + const bin = NodePath.join(root, "bin"); + const assets = NodePath.join(root, "release-assets"); + const attestation = NodePath.join(assets, "signing-win-x64.json"); + const candidateHandoff = NodePath.join(assets, "scient-release-handoff.json"); + const statePath = NodePath.join(root, "published.json"); + NodeFS.mkdirSync(bin); + NodeFS.mkdirSync(assets); + NodeFS.writeFileSync(NodePath.join(assets, "Scient-0.6.31-x64.exe"), "fixture installer"); + const setCandidate = ( + windows: { signed: boolean; publisherName?: string }, + channel: "stable" | "beta" = "stable", + ) => { + NodeFS.writeFileSync(attestation, JSON.stringify({ platform: "win", arch: "x64", ...windows })); + createScientReleaseHandoff([ + "--assets-dir", + assets, + "--version", + channel === "beta" ? betaTag.slice(1) : "0.6.31", + "--channel", + channel, + "--repository", + stableRepository, + "--source-sha", + "a".repeat(40), + "--source-tree", + "b".repeat(40), + "--windows-signing", + attestation, + "--output", + candidateHandoff, + ]); + }; + const setPublished = (state: { + betaPages?: unknown; + stableHandoff?: unknown; + betaHandoff?: unknown; + failure?: { key: string; status: number }; + }) => NodeFS.writeFileSync(statePath, JSON.stringify(state)); + NodeFS.writeFileSync( + NodePath.join(bin, "gh"), + `#!/usr/bin/env node const fs = require('node:fs'); const args = process.argv.slice(2); -if (args[0] === 'api' && args[1].endsWith('/scient-desktop/releases/latest')) process.stdout.write('v0.6.30\\n'); -else if (args[0] === 'api' && args[1].startsWith('repos/ScientFactory/scient-desktop-beta/releases')) process.stdout.write(process.env.BETA_TAG + '\\n'); -else if (args[0] === 'release' && args[1] === 'download') { - const handoff = args[2] === 'v0.6.30' ? { version: '0.6.30' } : { windows: { signed: true, publisherName: 'ScientFactory Ltd' } }; - fs.writeFileSync(args[args.indexOf('--output') + 1], JSON.stringify(handoff)); -} else process.exit(2); +const state = JSON.parse(fs.readFileSync(process.env.PUBLISHED_STATE, 'utf8')); +const endpoint = args.find(arg => arg.startsWith('repos/')); +let key; +let result; +if (args[0] === 'api' && endpoint === 'repos/${stableRepository}/releases/latest') { + key = 'stable'; + result = { tag_name: '${stableTag}' }; +} else if (args[0] === 'api' && args.includes('--paginate') && args.includes('--slurp') && endpoint === 'repos/${betaRepository}/releases?per_page=100') { + key = 'beta'; + result = state.betaPages ?? [[]]; +} else if (args[0] === 'release' && args[1] === 'download' && args[args.indexOf('--output') + 1] === '-') { + const repo = args[args.indexOf('--repo') + 1]; + if (repo === '${stableRepository}' && args[2] === '${stableTag}') { + key = 'stable-handoff'; + result = state.stableHandoff ?? { version: '0.6.30' }; + } else if (repo === '${betaRepository}' && args[2] === '${betaTag}') { + key = 'beta-handoff'; + result = state.betaHandoff ?? { windows: { signed: true, publisherName: '${PUBLISHER}' } }; + } +} +if (!key) process.exit(2); +if (state.failure?.key === key) { + process.stderr.write('HTTP ' + state.failure.status + '\\n'); + process.exit(1); +} +process.stdout.write(JSON.stringify(result)); `, - { mode: 0o700 }, - ); - const run = (betaTag: string) => - NodeChildProcess.spawnSync("bash", ["-c", gateStep], { + { mode: 0o700 }, + ); + setCandidate({ signed: false }); + setPublished({}); + return { + setCandidate, + setPublished, + candidateHandoff, + attestation, + run(phase: "assembly" | "publication", allowUnsigned = true) { + return NodeChildProcess.spawnSync( + process.execPath, + [ + NodePath.join(import.meta.dirname, "windows-publication-signing.ts"), + phase === "assembly" ? "--attestation" : "--candidate-handoff", + phase === "assembly" ? attestation : candidateHandoff, + "--allow-unsigned", + String(allowUnsigned), + "--stable-repository", + stableRepository, + "--beta-repository", + betaRepository, + ], + { cwd: root, env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, - RUNNER_TEMP: root, - BETA_TAG: betaTag, - ALLOW_UNSIGNED_WINDOWS: "true", - SCIENT_DESKTOP_CANONICAL_REPOSITORY: "ScientFactory/scient-desktop", - SCIENT_DESKTOP_BETA_REPOSITORY: "ScientFactory/scient-desktop-beta", + GH_TOKEN: "local-fixture", + PUBLISHED_STATE: statePath, }, encoding: "utf8", - }); - - const pinned = run("v0.6.31-beta.20261116.1"); - assert.notStrictEqual(pinned.status, 0); - assert.match(pinned.stderr, /pin publisher 'ScientFactory Ltd'/); - // With no Beta published, the explicit exception still allows today's unsigned build. - const none = run(""); - assert.strictEqual(none.status, 0, none.stderr); - assert.match(none.stdout, /WARNING: Publishing an unsigned Windows installer/); - } finally { + }, + ); + }, + [Symbol.dispose]() { NodeFS.rmSync(root, { recursive: true, force: true }); + }, + }; +} + +// oxlint-disable-next-line t3code/no-global-process-runtime -- The gh fixture uses a POSIX shebang, like the Linux publication job. +describe.skipIf(process.platform === "win32")("Windows publication gate CLI", () => { + it("reads the newest Stable and Beta handoffs before deciding", () => { + using fixture = publicationFixture(); + // A full first page of drafts must not hide the newest published Beta on the next page. + fixture.setPublished({ + betaPages: [ + Array.from({ length: 100 }, () => ({ ...betaRelease, draft: true })), + [betaRelease], + ], + }); + const pinned = fixture.run("assembly"); + assert.notStrictEqual(pinned.status, 0); + assert.match(pinned.stderr, /pin publisher 'ScientFactory Ltd'/); + fixture.setPublished({}); + const none = fixture.run("assembly"); + assert.strictEqual(none.status, 0, none.stderr); + assert.match(none.stdout, /WARNING: Publishing an unsigned Windows installer/); + const unapproved = fixture.run("assembly", false); + assert.notStrictEqual(unapproved.status, 0); + assert.match(unapproved.stderr, /allow_unsigned_windows=true/); + }); + + it("rejects the same unsigned candidate when Beta gains a pin while Stable awaits approval", () => { + using fixture = publicationFixture(); + const candidate = NodeFS.readFileSync(fixture.candidateHandoff, "utf8"); + const assembled = fixture.run("assembly"); + assert.strictEqual(assembled.status, 0, assembled.stderr); + NodeFS.unlinkSync(fixture.attestation); + const beforePin = fixture.run("publication"); + assert.strictEqual(beforePin.status, 0, beforePin.stderr); + + fixture.setPublished({ betaPages: [[betaRelease]] }); + const afterPin = fixture.run("publication"); + assert.notStrictEqual(afterPin.status, 0); + assert.match(afterPin.stderr, /cannot override/); + assert.strictEqual(NodeFS.readFileSync(fixture.candidateHandoff, "utf8"), candidate); + + fixture.setCandidate({ signed: true, publisherName: PUBLISHER }); + const compatible = fixture.run("publication", false); + assert.strictEqual(compatible.status, 0, compatible.stderr); + fixture.setCandidate({ signed: true, publisherName: "Someone Else" }); + const incompatible = fixture.run("publication"); + assert.notStrictEqual(incompatible.status, 0); + assert.match(incompatible.stderr, /signed by 'Someone Else'/); + }); + + it("also enforces a Stable pin when publishing Beta and refuses missing candidate signing", () => { + using fixture = publicationFixture(); + fixture.setCandidate({ signed: false }, "beta"); + fixture.setPublished({ stableHandoff: { windows: signed } }); + const pinned = fixture.run("publication"); + assert.notStrictEqual(pinned.status, 0); + assert.match(pinned.stderr, /pin publisher 'ScientFactory Ltd'/); + + fixture.setPublished({}); + const handoff = JSON.parse(NodeFS.readFileSync(fixture.candidateHandoff, "utf8")); + delete handoff.windows; + NodeFS.writeFileSync(fixture.candidateHandoff, JSON.stringify(handoff)); + const missing = fixture.run("publication"); + assert.notStrictEqual(missing.status, 0); + assert.match(missing.stderr, /candidate handoff must record Windows signing/); + }); + + it("fails closed on public lookup and download errors, including 404 and 5xx", () => { + using fixture = publicationFixture(); + for (const key of ["stable", "beta", "stable-handoff", "beta-handoff"]) { + for (const status of [404, 503]) { + fixture.setPublished({ betaPages: [[betaRelease]], failure: { key, status } }); + const failed = fixture.run("publication"); + assert.notStrictEqual(failed.status, 0, `${key}: HTTP ${status}`); + assert.match(failed.stderr, new RegExp(`HTTP ${status}`)); + } + } + }); + + it("fails closed on malformed public inventories and Windows signing records", () => { + using fixture = publicationFixture(); + for (const state of [ + { betaPages: {} }, + { betaPages: [[{ tag_name: betaTag }]] }, + { stableHandoff: [] }, + { stableHandoff: { windows: null } }, + { betaPages: [[betaRelease]], betaHandoff: { windows: { signed: true } } }, + ]) { + fixture.setPublished(state); + const failed = fixture.run("publication"); + assert.notStrictEqual(failed.status, 0, JSON.stringify(state)); + assert.match(failed.stderr, /Cannot read|Invalid|must name its publisher/); } }); }); diff --git a/scripts/windows-publication-signing.ts b/scripts/windows-publication-signing.ts index 4c88de5410..2fb165ed18 100644 --- a/scripts/windows-publication-signing.ts +++ b/scripts/windows-publication-signing.ts @@ -1,6 +1,7 @@ #!/usr/bin/env node // @effect-diagnostics nodeBuiltinImport:off - Standalone release gate reads build attestations and published handoffs before publication. +import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; import * as NodeURL from "node:url"; @@ -12,7 +13,7 @@ export interface WindowsSigningRecord { } function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null; + return typeof value === "object" && value !== null && !Array.isArray(value); } /** The Windows build's signing attestation, as the release workflow writes it. */ @@ -46,12 +47,8 @@ export function readWindowsSigningAttestation(value: unknown): WindowsSigningRec export function pinnedWindowsPublishers(handoffs: ReadonlyArray): ReadonlyArray { const pinned = new Set(); for (const handoff of handoffs) { - if (!isRecord(handoff) || handoff.windows === undefined) continue; - const record = readWindowsSigningAttestation({ - platform: "win", - arch: "x64", - ...(handoff.windows as object), - }); + const record = readHandoffWindowsSigning(handoff); + if (record === null) continue; if (record.publisherName !== null) pinned.add(record.publisherName); } return [...pinned]; @@ -75,7 +72,7 @@ export function decideWindowsPublication(input: { `Installed Windows clients pin publisher ${names}; an unsigned installer would fail their update signature check on Stable and Beta. allow_unsigned_windows cannot override this.`, ); } - if (!pinned.includes(candidate.publisherName!)) { + if (pinned.some((publisher) => publisher !== candidate.publisherName)) { throw new Error( `Installed Windows clients pin publisher ${names}; this installer is signed by '${candidate.publisherName}' and would fail their update signature check.`, ); @@ -99,13 +96,89 @@ function readJson(path: string): unknown { return JSON.parse(NodeFS.readFileSync(path, "utf8")) as unknown; } +function readHandoffWindowsSigning(handoff: unknown): WindowsSigningRecord | null { + if (!isRecord(handoff)) throw new Error("Invalid release handoff."); + if (handoff.windows === undefined) return null; + if (!isRecord(handoff.windows)) throw new Error("Invalid Windows signing record in handoff."); + return readWindowsSigningAttestation({ platform: "win", arch: "x64", ...handoff.windows }); +} + +function githubJson(args: ReadonlyArray): unknown { + return JSON.parse( + NodeChildProcess.execFileSync("gh", args, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + maxBuffer: 16 * 1024 * 1024, + }), + ) as unknown; +} + +function newestPublishedHandoffs(stableRepository: string, betaRepository: string): unknown[] { + const stable = githubJson(["api", `repos/${stableRepository}/releases/latest`]); + if (!isRecord(stable) || typeof stable.tag_name !== "string" || !stable.tag_name) { + throw new Error("Cannot identify the newest published Stable release."); + } + const pages = githubJson([ + "api", + "--paginate", + "--slurp", + `repos/${betaRepository}/releases?per_page=100`, + ]); + if (!Array.isArray(pages) || !pages.every(Array.isArray)) { + throw new Error("Cannot read published Beta releases."); + } + const releases: unknown[] = pages.flat(); + let betaTag: string | null = null; + for (const release of releases) { + if ( + !isRecord(release) || + typeof release.draft !== "boolean" || + typeof release.tag_name !== "string" + ) { + throw new Error("Invalid published Beta release record."); + } + if (!release.draft && release.tag_name.includes("-beta.")) { + betaTag = release.tag_name; + break; + } + } + const download = (repository: string, tag: string) => + githubJson([ + "release", + "download", + tag, + "--repo", + repository, + "--pattern", + "scient-release-handoff.json", + "--output", + "-", + ]); + return [ + download(stableRepository, stable.tag_name), + ...(betaTag === null ? [] : [download(betaRepository, betaTag)]), + ]; +} + export function runWindowsPublicationGate(args: ReadonlyArray): string | null { const [attestation] = values(args, "--attestation"); - if (!attestation) throw new Error("--attestation is required."); + const [candidateHandoff] = values(args, "--candidate-handoff"); + if (Boolean(attestation) === Boolean(candidateHandoff)) { + throw new Error("Exactly one of --attestation or --candidate-handoff is required."); + } + const candidate = attestation + ? readWindowsSigningAttestation(readJson(attestation)) + : readHandoffWindowsSigning(readJson(candidateHandoff!)); + if (candidate === null) throw new Error("The candidate handoff must record Windows signing."); + const [stableRepository] = values(args, "--stable-repository"); + const [betaRepository] = values(args, "--beta-repository"); + if (!stableRepository || !betaRepository) { + throw new Error("--stable-repository and --beta-repository are required."); + } const allowUnsigned = values(args, "--allow-unsigned")[0] === "true"; const { warning } = decideWindowsPublication({ - candidate: readWindowsSigningAttestation(readJson(attestation)), - pinned: pinnedWindowsPublishers(values(args, "--published-handoff").map(readJson)), + candidate, + pinned: pinnedWindowsPublishers(newestPublishedHandoffs(stableRepository, betaRepository)), allowUnsigned, }); return warning; From d3aef5d1b96dcc1ce5f35231743857bd040744b7 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Tue, 17 Nov 2026 00:14:25 +0200 Subject: [PATCH 09/12] fix(recovery): honor Agent stop policy and acknowledge only durable cancellation --- .../ProviderRuntimeManager.test.ts | 356 +++++++++++++++++- .../ProviderRuntimeManager.ts | 148 ++++++-- .../ProviderRuntimeRecovery.test.ts | 110 +++++- .../ProviderRuntimeRequests.ts | 31 +- .../ScientAgentBackgroundRuntime.test.ts | 52 ++- .../ScientAgentBackgroundRuntime.ts | 186 ++++----- .../ScientAgentRuntimePolicy.ts | 124 ++++++ docs/internals/provider-lifecycle.md | 14 +- 8 files changed, 827 insertions(+), 194 deletions(-) create mode 100644 apps/server/src/scient/providerLifecycle/ScientAgentRuntimePolicy.ts diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts index 9c5b35cf06..6b4d5d84e1 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts @@ -17,6 +17,8 @@ import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; +import * as FileSystem from "effect/FileSystem"; +import * as PlatformError from "effect/PlatformError"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; @@ -52,6 +54,10 @@ import { import { make as makeRequests, ProviderRuntimeRequests } from "./ProviderRuntimeRequests.ts"; import { make as makeRecovery } from "./ProviderRuntimeRecovery.ts"; import * as ServerConfig from "../../config.ts"; +import { ProcessRunner } from "../../processRunner.ts"; +import { AnalyticsService } from "../../telemetry/AnalyticsService.ts"; +import { make as makeAgentBackground } from "./ScientAgentBackgroundRuntime.ts"; +import * as AgentPolicy from "./ScientAgentRuntimePolicy.ts"; const CODEX = ProviderDriverKind.make("codex"); const INSTANCE = ProviderInstanceId.make("codex"); @@ -139,6 +145,8 @@ function makeHarness( readonly isBusy?: ProviderActivityShape["isBusy"]; readonly closeInstance?: ProviderSessionManagerV2Shape["closeInstance"]; readonly requests?: ProviderRuntimeRequests["Service"]; + readonly agentPolicy?: AgentPolicy.ScientAgentRuntimePolicy["Service"]; + readonly observeRegistryChanges?: boolean; } = {}, ) { return Effect.gen(function* () { @@ -209,7 +217,9 @@ function makeHarness( setProviderConnectionOperation: () => SubscriptionRef.get(providersRef), setProviderAuthenticationFailure: () => SubscriptionRef.get(providersRef), setProviderManagedRuntimeSummary: setRuntime, - streamChanges: Stream.empty, + streamChanges: hooks.observeRegistryChanges + ? SubscriptionRef.changes(providersRef) + : Stream.empty, }; const coordinator = yield* makeLifecycleCoordinator; const lifecycleReleaseCountRef = yield* SubscriptionRef.make(0); @@ -240,6 +250,13 @@ function makeHarness( settled: (input) => SubscriptionRef.update(requestLogRef, (log) => [...log, `settled:${input.outcome}`]), }); + const policyRef = yield* Ref.make({}); + const agentPolicy = + hooks.agentPolicy ?? + AgentPolicy.ScientAgentRuntimePolicy.of({ + read: Ref.get(policyRef), + update: (change) => Ref.modify(policyRef, (before) => [before, change(before)]), + }); const providerSessionsLayer = Layer.mock(ProviderSessionManagerV2)({ closeInstance: (instanceId) => Ref.update(stopCountRef, (count) => count + 1).pipe( @@ -259,6 +276,9 @@ function makeHarness( requestsLayer, NodeServices.layer, NodeCrypto.layer, + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-manager-layer-" }).pipe( + Layer.provide(NodeServices.layer), + ), ), ), ), @@ -267,6 +287,7 @@ function makeHarness( Effect.map((services) => Context.get(services, ProviderRuntimeManager)), ) : yield* make().pipe( + Effect.provideService(AgentPolicy.ScientAgentRuntimePolicy, agentPolicy), Effect.provideService(ProviderRegistry, registry), Effect.provideService(ProviderLifecycleCoordinator, trackedCoordinator), Effect.provideService(ProviderActivity, activity), @@ -283,6 +304,7 @@ function makeHarness( return { manager, registry, + agentPolicy, providersRef, reloadCountRef, reloadedInstancesRef, @@ -309,6 +331,48 @@ function installPlan() { }; } +const AGENT = ProviderDriverKind.make("scient"); +const agentRuntime: ProviderRuntimeSummary = { + ...missingRuntime, + qualifiedVersion: "0.1.6", + availableManagedVersion: "0.1.6", + actions: ["install", "update", "repair"], +}; +const agentProvider: ServerProvider = { + ...provider, + driver: AGENT, + connection: { ...provider.connection!, runtime: agentRuntime }, +}; + +const agentTestLayer = (prefix: string) => + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ); + +const startAgentLoop = Effect.fn("test.startAgentLoop")(function* ( + manager: ProviderRuntimeManager["Service"], + registry: ProviderRegistry["Service"], + policy: AgentPolicy.ScientAgentRuntimePolicy["Service"], +) { + yield* makeAgentBackground().pipe( + Effect.provideService(ProviderRuntimeManager, manager), + Effect.provideService(ProviderRegistry, registry), + Effect.provideService(AgentPolicy.ScientAgentRuntimePolicy, policy), + Effect.provide(Layer.mergeAll(Layer.mock(ProcessRunner)({}), Layer.mock(AnalyticsService)({}))), + Effect.provideServiceEffect( + ServerConfig.ServerConfig, + ServerConfig.ServerConfig.pipe( + Effect.map((config) => ({ ...config, mode: "desktop" as const })), + ), + ), + Effect.provideService(HostProcess.Platform, "darwin"), + ); +}); + +const loadAgentPolicy = Layer.build(AgentPolicy.layer).pipe( + Effect.map((services) => Context.get(services, AgentPolicy.ScientAgentRuntimePolicy)), +); + describe("ProviderRuntimeManager", () => { it.effect("publishes one managed install and reloads every instance of the driver", () => Effect.gen(function* () { @@ -1336,6 +1400,296 @@ describe("ProviderRuntimeManager", () => { ), ); + it.effect.each(["install", "update", "repair"] as const)( + "keeps pending Agent %s cancellation across restart and automatic startup", + (action) => + Effect.gen(function* () { + const requests = yield* makeRequests(); + const request = { + requestId: "pending-agent", + instanceId: INSTANCE, + driver: AGENT, + action, + catalogRevision: "reviewed:1", + }; + yield* requests.requested(request); + yield* requests.settled({ + ...request, + outcome: "retryable", + retryAfterMs: 2 * 60 * 60_000, + }); + const restarted = yield* makeRequests(); + const policy = yield* loadAgentPolicy; + const runs = yield* Ref.make(0); + const summary: ProviderRuntimeSummary = { + ...agentRuntime, + source: action === "install" ? "missing" : "scient_managed", + managedVersion: action === "install" ? null : "0.1.5", + }; + const { manager, registry } = yield* makeHarness( + { + getSummary: Effect.succeed(summary), + plan: () => Effect.succeed({ ...installPlan(), action, version: "0.1.6" }), + run: () => Ref.update(runs, (count) => count + 1), + }, + [{ ...agentProvider, connection: { ...agentProvider.connection!, runtime: summary } }], + undefined, + undefined, + undefined, + { requests: restarted, agentPolicy: policy, observeRegistryChanges: true }, + ); + yield* startAgentLoop(manager, registry, policy); + const cancelled = yield* manager.cancel({ + instanceId: INSTANCE, + requestId: request.requestId, + }); + assert.isNull(cancelled.providers[0]?.connection?.runtime?.operation); + assert.isNull(cancelled.providers[0]?.connection?.runtime?.pendingRecovery); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + const persisted = yield* (yield* loadAgentPolicy).read; + assert.deepStrictEqual( + persisted, + action === "update" + ? { skippedVersion: "0.1.6" } + : action === "repair" + ? { userStopped: true, skippedVersion: "0.1.6" } + : { userStopped: true }, + ); + yield* TestClock.adjust("61 seconds"); + assert.strictEqual(yield* Ref.get(runs), 0); + }).pipe(Effect.provide(agentTestLayer("scient-agent-pending-cancel-"))), + ); + + it.effect.each(["reviewed:1", "reviewed:2"])( + "defers automatic Agent starts, including shared instances, until requested recovery revalidates %s", + (reviewedPlan) => + Effect.gen(function* () { + const requests = yield* makeRequests(); + const request = { + requestId: "cooling-agent", + instanceId: INSTANCE, + driver: AGENT, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + yield* requests.requested(request); + yield* requests.settled({ + ...request, + outcome: "retryable", + retryAfterMs: 2 * 60 * 60_000, + }); + const runs = yield* Ref.make(0); + const revision = yield* Ref.make("reviewed:1"); + const policy = yield* loadAgentPolicy; + const { manager, registry, providersRef, lifecycleReleaseCountRef } = yield* makeHarness( + { + getSummary: Effect.succeed(agentRuntime), + plan: () => + Ref.get(revision).pipe( + Effect.map((catalogRevision) => ({ + ...installPlan(), + version: "0.1.6", + catalogRevision, + })), + ), + run: () => Ref.update(runs, (count) => count + 1), + }, + [agentProvider, { ...agentProvider, instanceId: SECOND_INSTANCE }], + undefined, + undefined, + undefined, + { requests, agentPolicy: policy, observeRegistryChanges: true }, + ); + yield* startAgentLoop(manager, registry, policy); + yield* makeRecovery().pipe( + Effect.provideService(ProviderRuntimeRequests, requests), + Effect.provideService(ProviderRuntimeManager, manager), + Effect.provideService(ProviderRegistry, registry), + ); + for (const instanceId of [INSTANCE, SECOND_INSTANCE]) { + const error = yield* manager + .startInBackground({ + instanceId, + action: "install", + catalogRevision: "reviewed:1", + attempt: 1, + }) + .pipe(Effect.flip); + assert.strictEqual(error.reason, "runtime_busy"); + } + yield* TestClock.adjust("61 seconds"); + assert.strictEqual(yield* Ref.get(runs), 0); + yield* Ref.set(revision, reviewedPlan); + yield* TestClock.adjust(2 * 60 * 60_000 - 61_000); + if (reviewedPlan === "reviewed:1") { + yield* awaitState(lifecycleReleaseCountRef, (count) => count === 1); + assert.strictEqual(yield* Ref.get(runs), 1); + const operation = (yield* registry.getProviders)[0]?.connection?.runtime?.operation; + assert.strictEqual(operation?.status, "succeeded"); + assert.isTrue(operation?.background); + } else { + yield* awaitState( + providersRef, + (providers) => providers[0]?.connection?.runtime?.pendingRecovery === null, + ); + assert.strictEqual(yield* Ref.get(runs), 0); + } + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + }).pipe(Effect.provide(agentTestLayer("scient-agent-cooldown-"))), + ); + + it.effect.each([ + { phase: "pending", file: "requests.json" }, + { phase: "active", file: "requests.json" }, + { phase: "pending", file: "scient-agent.json" }, + { phase: "active", file: "scient-agent.json" }, + ] as const)( + "does not acknowledge $phase Agent cancellation when $file cannot be written", + ({ phase, file }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const denied = yield* Ref.make(false); + const guardedFs = FileSystem.FileSystem.of({ + ...fs, + rename: (from, to) => + Ref.get(denied).pipe( + Effect.flatMap((fail) => + fail && to.endsWith(file) + ? Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "rename", + pathOrDescriptor: to, + }), + ) + : fs.rename(from, to), + ), + ), + }); + const requests = yield* makeRequests().pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + const request = { + requestId: "cancel-denied", + instanceId: INSTANCE, + driver: AGENT, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + yield* requests.requested(request); + const policy = yield* loadAgentPolicy.pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + const running = yield* Deferred.make(); + const { manager, coordinator } = yield* makeHarness( + { + getSummary: Effect.succeed(agentRuntime), + plan: () => Effect.succeed({ ...installPlan(), version: "0.1.6" }), + run: () => Deferred.succeed(running, undefined).pipe(Effect.andThen(Effect.never)), + }, + [agentProvider], + undefined, + undefined, + undefined, + { requests, agentPolicy: policy }, + ); + const background = + phase === "active" + ? yield* manager.startInBackground({ ...request, attempt: 1 }).pipe(Effect.forkChild) + : undefined; + if (background) yield* Deferred.await(running); + const before = yield* requests.list; + yield* Ref.set(denied, true); + const error = yield* manager + .cancel({ instanceId: INSTANCE, requestId: request.requestId }) + .pipe(Effect.flip); + assert.strictEqual(error.reason, "runtime_operation_failed"); + assert.deepStrictEqual(yield* requests.list, before); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, before); + if (background) assert.isDefined(yield* coordinator.current(INSTANCE)); + yield* Ref.set(denied, false); + yield* manager.cancel({ instanceId: INSTANCE, requestId: request.requestId }); + if (background) assert.strictEqual(yield* Fiber.join(background), "cancelled"); + assert.strictEqual(yield* coordinator.current(INSTANCE), undefined); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + assert.deepStrictEqual(yield* (yield* loadAgentPolicy).read, { userStopped: true }); + }).pipe(Effect.provide(agentTestLayer("scient-agent-cancel-denied-"))), + ); + + it.effect( + "settles background ownership after a failed persistence write and recovers on a later pass", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const denied = yield* Ref.make(false); + const guardedFs = FileSystem.FileSystem.of({ + ...fs, + rename: (from, to) => + Ref.get(denied).pipe( + Effect.flatMap((fail) => + fail && to.endsWith("requests.json") + ? Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "rename", + pathOrDescriptor: to, + }), + ) + : fs.rename(from, to), + ), + ), + }); + const requests = yield* makeRequests().pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + const request = { + requestId: "settlement-denied", + instanceId: INSTANCE, + driver: CODEX, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + yield* requests.requested(request); + const runs = yield* Ref.make(0); + const { manager, registry, coordinator, lifecycleReleaseCountRef } = yield* makeHarness( + { + getSummary: Effect.succeed(missingRuntime), + plan: () => Effect.succeed(installPlan()), + run: () => + Ref.update(runs, (count) => count + 1).pipe(Effect.andThen(Ref.set(denied, true))), + }, + [provider], + undefined, + undefined, + undefined, + { requests }, + ); + assert.strictEqual( + yield* manager.startInBackground({ ...request, attempt: 1 }), + "succeeded", + ); + yield* awaitState(lifecycleReleaseCountRef, (count) => count === 1); + assert.strictEqual(yield* coordinator.current(INSTANCE), undefined); + assert.lengthOf(yield* requests.list, 1); + assert.lengthOf(yield* (yield* makeRequests()).list, 1); + yield* makeRecovery().pipe( + Effect.provideService(ProviderRuntimeRequests, requests), + Effect.provideService(ProviderRuntimeManager, manager), + Effect.provideService(ProviderRegistry, registry), + ); + // A failed lease write must not terminate recovery or start an unleased operation. + yield* TestClock.adjust("7 minutes"); + assert.strictEqual(yield* Ref.get(runs), 1); + yield* Ref.set(denied, false); + yield* TestClock.adjust("1 minute"); + yield* awaitState(lifecycleReleaseCountRef, (count) => count === 2); + assert.strictEqual(yield* Ref.get(runs), 2); + assert.strictEqual(yield* coordinator.current(INSTANCE), undefined); + }).pipe(Effect.provide(agentTestLayer("scient-settlement-denied-"))), + ); + it.effect("cancels a requested foreground startup before its operation is published", () => Effect.gen(function* () { const requests = yield* makeRequests(); diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts index 10a11b9615..b843a3955c 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts @@ -42,7 +42,13 @@ import { ProviderSessionManagerV2 } from "../../orchestration-v2/ProviderSession import { ProviderActivity } from "./ProviderActivity.ts"; import { ProviderConnectionActionError } from "./ProviderConnectionActions.ts"; import { ProviderLifecycleCoordinator } from "./ProviderLifecycleCoordinator.ts"; -import { ProviderRuntimeRequests, type RuntimeRequest } from "./ProviderRuntimeRequests.ts"; +import { + ProviderRuntimeRequests, + sameRuntime, + type RuntimeRequest, +} from "./ProviderRuntimeRequests.ts"; +import * as ScientAgentRuntimePolicy from "./ScientAgentRuntimePolicy.ts"; +import { recordAfterUserChange } from "./ScientAgentRuntimePolicy.ts"; export interface ProviderRuntimeManagerShape { readonly plan: ( @@ -203,6 +209,7 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { const lifecycleCoordinator = yield* ProviderLifecycleCoordinator; const activity = yield* ProviderActivity; const requests = yield* ProviderRuntimeRequests; + const agentPolicy = yield* ScientAgentRuntimePolicy.ScientAgentRuntimePolicy; const random = yield* SecureRandom.SecureRandom; const activeRef = yield* Ref.make>( new Map(), @@ -295,6 +302,35 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { return { actions, snapshot, provider }; }); + const persistCancellation = Effect.fn("ProviderRuntimeManager.persistCancellation")(function* ( + target: RuntimeTarget, + action: ProviderManagedRuntimeAction, + requestId: string | undefined, + ) { + if (target.provider === ProviderDriverKind.make("scient")) { + const runtime = target.snapshot.connection?.runtime; + const change = recordAfterUserChange( + { action, status: "cancelled" }, + runtime?.availableManagedVersion ?? runtime?.qualifiedVersion ?? null, + ); + if (change) yield* agentPolicy.update(change); + } + yield* requests.settled({ + instanceId: target.snapshot.instanceId, + driver: target.provider, + outcome: "cancelled", + ...(requestId ? { requestId } : {}), + }); + }); + + const persistenceActionError = (target: RuntimeTarget, message: string) => + makeError({ + provider: target.provider, + instanceId: target.snapshot.instanceId, + reason: "runtime_operation_failed", + message, + }); + const takeIfCurrent = (instanceId: ProviderInstanceId, operationId: string) => Ref.modify(activeRef, (active) => { const current = active.get(instanceId); @@ -567,6 +603,20 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { yield* Deferred.succeed(outcome, "cancelled"); return yield* Effect.interrupt; } + if ( + background && + !background.requestId && + (yield* requests.list).some((request) => + sameRuntime(request, { instanceId: input.instanceId, driver: target.provider }), + ) + ) { + return yield* makeError({ + provider: target.provider, + instanceId: input.instanceId, + reason: "runtime_busy", + message: "A requested runtime recovery owns this runtime.", + }); + } const reserved = yield* lifecycleCoordinator.reserve({ instanceId: input.instanceId, provider: target.provider, @@ -581,13 +631,15 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { }); } if (background === undefined) { - yield* requests.requested({ - requestId: operationId, - instanceId: input.instanceId, - driver: target.provider, - action: input.action, - catalogRevision: input.catalogRevision, - }); + yield* requests + .requested({ + requestId: operationId, + instanceId: input.instanceId, + driver: target.provider, + action: input.action, + catalogRevision: input.catalogRevision, + }) + .pipe(Effect.mapError((error) => persistenceActionError(target, error.message))); } const startedAt = yield* nowIso; const baseSummary = yield* target.actions.getSummary.pipe( @@ -629,7 +681,9 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { requested.action !== input.action || requested.catalogRevision !== input.catalogRevision ) { - yield* requests.update(background.requestId, () => undefined); + yield* requests + .update(background.requestId, () => undefined) + .pipe(Effect.mapError((error) => persistenceActionError(target, error.message))); yield* Deferred.succeed(outcome, "cancelled"); return yield* Effect.interrupt; } @@ -852,18 +906,27 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { }).pipe( Effect.catchCause(() => publishUnexpectedFailure), Effect.andThen( - requests.settled({ - instanceId: input.instanceId, - driver: target.provider, - ...(active.requestId ? { requestId: active.requestId } : {}), - ...(retryAfterMs === undefined ? {} : { retryAfterMs }), - outcome: - completion._tag === "Success" - ? "succeeded" - : managedRuntimeFailureRetryable(completion.failure, lastStage) - ? "retryable" - : "failed", - }), + requests + .settled({ + instanceId: input.instanceId, + driver: target.provider, + ...(active.requestId ? { requestId: active.requestId } : {}), + ...(retryAfterMs === undefined ? {} : { retryAfterMs }), + outcome: + completion._tag === "Success" + ? "succeeded" + : managedRuntimeFailureRetryable(completion.failure, lastStage) + ? "retryable" + : "failed", + }) + .pipe( + Effect.catchTags({ + ProviderRuntimeRequestsPersistenceError: () => + Effect.logWarning( + "Runtime request settlement could not be persisted; retaining recovery intent", + ), + }), + ), ), Effect.andThen( Deferred.succeed(outcome, completion._tag === "Success" ? "succeeded" : "failed"), @@ -1057,18 +1120,24 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { ? candidate.operationId === input.operationId : pending !== undefined && candidate.requestId === input.requestId); if (!matchesActive && pending) { - yield* requests.update(pending.requestId, () => undefined); + yield* persistCancellation(target, pending.action, pending.requestId).pipe( + Effect.mapError((error) => persistenceActionError(target, error.message)), + ); for (const [operationId, starting] of yield* Ref.get(startupsRef)) { if ( operationId === pending.requestId || starting.background?.requestId === pending.requestId ) { + if ((yield* Ref.get(activeRef)).get(starting.instanceId)?.operationId === operationId) + continue; yield* interruptStartup(operationId, "cancelled"); } } candidate = (yield* Ref.get(activeRef)).get(input.instanceId); if (candidate?.requestId !== pending.requestId) { - yield* syncPendingRecovery(); + yield* syncPendingRecovery().pipe( + Effect.mapError((error) => persistenceActionError(target, error.message)), + ); return { providers: yield* providerRegistry.getProviders }; } matchesActive = true; @@ -1087,6 +1156,10 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { const current = (yield* Ref.get(activeRef)).get(input.instanceId); if (current?.operationId !== operationId) return undefined; if (yield* Ref.get(current.committedRef)) return "committed" as const; + yield* persistCancellation(target, current.action, current.requestId).pipe( + Effect.mapError((error) => persistenceActionError(target, error.message)), + ); + if (yield* Ref.get(current.committedRef)) return "committed" as const; return yield* takeIfCurrent(input.instanceId, operationId); }), ); @@ -1106,12 +1179,6 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { message: "The provider runtime operation is no longer active.", }); } - yield* requests.settled({ - instanceId: input.instanceId, - driver: target.provider, - outcome: "cancelled", - ...(claim.requestId ? { requestId: claim.requestId } : {}), - }); yield* interruptStartup(claim.operationId, "cancelled"); const providers = yield* settleActive( claim, @@ -1121,6 +1188,7 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { ); return { providers }; }, + Effect.uninterruptible, ); // Instances of one driver share its runtime, so one reselection per driver @@ -1213,15 +1281,31 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { yield* syncPendingRecovery(); yield* requests.changes.pipe( - Stream.runForEach(() => syncPendingRecovery()), + Stream.runForEach(() => + syncPendingRecovery().pipe( + Effect.catchTags({ + ProviderRuntimeRequestsPersistenceError: () => + Effect.logWarning("Runtime recovery intent could not be pruned"), + }), + ), + ), Effect.forkScoped, ); yield* providerRegistry.streamChanges.pipe( - Stream.runForEach(() => syncPendingRecovery()), + Stream.runForEach(() => + syncPendingRecovery().pipe( + Effect.catchTags({ + ProviderRuntimeRequestsPersistenceError: () => + Effect.logWarning("Runtime recovery intent could not be pruned"), + }), + ), + ), Effect.forkScoped, ); return ProviderRuntimeManager.of({ plan, start, cancel, reselect, startInBackground }); }); -export const layer = Layer.effect(ProviderRuntimeManager, make()); +export const layer = Layer.effect(ProviderRuntimeManager, make()).pipe( + Layer.provide(ScientAgentRuntimePolicy.layer), +); diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts index 22078d31f8..1684ad7ace 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts @@ -14,7 +14,9 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; +import * as PlatformError from "effect/PlatformError"; import * as Stream from "effect/Stream"; +import * as SubscriptionRef from "effect/SubscriptionRef"; import * as TestClock from "effect/testing/TestClock"; import * as ServerConfig from "../../config.ts"; @@ -93,6 +95,55 @@ describe("ProviderRuntimeRequests", () => { ), ); + it.effect( + "retains the visible and persisted request when cancellation cannot be written, then retries", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const denied = yield* Ref.make(false); + const guardedFs = FileSystem.FileSystem.of({ + ...fs, + rename: (from, to) => + Ref.get(denied).pipe( + Effect.flatMap((fail) => + fail + ? Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "rename", + pathOrDescriptor: to, + }), + ) + : fs.rename(from, to), + ), + ), + }); + const requests = yield* makeRequests().pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + yield* requests.requested(request); + const before = yield* requests.list; + yield* Ref.set(denied, true); + const failure = yield* requests + .settled({ ...request, outcome: "cancelled" }) + .pipe(Effect.flip); + assert.strictEqual(failure._tag, "ProviderRuntimeRequestsPersistenceError"); + assert.deepStrictEqual(yield* requests.list, before); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, before); + yield* Ref.set(denied, false); + yield* requests.settled({ ...request, outcome: "cancelled" }); + assert.deepStrictEqual(yield* requests.list, []); + assert.deepStrictEqual(yield* (yield* makeRequests()).list, []); + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-requests-denied-" }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), + ); + it.effect.each([1_000, 2 * 60 * MINUTE])( "persists the later of retry jitter and the source cooldown (%s ms)", (retryAfterMs) => @@ -121,11 +172,13 @@ describe("ProviderRuntimeRequests", () => { }); describe("ProviderRuntimeRecovery", () => { - // Requests stay in memory here, so the loop's only waits are the clock and these calls. + // The clock owns retry deadlines; starts record completed attempts. const harness = (catalogRevision: string) => Effect.gen(function* () { - const starts = yield* Ref.make>([]); - const requests = yield* makeRequests().pipe(Effect.provide(FileSystem.layerNoop({}))); + const starts = yield* SubscriptionRef.make>( + [], + ); + const requests = yield* makeRequests(); yield* requests.requested(request); // The user's own try failed in a way a later try can fix. yield* requests.settled({ instanceId: INSTANCE, driver: CODEX, outcome: "retryable" }); @@ -143,12 +196,17 @@ describe("ProviderRuntimeRecovery", () => { startInBackground: (input) => Effect.gen(function* () { const at = yield* Clock.currentTimeMillis; - yield* Ref.update(starts, (all) => [...all, { attempt: input.attempt, at }]); - yield* requests.settled({ - instanceId: input.instanceId, - driver: CODEX, - outcome: "retryable", - }); + yield* requests + .settled({ + instanceId: input.instanceId, + driver: CODEX, + outcome: "retryable", + }) + .pipe(Effect.orDie); + yield* SubscriptionRef.update(starts, (all) => [ + ...all, + { attempt: input.attempt, at }, + ]); return "failed" as ProviderRuntimeOutcome; }), }), @@ -156,9 +214,33 @@ describe("ProviderRuntimeRecovery", () => { ), ); const runFor = (minutes: number) => - Effect.forEach(Array.from({ length: minutes }), () => TestClock.adjust("1 minute"), { - discard: true, - }); + Effect.forEach( + Array.from({ length: minutes }), + () => + Effect.gen(function* () { + const nextMinute = (yield* Clock.currentTimeMillis) + MINUTE; + const due = (yield* requests.list).some((pending) => pending.dueAt <= nextMinute); + const completed = (yield* SubscriptionRef.get(starts)).length; + yield* TestClock.adjust("1 minute"); + if (!due) return; + // Real durable writes are external to TestClock. Await the completed transition + // before advancing again, rather than racing fsync with the simulated clock. + if (catalogRevision === request.catalogRevision) { + yield* SubscriptionRef.changes(starts).pipe( + Stream.filter((entries) => entries.length > completed), + Stream.runHead, + ); + } else { + yield* requests.changes.pipe( + Stream.filter( + (pending) => !pending.some((entry) => entry.requestId === request.requestId), + ), + Stream.runHead, + ); + } + }), + { discard: true }, + ); return { starts, requests, runFor }; }).pipe( Effect.provide( @@ -172,7 +254,7 @@ describe("ProviderRuntimeRecovery", () => { Effect.gen(function* () { const { starts, runFor } = yield* harness("reviewed:1"); yield* runFor(4 * 60); - const tries = yield* Ref.get(starts); + const tries = yield* SubscriptionRef.get(starts); assert.deepStrictEqual( tries.map((entry) => entry.attempt), [1, 2, 3], @@ -189,7 +271,7 @@ describe("ProviderRuntimeRecovery", () => { Effect.gen(function* () { const { starts, requests, runFor } = yield* harness("reviewed:2"); yield* runFor(10); - assert.deepStrictEqual(yield* Ref.get(starts), []); + assert.deepStrictEqual(yield* SubscriptionRef.get(starts), []); assert.deepStrictEqual(yield* requests.list, []); }), ); diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts index f6fe003cdc..9284eb4996 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts @@ -60,13 +60,22 @@ export const nextTryAt = Effect.fnUntraced(function* (retries: number) { }); // Instances of a driver share one managed runtime; each ACP Registry agent has its own. -const sameRuntime = ( +export const sameRuntime = ( request: RuntimeRequest, change: { readonly instanceId: ProviderInstanceId; readonly driver: ProviderDriverKind }, ) => request.driver === change.driver && (change.driver !== ACP_REGISTRY || request.instanceId === change.instanceId); +export class ProviderRuntimeRequestsPersistenceError extends Schema.TaggedError()( + "ProviderRuntimeRequestsPersistenceError", + { path: Schema.String, cause: Schema.Defect() }, +) { + override get message(): string { + return "Scient could not save the requested runtime change. Try again."; + } +} + /** * Runtime changes the user asked for, kept across restarts until one succeeds, the user cancels or * removes it, or chooses again. The runtime manager records them; recovery tries them again. @@ -81,21 +90,21 @@ export class ProviderRuntimeRequests extends Context.Service< readonly driver: ProviderDriverKind; readonly action: ProviderManagedRuntimeAction; readonly catalogRevision: string; - }) => Effect.Effect; + }) => Effect.Effect; readonly settled: (input: { readonly instanceId: ProviderInstanceId; readonly driver: ProviderDriverKind; readonly outcome: RuntimeChangeOutcome; readonly requestId?: string; readonly retryAfterMs?: number; - }) => Effect.Effect; + }) => Effect.Effect; readonly list: Effect.Effect>; readonly changes: Stream.Stream>; /** Changes this exact request if it still exists; `undefined` drops it. */ readonly update: ( requestId: string, change: (request: RuntimeRequest) => RuntimeRequest | undefined, - ) => Effect.Effect; + ) => Effect.Effect; } >()("t3/scient/providerLifecycle/ProviderRuntimeRequests") {} @@ -108,8 +117,14 @@ export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { // Kept outside the runtime folders, which Remove deletes. const filePath = path.join(config.baseDir, "provider-runtimes", ".background", "requests.json"); const stored = yield* fs.readFileString(filePath).pipe( + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed("[]"), + ), Effect.flatMap(decodeRequests), - Effect.orElseSucceed(() => []), + Effect.mapError( + (cause) => new ProviderRuntimeRequestsPersistenceError({ path: filePath, cause }), + ), ); const requestsRef = yield* SubscriptionRef.make>( yield* Effect.forEach(stored, (request) => @@ -131,12 +146,14 @@ export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { if (updated === current) return; yield* encodeRequests(updated).pipe( Effect.flatMap((contents) => - writeFileStringAtomically({ filePath, contents, mode: 0o600 }), + writeFileStringAtomically({ filePath, contents, mode: 0o600, durable: true }), ), Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), Effect.provideService(HostProcess.Platform, platform), - Effect.ignoreCause({ log: true }), + Effect.mapError( + (cause) => new ProviderRuntimeRequestsPersistenceError({ path: filePath, cause }), + ), ); yield* SubscriptionRef.set(requestsRef, updated); }), diff --git a/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.test.ts b/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.test.ts index 7a9028322a..de556a5918 100644 --- a/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.test.ts +++ b/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.test.ts @@ -10,9 +10,7 @@ import { } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; -import * as PlatformError from "effect/PlatformError"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; @@ -28,9 +26,10 @@ import { firstPickState, make, parseMeteredOutput, - recordAfterUserChange, } from "./ScientAgentBackgroundRuntime.ts"; +import { recordAfterUserChange, ScientAgentRuntimePolicy } from "./ScientAgentRuntimePolicy.ts"; + const INSTANCE = ProviderInstanceId.make("scient"); function scientAgent(runtime: Partial = {}): ServerProvider { @@ -108,6 +107,19 @@ describe("backgroundRuntimeAction", () => { backgroundRuntimeAction(scientAgent(), { failures: { version: "0.1.5", count: 5 } }).kind, "start", ); + assert.deepStrictEqual( + backgroundRuntimeAction( + scientAgent({ + pendingRecovery: { + requestId: "reviewed", + action: "install", + nextRetryAt: "2026-11-16T02:00:00.000Z", + }, + }), + {}, + ), + { kind: "wait", reason: "requested-recovery" }, + ); }); }); @@ -130,6 +142,12 @@ describe("recordAfterUserChange", () => { skippedVersion: "0.1.6", }), ).toEqual({}); + expect(recordAfterUserChange({ action: "repair", status: "cancelled" }, "0.1.6")?.({})).toEqual( + { + userStopped: true, + skippedVersion: "0.1.6", + }, + ); expect(recordAfterUserChange({ action: "repair", status: "failed" }, "0.1.6")).toBeUndefined(); }); }); @@ -226,23 +244,19 @@ describe("ScientAgentBackgroundRuntime", () => { Layer.mock(AnalyticsService)({ record: (event) => Ref.update(events, (all) => [...all, event]), }), - FileSystem.layerNoop({ - makeDirectory: () => Effect.void, - readFileString: (path) => - SubscriptionRef.get(record).pipe( - Effect.filterOrFail( - (text): text is string => text !== null, - () => - PlatformError.systemError({ - _tag: "NotFound", - module: "FileSystem", - method: "readFileString", - pathOrDescriptor: path, - }), - ), + Layer.succeed( + ScientAgentRuntimePolicy, + ScientAgentRuntimePolicy.of({ + read: SubscriptionRef.get(record).pipe( + Effect.map((text) => JSON.parse(text ?? "{}")), ), - writeFileString: (_path, text) => SubscriptionRef.set(record, text), - }), + update: (change) => + SubscriptionRef.modify(record, (text) => { + const before = JSON.parse(text ?? "{}"); + return [before, JSON.stringify(change(before))]; + }), + }), + ), ), ), Effect.provideServiceEffect( diff --git a/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.ts b/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.ts index 04367673a0..b68974884e 100644 --- a/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.ts +++ b/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.ts @@ -5,19 +5,14 @@ import { ProviderDriverKind, type ProviderInstanceId, type ProviderManagedRuntimeAction, - type ProviderRuntimeSummary, type ServerProvider, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as Path from "effect/Path"; import * as Ref from "effect/Ref"; -import * as Schema from "effect/Schema"; -import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import { ServerConfig } from "../../config.ts"; @@ -25,6 +20,8 @@ import * as ProcessRunner from "../../processRunner.ts"; import { ProviderRegistry } from "../../provider/ProviderRegistry.ts"; import { AnalyticsService } from "../../telemetry/AnalyticsService.ts"; import { ProviderRuntimeManager } from "./ProviderRuntimeManager.ts"; +import * as ScientAgentRuntimePolicy from "./ScientAgentRuntimePolicy.ts"; +import { recordAfterUserChange, type BackgroundRecord } from "./ScientAgentRuntimePolicy.ts"; const SCIENT_AGENT = ProviderDriverKind.make("scient"); const START_DELAY = "60 seconds"; @@ -37,20 +34,6 @@ const MAX_FAILED_ATTEMPTS_PER_RELEASE = 5; /** The download, the binary, the add-on it unpacks on first run, and staging. */ const MIN_FREE_BYTES = 2 * 1024 ** 3; -/** Kept outside the runtime folder, which Remove deletes. */ -const BackgroundRecord = Schema.Struct({ - /** The user removed Scient Agent or cancelled its install: install only on request. */ - userStopped: Schema.optionalKey(Schema.Boolean), - /** An update the user cancelled; a newer release is offered again. */ - skippedVersion: Schema.optionalKey(Schema.String), - failures: Schema.optionalKey(Schema.Struct({ version: Schema.String, count: Schema.Number })), - firstPickRecorded: Schema.optionalKey(Schema.Boolean), -}); -type BackgroundRecord = typeof BackgroundRecord.Type; -const BackgroundRecordJson = Schema.fromJsonString(BackgroundRecord); -const decodeRecord = Schema.decodeEffect(BackgroundRecordJson); -const encodeRecord = Schema.encodeEffect(BackgroundRecordJson); - export type BackgroundCheck = | { readonly kind: "start"; readonly action: "install" | "update"; readonly version: string } | { readonly kind: "wait"; readonly reason: string }; @@ -61,6 +44,7 @@ export function backgroundRuntimeAction( record: BackgroundRecord, ): BackgroundCheck { const runtime = provider.connection?.runtime; + if (runtime?.pendingRecovery) return { kind: "wait", reason: "requested-recovery" }; if (!provider.enabled || !runtime) return { kind: "wait", reason: "unavailable" }; const action: ProviderManagedRuntimeAction | undefined = runtime.source === "missing" && runtime.actions.includes("install") @@ -84,28 +68,6 @@ export function backgroundRuntimeAction( return { kind: "start", action, version }; } -/** - * How a finished change the user made moves the record: Remove, or Cancel of an install, stops - * background installs; Cancel of an update skips that release; their own install or update - * lets Scient take over again. - */ -export function recordAfterUserChange( - operation: Pick, "action" | "status">, - offeredVersion: string | null, -): ((record: BackgroundRecord) => BackgroundRecord) | undefined { - const { action, status } = operation; - if ( - (action === "remove" && status === "succeeded") || - (action === "install" && status === "cancelled") - ) - return (record) => ({ ...record, userStopped: true }); - if (action === "update" && status === "cancelled" && offeredVersion !== null) - return (record) => ({ ...record, skippedVersion: offeredVersion }); - if (action !== "remove" && status === "succeeded") - return ({ userStopped: _stopped, skippedVersion: _skipped, ...rest }) => rest; - return undefined; -} - /** True when the OS says the connection is metered. Unknown counts as not metered. */ export function parseMeteredOutput(platform: NodeJS.Platform, output: string): boolean { if (platform === "win32") return /^(Fixed|Variable)$/mu.test(output.trim()); @@ -137,33 +99,11 @@ export const make = Effect.fn("ScientAgentBackgroundRuntime.make")(function* () const processes = yield* ProcessRunner.ProcessRunner; const analytics = yield* AnalyticsService; const platform = yield* HostProcess.Platform; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const recordPath = path.join( - config.baseDir, - "provider-runtimes", - ".background", - "scient-agent.json", - ); - - const readRecord = fs.readFileString(recordPath).pipe( - Effect.flatMap(decodeRecord), - Effect.orElseSucceed((): BackgroundRecord => ({})), - ); - const recordLock = yield* Semaphore.make(1); - /** Applies a change under the lock and returns the record from before it. */ - const updateRecord = (change: (record: BackgroundRecord) => BackgroundRecord) => - recordLock.withPermits(1)( - Effect.gen(function* () { - const record = yield* readRecord; - const text = yield* encodeRecord(change(record)); - yield* fs.makeDirectory(path.dirname(recordPath), { recursive: true }); - yield* fs.writeFileString(recordPath, text); - return record; - }), - ); + const policy = yield* ScientAgentRuntimePolicy.ScientAgentRuntimePolicy; + const readRecord = policy.read; + const updateRecord = policy.update; const changeRecord = (change: (record: BackgroundRecord) => BackgroundRecord) => - updateRecord(change).pipe(Effect.ignore); + updateRecord(change).pipe(Effect.asVoid); const freeBytes = Effect.promise(() => NodeFSP.statfs(config.baseDir).then( @@ -219,54 +159,56 @@ export const make = Effect.fn("ScientAgentBackgroundRuntime.make")(function* () ); type Pass = "ready" | "failed" | "busy" | "waiting"; - const tryOnce = Effect.fn("ScientAgentBackgroundRuntime.tryOnce")( - function* (): Effect.fn.Return { - const provider = yield* scientAgent; - if (!provider) return "waiting"; - const record = yield* readRecord; - const check = backgroundRuntimeAction(provider, record); - if (check.kind === "wait") return "waiting"; - if (yield* anyRuntimeBusy) return "busy"; - if ((yield* freeBytes) < MIN_FREE_BYTES) return "waiting"; - if (yield* metered) return "waiting"; - const planned = yield* manager - .plan({ instanceId: provider.instanceId, action: check.action }) - .pipe(Effect.option); - if (Option.isNone(planned)) return "waiting"; - const failures = record.failures?.version === check.version ? record.failures.count : 0; - const outcome = yield* manager - .startInBackground({ - instanceId: provider.instanceId, - action: check.action, - catalogRevision: planned.value.catalogRevision, - attempt: failures + 1, - }) - .pipe(Effect.orElseSucceed(() => "busy" as const)); - switch (outcome) { - case "succeeded": - yield* changeRecord(({ failures: _failures, ...rest }) => rest); - return "ready"; - case "failed": - yield* changeRecord((current) => ({ - ...current, - failures: { version: check.version, count: failures + 1 }, - })); - return "failed"; - case "cancelled": - yield* changeRecord((current) => - check.action === "install" - ? { ...current, userStopped: true } - : { ...current, skippedVersion: check.version }, - ); - return "waiting"; - case "yielded": - case "busy": - return "busy"; - case "interrupted": - return yield* Effect.interrupt; + const tryOnce = Effect.fn("ScientAgentBackgroundRuntime.tryOnce")(function* (): Effect.fn.Return< + Pass, + ScientAgentRuntimePolicy.ScientAgentRuntimePolicyError + > { + const provider = yield* scientAgent; + if (!provider) return "waiting"; + const record = yield* readRecord; + const check = backgroundRuntimeAction(provider, record); + if (check.kind === "wait") return "waiting"; + if (yield* anyRuntimeBusy) return "busy"; + if ((yield* freeBytes) < MIN_FREE_BYTES) return "waiting"; + if (yield* metered) return "waiting"; + const planned = yield* manager + .plan({ instanceId: provider.instanceId, action: check.action }) + .pipe(Effect.option); + if (Option.isNone(planned)) return "waiting"; + const failures = record.failures?.version === check.version ? record.failures.count : 0; + const outcome = yield* manager + .startInBackground({ + instanceId: provider.instanceId, + action: check.action, + catalogRevision: planned.value.catalogRevision, + attempt: failures + 1, + }) + .pipe(Effect.orElseSucceed(() => "busy" as const)); + switch (outcome) { + case "succeeded": + yield* changeRecord(({ failures: _failures, ...rest }) => rest); + return "ready"; + case "failed": + yield* changeRecord((current) => ({ + ...current, + failures: { version: check.version, count: failures + 1 }, + })); + return "failed"; + case "cancelled": { + const change = recordAfterUserChange( + { action: check.action, status: "cancelled" }, + check.version, + ); + if (change) yield* changeRecord(change); + return "waiting"; } - }, - ); + case "yielded": + case "busy": + return "busy"; + case "interrupted": + return yield* Effect.interrupt; + } + }); // A newer release wakes the loop at once, so no update notice shows first. const offeredVersion = (provider: ServerProvider | undefined) => @@ -287,7 +229,9 @@ export const make = Effect.fn("ScientAgentBackgroundRuntime.make")(function* () let failedInARow = 0; while (true) { yield* Ref.set(lastOfferRef, offeredVersion(yield* scientAgent)); - const pass = yield* tryOnce(); + const pass = yield* tryOnce().pipe( + Effect.catchTags({ ScientAgentRuntimePolicyError: () => Effect.succeed("busy" as const) }), + ); if (pass === "failed") { const delay = FAILURE_BACKOFF[failedInARow]; failedInARow += 1; @@ -313,11 +257,16 @@ export const make = Effect.fn("ScientAgentBackgroundRuntime.make")(function* () ?.runtime; const operation = runtime?.operation; if (!operation || operation.background || operation.finishedAt === null) return; - if ((yield* Ref.getAndSet(handled, operation.operationId)) === operation.operationId) - return; + if ((yield* Ref.get(handled)) === operation.operationId) return; const change = recordAfterUserChange(operation, runtime.availableManagedVersion ?? null); if (change) yield* changeRecord(change); - }), + yield* Ref.set(handled, operation.operationId); + }).pipe( + Effect.catchTags({ + ScientAgentRuntimePolicyError: () => + Effect.logWarning("Agent background policy persistence failed"), + }), + ), ), ); @@ -368,4 +317,5 @@ export function firstPickState(provider: ServerProvider): { export const layer = Layer.effect(ScientAgentBackgroundRuntime, make()).pipe( Layer.provide(ProcessRunner.layer), + Layer.provide(ScientAgentRuntimePolicy.layer), ); diff --git a/apps/server/src/scient/providerLifecycle/ScientAgentRuntimePolicy.ts b/apps/server/src/scient/providerLifecycle/ScientAgentRuntimePolicy.ts new file mode 100644 index 0000000000..578208691c --- /dev/null +++ b/apps/server/src/scient/providerLifecycle/ScientAgentRuntimePolicy.ts @@ -0,0 +1,124 @@ +import type { ProviderRuntimeSummary } from "@t3tools/contracts"; +import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; + +import { ServerConfig } from "../../config.ts"; + +/** Kept outside the runtime folder, which Remove deletes. */ +const BackgroundRecord = Schema.Struct({ + /** The user removed Scient Agent or cancelled install/repair: install only on request. */ + userStopped: Schema.optionalKey(Schema.Boolean), + /** An update the user cancelled; a newer release is offered again. */ + skippedVersion: Schema.optionalKey(Schema.String), + failures: Schema.optionalKey(Schema.Struct({ version: Schema.String, count: Schema.Number })), + firstPickRecorded: Schema.optionalKey(Schema.Boolean), +}); +export type BackgroundRecord = typeof BackgroundRecord.Type; +const BackgroundRecordJson = Schema.fromJsonString(BackgroundRecord); +const decodeRecord = Schema.decodeEffect(BackgroundRecordJson); +const encodeRecord = Schema.encodeEffect(BackgroundRecordJson); + +/** + * Remove or cancelled install stops automatic installs; cancelled update skips that release. + * Cancelled repair does both. A successful requested change lets automatic setup resume. + */ +export function recordAfterUserChange( + operation: Pick, "action" | "status">, + offeredVersion: string | null, +): ((record: BackgroundRecord) => BackgroundRecord) | undefined { + const { action, status } = operation; + if ( + (action === "remove" && status === "succeeded") || + (action === "install" && status === "cancelled") + ) + return (record) => ({ ...record, userStopped: true }); + if (action === "repair" && status === "cancelled") + return (record) => ({ + ...record, + userStopped: true, + ...(offeredVersion === null ? {} : { skippedVersion: offeredVersion }), + }); + if (action === "update" && status === "cancelled" && offeredVersion !== null) + return (record) => ({ ...record, skippedVersion: offeredVersion }); + if (action !== "remove" && status === "succeeded") + return ({ userStopped: _stopped, skippedVersion: _skipped, ...rest }) => rest; + return undefined; +} + +export class ScientAgentRuntimePolicyError extends Schema.TaggedError()( + "ScientAgentRuntimePolicyError", + { path: Schema.String, cause: Schema.Defect() }, +) { + override get message(): string { + return "Scient could not save the Agent background setup policy. Try again."; + } +} + +export class ScientAgentRuntimePolicy extends Context.Service< + ScientAgentRuntimePolicy, + { + readonly read: Effect.Effect; + /** Returns the previous record only after the change is persisted. */ + readonly update: ( + change: (record: BackgroundRecord) => BackgroundRecord, + ) => Effect.Effect; + } +>()("t3/scient/providerLifecycle/ScientAgentRuntimePolicy") {} + +const make = Effect.gen(function* () { + const config = yield* ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const platform = yield* HostProcess.Platform; + const recordPath = path.join( + config.baseDir, + "provider-runtimes", + ".background", + "scient-agent.json", + ); + const read = fs.readFileString(recordPath).pipe( + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed("{}"), + ), + Effect.flatMap(decodeRecord), + Effect.mapError((cause) => new ScientAgentRuntimePolicyError({ path: recordPath, cause })), + ); + const lock = yield* Semaphore.make(1); + const update = Effect.fn("ScientAgentRuntimePolicy.update")(function* ( + change: (record: BackgroundRecord) => BackgroundRecord, + ) { + return yield* lock.withPermits(1)( + Effect.uninterruptible( + Effect.gen(function* () { + const record = yield* read; + const contents = yield* encodeRecord(change(record)).pipe(Effect.orDie); + yield* writeFileStringAtomically({ + filePath: recordPath, + contents, + mode: 0o600, + durable: true, + }).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(HostProcess.Platform, platform), + Effect.mapError( + (cause) => new ScientAgentRuntimePolicyError({ path: recordPath, cause }), + ), + ); + return record; + }), + ), + ); + }); + return ScientAgentRuntimePolicy.of({ read, update }); +}); + +export const layer = Layer.effect(ScientAgentRuntimePolicy, make); diff --git a/docs/internals/provider-lifecycle.md b/docs/internals/provider-lifecycle.md index 6a8a5d9174..26650813af 100644 --- a/docs/internals/provider-lifecycle.md +++ b/docs/internals/provider-lifecycle.md @@ -589,16 +589,24 @@ change can interrupt it even while that read is blocked. Cleanup releases only i Requested installs, updates and repairs retain their reviewed plan and a stable request identity across transient failures and process restarts. While waiting, the runtime snapshot exposes scheduled recovery separately from a running operation. Settings shows the next retry time and **Cancel retry**; -cancel clears the durable request, including during background startup. Success, removal, disabling +cancel clears the durable request, including during background startup. Agent cancellation first +saves the same stop/skip policy used for an active change: install stops automatic installs, update +skips that release, and repair does both. No synthetic running operation is created for a pending +request. A failed policy or request write reports an error and retains the request for a later try. +Success, removal, disabling the instance, or a superseding choice also clears it. Recovery rechecks the enabled instance, runtime identity and exact reviewed plan before running. It never provisions a new third-party runtime on -its own; Scient Agent retains its existing automatic provisioning policy. +its own; Scient Agent retains its existing automatic provisioning policy. Automatic setup defers +while a requested recovery owns the runtime, across all instances sharing that runtime. Only the +exact recovery request can use that reservation of intent; it still revalidates the reviewed plan. Recovery waits a jittered five minutes, thirty minutes, then two hours after failures, followed by a four-hour cooldown. A server's Retry-After delay, capped at four hours of remaining time, can extend the next wait: recovery uses the later of its jittered backoff and that remaining delay. A foreground change takes priority over uncommitted background work; yielding preserves the reviewed request and partial -download for a later try. +download for a later try. The request store publishes mutations only after an atomic durable write. +If settlement cannot persist, the manager still finishes operation cleanup and keeps the previous +request and retry deadline. A failed recovery write ends that pass, and the next pass can try again. One lifecycle coordinator serializes connection, managed runtime mutation, and inherited provider maintenance for the same instance. Shared runtime mutation also takes a driver-wide reservation From 9e09eca3b1d009f89bf064521ca93fe998e83b62 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Tue, 17 Nov 2026 00:29:32 +0200 Subject: [PATCH 10/12] fix(recovery): isolate journal failures and revalidate automatic admission policy --- .../ProviderRuntimeManager.test.ts | 163 ++++++++++++++++++ .../ProviderRuntimeManager.ts | 37 +++- .../ProviderRuntimeRecovery.test.ts | 53 ++++++ .../ProviderRuntimeRecovery.ts | 1 + .../ProviderRuntimeRequests.ts | 21 ++- .../ScientAgentBackgroundRuntime.test.ts | 11 +- .../ScientAgentBackgroundRuntime.ts | 42 +---- .../ScientAgentRuntimePolicy.ts | 40 ++++- docs/internals/provider-lifecycle.md | 5 + 9 files changed, 326 insertions(+), 47 deletions(-) diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts index 6b4d5d84e1..5c0285a369 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.test.ts @@ -243,6 +243,7 @@ function makeHarness( const requestsLayer = hooks.requests ? Layer.succeed(ProviderRuntimeRequests, hooks.requests) : Layer.mock(ProviderRuntimeRequests)({ + assertAvailable: Effect.void, list: Effect.succeed([]), changes: Stream.empty, requested: (input) => @@ -1538,6 +1539,168 @@ describe("ProviderRuntimeManager", () => { }).pipe(Effect.provide(agentTestLayer("scient-agent-cooldown-"))), ); + it.effect( + "rechecks Agent stop policy after another instance cancels during automatic planning", + () => + Effect.gen(function* () { + const requests = yield* makeRequests(); + const pending = { + requestId: "cancel-other-instance", + instanceId: SECOND_INSTANCE, + driver: AGENT, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + yield* requests.requested(pending); + const policy = yield* loadAgentPolicy; + const planning = yield* Deferred.make(); + const resume = yield* Deferred.make(); + const runs = yield* Ref.make(0); + const { manager } = yield* makeHarness( + { + getSummary: Effect.succeed(agentRuntime), + plan: () => + Deferred.succeed(planning, undefined).pipe( + Effect.andThen(Deferred.await(resume)), + Effect.as({ ...installPlan(), version: "0.1.6" }), + ), + run: () => Ref.update(runs, (count) => count + 1), + }, + [agentProvider, { ...agentProvider, instanceId: SECOND_INSTANCE }], + undefined, + undefined, + undefined, + { requests, agentPolicy: policy }, + ); + const automatic = yield* manager + .startInBackground({ + instanceId: INSTANCE, + action: "install", + catalogRevision: "reviewed:1", + attempt: 1, + }) + .pipe(Effect.result, Effect.forkChild); + yield* Deferred.await(planning); + yield* manager.cancel({ instanceId: SECOND_INSTANCE, requestId: pending.requestId }); + assert.deepStrictEqual(yield* policy.read, { userStopped: true }); + assert.deepStrictEqual(yield* requests.list, []); + yield* Deferred.succeed(resume, undefined); + const result = yield* Fiber.join(automatic); + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") + assert.strictEqual(result.failure.reason, "runtime_plan_stale"); + assert.strictEqual(yield* Ref.get(runs), 0); + }).pipe(Effect.provide(agentTestLayer("scient-agent-policy-race-"))), + ); + + it.effect("starts the manager with an unavailable journal but rejects runtime mutations", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const guardedFs = FileSystem.FileSystem.of({ + ...fs, + readFileString: (path, encoding) => + path.endsWith("requests.json") + ? Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "readFileString", + pathOrDescriptor: path, + }), + ) + : fs.readFileString(path, encoding), + }); + const requests = yield* makeRequests().pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + const runs = yield* Ref.make(0); + const { manager, registry } = yield* makeHarness( + { + getSummary: Effect.succeed(missingRuntime), + plan: () => Effect.succeed(installPlan()), + run: () => Ref.update(runs, (count) => count + 1), + }, + undefined, + undefined, + undefined, + undefined, + { requests }, + ); + assert.strictEqual( + (yield* manager.plan({ instanceId: INSTANCE, action: "install" })).version, + "0.147.0", + ); + const input = { + instanceId: INSTANCE, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + for (const action of [ + manager.start(input).pipe(Effect.asVoid), + manager.startInBackground({ ...input, attempt: 1 }).pipe(Effect.asVoid), + manager + .cancel({ instanceId: INSTANCE, requestId: "unreadable-request" }) + .pipe(Effect.asVoid), + ]) { + const failure = yield* action.pipe(Effect.flip); + assert.strictEqual(failure.reason, "runtime_operation_failed"); + } + assert.strictEqual(yield* Ref.get(runs), 0); + assert.strictEqual((yield* registry.getProviders)[0]?.connection?.runtime?.operation, null); + }).pipe(Effect.provide(agentTestLayer("scient-manager-unavailable-"))), + ); + + it.effect("keeps the server usable and retains intent when initial pruning cannot persist", () => + Effect.gen(function* () { + const before = yield* makeRequests(); + const orphan = { + requestId: "removed-instance", + instanceId: ProviderInstanceId.make("removed"), + driver: CODEX, + action: "install" as const, + catalogRevision: "reviewed:1", + }; + yield* before.requested(orphan); + const fs = yield* FileSystem.FileSystem; + const guardedFs = FileSystem.FileSystem.of({ + ...fs, + rename: (from, to) => + to.endsWith("requests.json") + ? Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "rename", + pathOrDescriptor: to, + }), + ) + : fs.rename(from, to), + }); + const requests = yield* makeRequests().pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + const { manager, registry } = yield* makeHarness( + { + getSummary: Effect.succeed(missingRuntime), + plan: () => Effect.succeed(installPlan()), + run: () => Effect.die("Unexpected runtime mutation"), + }, + undefined, + undefined, + undefined, + undefined, + { requests }, + ); + assert.strictEqual( + (yield* manager.plan({ instanceId: INSTANCE, action: "install" })).catalogRevision, + "reviewed:1", + ); + assert.strictEqual((yield* registry.getProviders)[0]?.instanceId, INSTANCE); + assert.deepInclude((yield* requests.list)[0], orphan); + assert.deepInclude((yield* (yield* makeRequests()).list)[0], orphan); + }).pipe(Effect.provide(agentTestLayer("scient-manager-prune-denied-"))), + ); + it.effect.each([ { phase: "pending", file: "requests.json" }, { phase: "active", file: "requests.json" }, diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts index b843a3955c..60c783c94d 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeManager.ts @@ -250,6 +250,7 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { const syncPendingRecovery = Effect.fn("ProviderRuntimeManager.syncPendingRecovery")(function* () { yield* publicationLock.withPermits(1)( Effect.gen(function* () { + yield* requests.assertAvailable; const providers = yield* providerRegistry.getProviders; const pending = yield* requests.list; for (const request of pending) { @@ -587,6 +588,9 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { registeredActiveRef: Ref.Ref, ) { const target = yield* readTarget(input.instanceId); + yield* requests.assertAvailable.pipe( + Effect.mapError((error) => persistenceActionError(target, error.message)), + ); const planned = yield* plan({ instanceId: input.instanceId, action: input.action }); if (planned.catalogRevision !== input.catalogRevision) { return yield* makeError({ @@ -617,6 +621,29 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { message: "A requested runtime recovery owns this runtime.", }); } + if ( + background && + !background.requestId && + target.provider === ProviderDriverKind.make("scient") + ) { + const latest = yield* readTarget(input.instanceId); + const record = yield* agentPolicy.read.pipe( + Effect.mapError((error) => persistenceActionError(target, error.message)), + ); + const eligible = ScientAgentRuntimePolicy.backgroundRuntimeAction(latest.snapshot, record); + if ( + eligible.kind !== "start" || + eligible.action !== input.action || + eligible.version !== planned.version + ) { + return yield* makeError({ + provider: target.provider, + instanceId: input.instanceId, + reason: "runtime_plan_stale", + message: "Automatic Agent setup policy changed. No runtime was started.", + }); + } + } const reserved = yield* lifecycleCoordinator.reserve({ instanceId: input.instanceId, provider: target.provider, @@ -1106,6 +1133,9 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { const cancel: ProviderRuntimeManagerShape["cancel"] = Effect.fn("ProviderRuntimeManager.cancel")( function* (input) { const target = yield* readTarget(input.instanceId); + yield* requests.assertAvailable.pipe( + Effect.mapError((error) => persistenceActionError(target, error.message)), + ); let candidate = (yield* Ref.get(activeRef)).get(input.instanceId); const pending = "requestId" in input @@ -1279,7 +1309,12 @@ export const make = Effect.fn("ProviderRuntimeManager.make")(function* () { ); }).pipe(Effect.delay(RESELECT_INTERVAL), Effect.forever, Effect.forkScoped); - yield* syncPendingRecovery(); + yield* syncPendingRecovery().pipe( + Effect.catchTags({ + ProviderRuntimeRequestsPersistenceError: () => + Effect.logWarning("Runtime recovery intent is unavailable; managed changes are disabled"), + }), + ); yield* requests.changes.pipe( Stream.runForEach(() => syncPendingRecovery().pipe( diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts index 1684ad7ace..924e5effe2 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.test.ts @@ -13,6 +13,7 @@ import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; import * as Ref from "effect/Ref"; import * as PlatformError from "effect/PlatformError"; import * as Stream from "effect/Stream"; @@ -144,6 +145,58 @@ describe("ProviderRuntimeRequests", () => { ), ); + it.effect.each(["malformed", "denied"] as const)( + "preserves an unreadable journal and disables only managed changes (%s)", + (mode) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfig.ServerConfig; + const file = path.join(config.baseDir, "provider-runtimes", ".background", "requests.json"); + const contents = mode === "malformed" ? "{broken" : "[]"; + yield* fs.makeDirectory(path.dirname(file), { recursive: true }); + yield* fs.writeFileString(file, contents); + const guardedFs = FileSystem.FileSystem.of({ + ...fs, + readFileString: (target, encoding) => + mode === "denied" && target === file + ? Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "readFileString", + pathOrDescriptor: target, + }), + ) + : fs.readFileString(target, encoding), + }); + const requests = yield* makeRequests().pipe( + Effect.provideService(FileSystem.FileSystem, guardedFs), + ); + const unavailable = yield* requests.assertAvailable.pipe(Effect.flip); + assert.strictEqual(unavailable._tag, "ProviderRuntimeRequestsPersistenceError"); + for (const mutation of [ + requests.requested(request), + requests.update(request.requestId, () => undefined), + requests.settled({ ...request, outcome: "cancelled" }), + ]) { + assert.strictEqual((yield* mutation.pipe(Effect.flip))._tag, unavailable._tag); + } + assert.strictEqual(yield* fs.readFileString(file), contents); + yield* fs.writeFileString(file, "[]"); + const restarted = yield* makeRequests(); + yield* restarted.assertAvailable; + yield* restarted.requested(request); + assert.deepInclude((yield* (yield* makeRequests()).list)[0], request); + }).pipe( + Effect.provide( + ServerConfig.layerTest(NodeOS.tmpdir(), { prefix: "scient-requests-unavailable-" }).pipe( + Layer.provideMerge(Layer.merge(NodeServices.layer, NodeCrypto.layer)), + ), + ), + ), + ); + it.effect.each([1_000, 2 * 60 * MINUTE])( "persists the later of retry jitter and the source cooldown (%s ms)", (retryAfterMs) => diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts index 81882887af..12a5313c5b 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRecovery.ts @@ -32,6 +32,7 @@ export const make = Effect.fn("ProviderRuntimeRecovery.make")(function* () { const registry = yield* ProviderRegistry; const tryDue = Effect.fn("ProviderRuntimeRecovery.tryDue")(function* () { + yield* requests.assertAvailable; const now = yield* Clock.currentTimeMillis; const request = (yield* requests.list) .filter((candidate) => candidate.dueAt <= now) diff --git a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts index 9284eb4996..cb9e91ceb9 100644 --- a/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts +++ b/apps/server/src/scient/providerLifecycle/ProviderRuntimeRequests.ts @@ -13,6 +13,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Random from "effect/Random"; +import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; @@ -98,6 +99,8 @@ export class ProviderRuntimeRequests extends Context.Service< readonly requestId?: string; readonly retryAfterMs?: number; }) => Effect.Effect; + /** Unreadable state disables managed changes, not the rest of the server. */ + readonly assertAvailable: Effect.Effect; readonly list: Effect.Effect>; readonly changes: Stream.Stream>; /** Changes this exact request if it still exists; `undefined` drops it. */ @@ -116,7 +119,7 @@ export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { const random = yield* SecureRandom.SecureRandom; // Kept outside the runtime folders, which Remove deletes. const filePath = path.join(config.baseDir, "provider-runtimes", ".background", "requests.json"); - const stored = yield* fs.readFileString(filePath).pipe( + const loaded = yield* fs.readFileString(filePath).pipe( Effect.catchIf( (error) => error.reason._tag === "NotFound", () => Effect.succeed("[]"), @@ -125,6 +128,14 @@ export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { Effect.mapError( (cause) => new ProviderRuntimeRequestsPersistenceError({ path: filePath, cause }), ), + Effect.result, + ); + const stored = loaded._tag === "Success" ? loaded.success : []; + const unavailable = yield* Ref.make( + loaded._tag === "Failure" ? loaded.failure : undefined, + ); + const assertAvailable = Ref.get(unavailable).pipe( + Effect.flatMap((error) => (error === undefined ? Effect.void : Effect.fail(error))), ); const requestsRef = yield* SubscriptionRef.make>( yield* Effect.forEach(stored, (request) => @@ -141,6 +152,7 @@ export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { lock.withPermits(1)( Effect.uninterruptible( Effect.gen(function* () { + yield* assertAvailable; const current = yield* SubscriptionRef.get(requestsRef); const updated = yield* next(current); if (updated === current) return; @@ -161,7 +173,11 @@ export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { ); if (stored.some((request) => request.requestId === undefined)) { - yield* change((requests) => Effect.succeed([...requests])); + yield* change((requests) => Effect.succeed([...requests])).pipe( + Effect.catchTags({ + ProviderRuntimeRequestsPersistenceError: (error) => Ref.set(unavailable, error), + }), + ); } const updateEach = ( @@ -233,6 +249,7 @@ export const make = Effect.fn("ProviderRuntimeRequests.make")(function* () { }); return ProviderRuntimeRequests.of({ + assertAvailable, requested, settled, list: SubscriptionRef.get(requestsRef), diff --git a/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.test.ts b/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.test.ts index de556a5918..6c53e113a5 100644 --- a/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.test.ts +++ b/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.test.ts @@ -21,14 +21,13 @@ import { ProcessRunner } from "../../processRunner.ts"; import { ProviderRegistry } from "../../provider/ProviderRegistry.ts"; import { AnalyticsService } from "../../telemetry/AnalyticsService.ts"; import { ProviderRuntimeManager, type ProviderRuntimeOutcome } from "./ProviderRuntimeManager.ts"; +import { firstPickState, make, parseMeteredOutput } from "./ScientAgentBackgroundRuntime.ts"; + import { backgroundRuntimeAction, - firstPickState, - make, - parseMeteredOutput, -} from "./ScientAgentBackgroundRuntime.ts"; - -import { recordAfterUserChange, ScientAgentRuntimePolicy } from "./ScientAgentRuntimePolicy.ts"; + recordAfterUserChange, + ScientAgentRuntimePolicy, +} from "./ScientAgentRuntimePolicy.ts"; const INSTANCE = ProviderInstanceId.make("scient"); diff --git a/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.ts b/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.ts index b68974884e..eb29c8b616 100644 --- a/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.ts +++ b/apps/server/src/scient/providerLifecycle/ScientAgentBackgroundRuntime.ts @@ -4,7 +4,6 @@ import * as NodeFSP from "node:fs/promises"; import { ProviderDriverKind, type ProviderInstanceId, - type ProviderManagedRuntimeAction, type ServerProvider, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -21,7 +20,11 @@ import { ProviderRegistry } from "../../provider/ProviderRegistry.ts"; import { AnalyticsService } from "../../telemetry/AnalyticsService.ts"; import { ProviderRuntimeManager } from "./ProviderRuntimeManager.ts"; import * as ScientAgentRuntimePolicy from "./ScientAgentRuntimePolicy.ts"; -import { recordAfterUserChange, type BackgroundRecord } from "./ScientAgentRuntimePolicy.ts"; +import { + backgroundRuntimeAction, + recordAfterUserChange, + type BackgroundRecord, +} from "./ScientAgentRuntimePolicy.ts"; const SCIENT_AGENT = ProviderDriverKind.make("scient"); const START_DELAY = "60 seconds"; @@ -30,44 +33,9 @@ const RECHECK_INTERVAL = "1 hour"; const FAILURE_BACKOFF = ["5 minutes", "30 minutes", "2 hours"] as const; /** Busy with other work, or stopped for the user's own change. */ const BUSY_RETRY = "5 minutes"; -const MAX_FAILED_ATTEMPTS_PER_RELEASE = 5; /** The download, the binary, the add-on it unpacks on first run, and staging. */ const MIN_FREE_BYTES = 2 * 1024 ** 3; -export type BackgroundCheck = - | { readonly kind: "start"; readonly action: "install" | "update"; readonly version: string } - | { readonly kind: "wait"; readonly reason: string }; - -/** What, if anything, Scient may get ready on its own for this instance. */ -export function backgroundRuntimeAction( - provider: ServerProvider, - record: BackgroundRecord, -): BackgroundCheck { - const runtime = provider.connection?.runtime; - if (runtime?.pendingRecovery) return { kind: "wait", reason: "requested-recovery" }; - if (!provider.enabled || !runtime) return { kind: "wait", reason: "unavailable" }; - const action: ProviderManagedRuntimeAction | undefined = - runtime.source === "missing" && runtime.actions.includes("install") - ? "install" - : runtime.source === "scient_managed" && runtime.actions.includes("update") - ? "update" - : undefined; - const version = - action === "update" - ? runtime.availableManagedVersion - : (runtime.availableManagedVersion ?? runtime.qualifiedVersion); - if (action === undefined || !version) return { kind: "wait", reason: "nothing-to-do" }; - if (action === "install" && record.userStopped) return { kind: "wait", reason: "user-stopped" }; - if (action === "update" && record.skippedVersion === version) - return { kind: "wait", reason: "user-stopped" }; - if ( - record.failures?.version === version && - record.failures.count >= MAX_FAILED_ATTEMPTS_PER_RELEASE - ) - return { kind: "wait", reason: "gave-up" }; - return { kind: "start", action, version }; -} - /** True when the OS says the connection is metered. Unknown counts as not metered. */ export function parseMeteredOutput(platform: NodeJS.Platform, output: string): boolean { if (platform === "win32") return /^(Fixed|Variable)$/mu.test(output.trim()); diff --git a/apps/server/src/scient/providerLifecycle/ScientAgentRuntimePolicy.ts b/apps/server/src/scient/providerLifecycle/ScientAgentRuntimePolicy.ts index 578208691c..ff02217e67 100644 --- a/apps/server/src/scient/providerLifecycle/ScientAgentRuntimePolicy.ts +++ b/apps/server/src/scient/providerLifecycle/ScientAgentRuntimePolicy.ts @@ -1,4 +1,8 @@ -import type { ProviderRuntimeSummary } from "@t3tools/contracts"; +import type { + ProviderManagedRuntimeAction, + ProviderRuntimeSummary, + ServerProvider, +} from "@t3tools/contracts"; import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Context from "effect/Context"; @@ -24,7 +28,41 @@ export type BackgroundRecord = typeof BackgroundRecord.Type; const BackgroundRecordJson = Schema.fromJsonString(BackgroundRecord); const decodeRecord = Schema.decodeEffect(BackgroundRecordJson); const encodeRecord = Schema.encodeEffect(BackgroundRecordJson); +const MAX_FAILED_ATTEMPTS_PER_RELEASE = 5; +type BackgroundCheck = + | { readonly kind: "start"; readonly action: "install" | "update"; readonly version: string } + | { readonly kind: "wait"; readonly reason: string }; + +/** What, if anything, Scient may get ready on its own for this instance. */ +export function backgroundRuntimeAction( + provider: ServerProvider, + record: BackgroundRecord, +): BackgroundCheck { + const runtime = provider.connection?.runtime; + if (runtime?.pendingRecovery) return { kind: "wait", reason: "requested-recovery" }; + if (!provider.enabled || !runtime) return { kind: "wait", reason: "unavailable" }; + const action: ProviderManagedRuntimeAction | undefined = + runtime.source === "missing" && runtime.actions.includes("install") + ? "install" + : runtime.source === "scient_managed" && runtime.actions.includes("update") + ? "update" + : undefined; + const version = + action === "update" + ? runtime.availableManagedVersion + : (runtime.availableManagedVersion ?? runtime.qualifiedVersion); + if (action === undefined || !version) return { kind: "wait", reason: "nothing-to-do" }; + if (action === "install" && record.userStopped) return { kind: "wait", reason: "user-stopped" }; + if (action === "update" && record.skippedVersion === version) + return { kind: "wait", reason: "user-stopped" }; + if ( + record.failures?.version === version && + record.failures.count >= MAX_FAILED_ATTEMPTS_PER_RELEASE + ) + return { kind: "wait", reason: "gave-up" }; + return { kind: "start", action, version }; +} /** * Remove or cancelled install stops automatic installs; cancelled update skips that release. * Cancelled repair does both. A successful requested change lets automatic setup resume. diff --git a/docs/internals/provider-lifecycle.md b/docs/internals/provider-lifecycle.md index 26650813af..2f748dccc1 100644 --- a/docs/internals/provider-lifecycle.md +++ b/docs/internals/provider-lifecycle.md @@ -607,6 +607,11 @@ takes priority over uncommitted background work; yielding preserves the reviewed download for a later try. The request store publishes mutations only after an atomic durable write. If settlement cannot persist, the manager still finishes operation cleanup and keeps the previous request and retry deadline. A failed recovery write ends that pass, and the next pass can try again. +An unreadable or malformed recovery journal does not block server startup. Managed-runtime +changes and recovery fail closed while that store is unavailable, preserving the file; +repair its permissions/content and restart to re-enable them. Startup pruning errors are +contained without discarding intent. Automatic Agent admission rechecks the current durable +stop/skip policy after planning, including across instances sharing one runtime. One lifecycle coordinator serializes connection, managed runtime mutation, and inherited provider maintenance for the same instance. Shared runtime mutation also takes a driver-wide reservation From 6bcdd01428040ee3c655d75569b68f5c6a53a4b6 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Tue, 17 Nov 2026 00:40:29 +0200 Subject: [PATCH 11/12] test(release): rehearse the final publisher pin gate with real helper --- docs/operations/release.md | 5 +++ scripts/scient-beta-publication.test.ts | 46 +++++++++++++++++++++---- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/docs/operations/release.md b/docs/operations/release.md index c0718fbb99..636bf2b377 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -313,6 +313,11 @@ only if its installer's signer CN matches the `publisherName` in the installed one ships. Changing the publisher later needs a planned transition, not a release that installed clients would reject. +The local publisher rehearsal runs the production signing helper against synthetic +candidate/public handoffs, including a publisher pin that appears before promotion. +It must withhold public promotion on either a changed upload or an incompatible pin; +the helper is not replaced with a successful stub. + Release jobs expose Apple credentials only to macOS packaging and Azure credentials only to the Linux-hosted Windows packaging job. Publication additionally verifies the final Windows Authenticode signature and the final macOS app signatures, diff --git a/scripts/scient-beta-publication.test.ts b/scripts/scient-beta-publication.test.ts index f4a1e0270c..bb2cabb076 100644 --- a/scripts/scient-beta-publication.test.ts +++ b/scripts/scient-beta-publication.test.ts @@ -99,6 +99,7 @@ else if (args.includes('repos/ScientFactory/scient-desktop-beta/releases?per_pag { channel: "beta", failure: "none" }, { channel: "beta", failure: "corrupt-upload" }, { channel: "beta", failure: "stable-advanced" }, + { channel: "beta", failure: "publisher-pinned" }, ])( "publishes $channel only into its owning repository, failure=$failure", ({ channel, failure }) => { @@ -110,6 +111,21 @@ else if (args.includes('repos/ScientFactory/scient-desktop-beta/releases?per_pag try { NodeFS.mkdirSync(bin); NodeFS.mkdirSync(NodePath.join(root, "release-assets")); + NodeFS.mkdirSync(NodePath.join(root, "scripts")); + NodeFS.copyFileSync( + NodePath.join(import.meta.dirname, "windows-publication-signing.ts"), + NodePath.join(root, "scripts/windows-publication-signing.ts"), + ); + NodeFS.writeFileSync( + NodePath.join(root, "release-assets/scient-release-handoff.json"), + JSON.stringify({ + version, + windows: + channel === "beta" + ? { signed: false } + : { signed: true, publisherName: "ScientFactory Ltd" }, + }), + ); NodeFS.writeFileSync( NodePath.join(root, "release-assets/fixture.txt"), "immutable fixture", @@ -122,17 +138,32 @@ const path = require('node:path'); const args = process.argv.slice(2); fs.appendFileSync(process.env.CALLS, JSON.stringify(args) + '\\n'); if (args[0] === 'api') { - if (args[1].endsWith('/releases/latest')) process.stdout.write(process.env.RELEASE_CHANNEL === 'beta' ? (process.env.FAILURE === 'stable-advanced' ? 'v0.6.23' : 'v0.6.22') : process.env.RELEASE_TAG); - else if (args[1] === 'repos/ScientFactory/scient-desktop-beta') process.stdout.write('main'); + const endpoint = args.find((arg) => arg.startsWith('repos/')); + if (endpoint?.endsWith('/releases/latest')) { + const tag = process.env.RELEASE_CHANNEL === 'beta' + ? (process.env.FAILURE === 'stable-advanced' ? 'v0.6.23' : 'v0.6.22') + : process.env.RELEASE_TAG; + process.stdout.write(args.includes('--jq') ? tag : JSON.stringify({ tag_name: 'v0.6.22' })); + } else if (endpoint === 'repos/ScientFactory/scient-desktop-beta/releases?per_page=100') { + process.stdout.write(JSON.stringify([[]])); + } else if (endpoint === 'repos/ScientFactory/scient-desktop-beta') process.stdout.write('main'); else process.exit(2); } else if (args[0] === 'release' && ['create', 'edit'].includes(args[1])) { if (args[1] === 'create') fs.writeFileSync(process.env.NOTES_RECEIPT, fs.readFileSync(args[args.indexOf('--notes-file') + 1])); } else if (args[0] === 'release' && args[1] === 'view') { process.stdout.write(JSON.stringify({ isDraft: false, isPrerelease: true })); } else if (args[0] === 'release' && args[1] === 'download') { - const destination = args[args.indexOf('--dir') + 1]; - for (const name of fs.readdirSync('release-assets')) fs.copyFileSync(path.join('release-assets', name), path.join(destination, name)); - if (process.env.FAILURE === 'corrupt-upload') fs.writeFileSync(path.join(destination, 'fixture.txt'), 'modified upload'); + if (args.includes('--output')) { + process.stdout.write(JSON.stringify({ + windows: process.env.FAILURE === 'publisher-pinned' + ? { signed: true, publisherName: 'ScientFactory Ltd' } + : { signed: false }, + })); + } else { + const destination = args[args.indexOf('--dir') + 1]; + for (const name of fs.readdirSync('release-assets')) fs.copyFileSync(path.join('release-assets', name), path.join(destination, name)); + if (process.env.FAILURE === 'corrupt-upload') fs.writeFileSync(path.join(destination, 'fixture.txt'), 'modified upload'); + } } else process.exit(2); `, { mode: 0o700 }, @@ -155,6 +186,7 @@ process.stdout.write(process.env.SOURCE_SHA + '\\trefs/tags/' + process.env.RELE ...process.env, PATH: `${bin}:${process.env.PATH}`, GH_TOKEN: "synthetic-unused-token", + PIN_READ_TOKEN: "synthetic-read-token", CALLS: calls, FAILURE: failure, NOTES_RECEIPT: NodePath.join(root, "published-notes.md"), @@ -184,11 +216,11 @@ process.stdout.write(process.env.SOURCE_SHA + '\\trefs/tags/' + process.env.RELE ); if (failure !== "none") { expect(result.status).not.toBe(0); - expect(mutations).toHaveLength(failure === "corrupt-upload" ? 1 : 0); + expect(mutations).toHaveLength(failure === "stable-advanced" ? 0 : 1); expect(mutations.some((args) => args[1] === "edit")).toBe(false); return; } - expect(result.status).toBe(0); + expect(result.status, result.stderr.toString()).toBe(0); expect(mutations).toHaveLength(2); for (const args of mutations) expect(args[args.indexOf("--repo") + 1]).toBe(repository); if (channel === "beta") { From e89536a2498d9d8d720216492218d6ebc90548a6 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Tue, 17 Nov 2026 08:57:15 +0200 Subject: [PATCH 12/12] test(analytics): the composed server provides recorded-event ingest Co-Authored-By: Claude Opus 5.5 --- apps/server/src/server.scient.test.ts | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/apps/server/src/server.scient.test.ts b/apps/server/src/server.scient.test.ts index 639203c00f..52fb85ac00 100644 --- a/apps/server/src/server.scient.test.ts +++ b/apps/server/src/server.scient.test.ts @@ -4,6 +4,7 @@ import type { AnalyticsRuntime } from "@scientfactory/analytics"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { assert, it } from "@effect/vitest"; import * as ConfigProvider from "effect/ConfigProvider"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as HttpClient from "effect/http/HttpClient"; @@ -14,9 +15,12 @@ import * as ServerConfig from "./config.ts"; import * as ManagedPolicy from "./policy/ManagedPolicy.ts"; import { layerAnalytics } from "./server.ts"; import { AnalyticsService } from "./telemetry/AnalyticsService.ts"; -import { ScientAnalyticsAvailability } from "./telemetry/ScientAnalytics.ts"; +import { + ScientAnalyticsAvailability, + ScientRecordedAnalyticsIngest, +} from "./telemetry/ScientAnalytics.ts"; -const scient = vi.hoisted(() => ({ recorded: [] as string[] })); +const scient = vi.hoisted(() => ({ recorded: [] as string[], ingested: [] as string[] })); vi.mock("@scientfactory/analytics", async (importOriginal) => ({ ...(await importOriginal()), createAnalyticsRuntime: (): AnalyticsRuntime => ({ @@ -25,6 +29,10 @@ vi.mock("@scientfactory/analytics", async (importOriginal) => ({ scient.recorded.push(name); return true; }, + ingest: async (events) => { + scient.ingested.push(...events.map((event) => event.name)); + return true; + }, flush: async () => 0, pendingCount: async () => 0, close: async () => undefined, @@ -36,6 +44,7 @@ it.effect( () => Effect.gen(function* () { scient.recorded.length = 0; + scient.ingested.length = 0; const posthogRequests: string[] = []; const posthog = HttpClient.make((request) => Effect.sync(() => { @@ -48,6 +57,17 @@ it.effect( yield* analytics.record("provider.turn.sent", { provider: "codex" }); yield* analytics.recordOwners([{ source: "openai", ownerId: "a".repeat(64) }], "server"); yield* analytics.flush; + // Facts another process journaled reach the same outbox, not the never-sent default. + yield* (yield* ScientRecordedAnalyticsIngest).ingest([ + { + id: "6f1c2a9e-4b7d-4e21-9a3f-0c5d8e7b1a24", + name: "app.update.summary", + occurredAt: DateTime.formatIso(yield* DateTime.now), + appVersion: "0.6.29", + buildChannel: "stable", + properties: { reason: "launch", status: "idle" }, + }, + ]); return (yield* ScientAnalyticsAvailability).available; }).pipe( Effect.provide( @@ -83,6 +103,7 @@ it.effect( "install.owner_observed", "app.session.ended", ]); + assert.deepEqual(scient.ingested, ["app.update.summary"]); assert.isTrue(available); }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), );