diff --git a/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx index a3b698dd9aff8..d256e5c548d15 100644 --- a/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx @@ -126,7 +126,9 @@ function managedProviders( function canManageProvider(provider: ServerProvider) { return Boolean( - provider.setup?.canAuthenticate || (provider.driver === "acpRegistry" && provider.installed), + provider.setup?.canAuthenticate || + provider.setup?.canInstall || + (provider.driver === "acpRegistry" && provider.installed), ); } @@ -143,6 +145,10 @@ function ProviderAccount({ const instanceId = provider.instanceId; const target = { environmentId, input: { instanceId } }; const auth = useEnvironmentQuery(serverEnvironment.providerAuthState(target)); + const installation = useEnvironmentQuery( + provider.setup?.canInstall ? serverEnvironment.providerInstallState(target) : null, + ); + const [installationDetailsOpen, setInstallationDetailsOpen] = useState(false); const options = { reportFailure: false, reportDefect: false }; const start = useAtomCommand(serverEnvironment.startProviderAuth, options); const respond = useAtomCommand(serverEnvironment.respondProviderAuth, options); @@ -408,6 +414,58 @@ function ProviderAccount({ {trailing} + {installation.data?.phase === "failed" && installation.data.diagnostics ? ( + + + {installation.data.message} + + setInstallationDetailsOpen((open) => !open)} + > + Installation details + + {installationDetailsOpen ? ( + <> + + {installation.data.diagnostics.step} + + {installation.data.operationId ? ( + + Attempt: {installation.data.operationId} + + ) : null} + {installation.data.diagnostics.systemError ? ( + + {installation.data.diagnostics.systemError} + + ) : null} + {installation.data.diagnostics.exitCode !== null ? ( + + Exit code: {installation.data.diagnostics.exitCode} + + ) : null} + {installation.data.diagnostics.signal !== null ? ( + + Exit signal: {installation.data.diagnostics.signal} + + ) : null} + + + {installation.data.diagnostics.stderr || + "No startup error output was captured for this check."} + + + {installation.data.diagnostics.truncated ? ( + + Some startup output was omitted or truncated. + + ) : null} + + ) : null} + + ) : null} {interaction?.type === "deviceCode" || interaction?.type === "terminal" || interaction?.type === "credentials" || diff --git a/apps/server/src/provider/AntigravityInstallation.test.ts b/apps/server/src/provider/AntigravityInstallation.test.ts index 8de41a4b3b364..ea13bcb934e38 100644 --- a/apps/server/src/provider/AntigravityInstallation.test.ts +++ b/apps/server/src/provider/AntigravityInstallation.test.ts @@ -718,6 +718,56 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("Antigravity inst }), ); + it.effect("retains startup diagnostics when ACP initialization times out", () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const spawner = ChildProcessSpawner.make(() => + Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.never, + isRunning: Effect.succeed(true), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.never, + stderr: Stream.succeed( + new TextEncoder().encode( + "Could not initialize local harness\nAuthorization: Bearer private-fixture-token\n", + ), + ).pipe(Stream.tap(() => Deferred.succeed(entered, undefined))), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ), + ); + const { installation } = yield* makeHarness({ + previous: true, + useDefaultValidation: true, + spawner, + }); + yield* installation.start; + yield* Deferred.await(entered); + yield* TestClock.adjust("90 seconds"); + const failed = yield* terminalState(installation); + expect(failed).toMatchObject({ + phase: "failed", + diagnostics: { + step: "Antigravity initialization", + stderr: "Could not initialize local harness\nAuthorization: [redacted]", + exitCode: null, + signal: null, + truncated: false, + }, + }); + yield* expectPreviousRelease(installation); + expect((yield* installation.start).diagnostics).toBeNull(); + const running = yield* installation.state; + expect((yield* installation.cancel(running.operationId!)).diagnostics).toBeNull(); + }), + ); + it.effect.each(["downloaded", "existing", "concurrent"] as const)( "reports a verification timeout for a %s runtime without changing the active release", (source) => @@ -765,8 +815,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("Antigravity inst yield* TestClock.adjust("90 seconds"); expect(yield* terminalState(installation)).toMatchObject({ phase: "failed", - message: - "Antigravity runtime verification timed out after 90 seconds. Check this environment's server logs for startup errors, then retry installation.", + message: expect.stringContaining( + "Antigravity runtime verification timed out after 90 seconds.", + ), }); if (source !== "existing") yield* Deferred.await(stagingReleased); yield* expectPreviousRelease(installation); diff --git a/apps/server/src/provider/AntigravityInstallation.ts b/apps/server/src/provider/AntigravityInstallation.ts index f16849e2f56ea..0bae876be9642 100644 --- a/apps/server/src/provider/AntigravityInstallation.ts +++ b/apps/server/src/provider/AntigravityInstallation.ts @@ -5,7 +5,11 @@ import { resolveProviderCompatibility } from "./providerCompatibility.ts"; import { cliArchiveTarCommand } from "@t3tools/shared/cliRelease"; import { compareSemverVersions } from "@t3tools/shared/semver"; import * as EffectNodeStream from "@effect/platform-node/NodeStream"; -import { ProviderDriverKind, type ProviderInstallState } from "@t3tools/contracts"; +import { + ProviderDriverKind, + ProviderInstallDiagnostics, + type ProviderInstallState, +} from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { resolveNodeExecutable, nodeRuntimeUnavailableMessage } from "@t3tools/shared/nodeRuntime"; import { resolveCommandPath } from "@t3tools/shared/shell"; @@ -48,6 +52,7 @@ import { prepareAntigravityProfile, } from "./antigravityAuthSupport.ts"; import { type AntigravityReleaseAsset } from "./antigravityRelease.ts"; +import { makeInstallationDiagnostics } from "./installationDiagnostics.ts"; const DRIVER = ProviderDriverKind.make("antigravity"); const DOWNLOAD_TIMEOUT = "45 minutes"; @@ -78,6 +83,7 @@ export class AntigravityInstallationError extends Schema.TaggedError - Effect.fail( - installationError( - "verify", - "Antigravity runtime verification timed out after 90 seconds. Check this environment's server logs for startup errors, then retry installation.", - ), - ), + orElse: () => Effect.fail(validationTimeoutError), }); - const validate = + const validateRuntime = options.validate ?? Effect.fn("AntigravityInstallation.validate")( function* (executable: AntigravityExecutable, expectedVersion: string) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const validationScope = yield* Effect.scope; const profileDirectory = yield* Effect.acquireRelease( fs.makeTempDirectory({ prefix: "t3-antigravity-validate-" }), @@ -527,7 +533,6 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma }, Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.provideService(SecureRandom.SecureRandom, random), Effect.mapError((cause) => { if (isInstallationError(cause)) return cause; @@ -541,6 +546,43 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma }), ); + const validate = Effect.fn("AntigravityInstallation.verify")(function* ( + executable: AntigravityExecutable, + expectedVersion: string, + ) { + const capture = yield* makeInstallationDiagnostics("Antigravity initialization").pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(HostProcess.Environment, environment), + Effect.provideService(HostProcess.HomeDirectory, homeDirectory), + ); + return yield* validateRuntime(executable, expectedVersion).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, capture.spawner), + Effect.scoped, + withValidationTimeout, + Effect.mapError( + (error) => + new AntigravityInstallationError({ + ...error, + diagnostics: { + ...capture.snapshot(isAcpProcessExitedError(error.cause) ? error.cause : undefined), + ...(error === validationTimeoutError ? { exitCode: null, signal: null } : {}), + }, + }), + ), + Effect.tapError((error) => + SubscriptionRef.get(state).pipe( + Effect.flatMap(({ operationId }) => + Effect.logWarning("Managed runtime verification failed", { + driver: DRIVER, + operationId, + diagnostics: error.diagnostics, + }), + ), + ), + ), + ); + }); + const install = Effect.fn("AntigravityInstallation.install")( function* (asset: AntigravityReleaseAsset | AntigravityCliReleaseAsset) { yield* ensureEligible(asset.version); @@ -616,7 +658,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma ); } yield* report("verifying", "Checking the installed runtime."); - yield* validate(existing, asset.version).pipe(Effect.scoped, withValidationTimeout); + yield* validate(existing, asset.version); yield* activate(); return; } @@ -873,7 +915,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma managedVersionDirectory: pairDirectory, }, asset.version, - ).pipe(Effect.scoped, withValidationTimeout); + ); const record: InstalledRelease = { releaseId, version: asset.version, @@ -896,7 +938,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma completedRelease(releaseId).pipe( Effect.flatMap((existing) => existing.version === asset.version - ? validate(existing, asset.version).pipe(Effect.scoped, withValidationTimeout) + ? validate(existing, asset.version) : Effect.fail( installationError( "activate", @@ -1014,6 +1056,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma version: reuse?.version ?? asset?.version ?? null, installedVersion: current.installedVersion, canRemove: current.canRemove, + diagnostics: null, message: reuse === undefined ? "Downloading Google's official Antigravity runtime." @@ -1024,8 +1067,6 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma reuse === undefined ? install(asset!) : validate(reuse, reuse.version!).pipe( - Effect.scoped, - withValidationTimeout, Effect.andThen(ensureEligible(reuse.version!)), Effect.andThen( SubscriptionRef.update(state, (value) => ({ @@ -1050,6 +1091,8 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma return { ...value, phase: cancelled ? "cancelled" : "failed", + diagnostics: + cancelled || Option.isNone(error) ? null : (error.value.diagnostics ?? null), message: cancelled ? "Installation cancelled. The previous runtime is unchanged." : Option.isSome(error) @@ -1134,6 +1177,7 @@ export const makeAntigravityInstallation = Effect.fn("AntigravityInstallation.ma installedVersion: null, canRemove: false, message: null, + diagnostics: null, }) satisfies ProviderInstallState, ); }).pipe(Effect.uninterruptible), diff --git a/apps/server/src/provider/CodexInstallation.test.ts b/apps/server/src/provider/CodexInstallation.test.ts index 59f19b822012c..2ff7462644a93 100644 --- a/apps/server/src/provider/CodexInstallation.test.ts +++ b/apps/server/src/provider/CodexInstallation.test.ts @@ -14,6 +14,7 @@ import * as PlatformError from "effect/PlatformError"; import * as Exit from "effect/Exit"; import * as Queue from "effect/Queue"; import * as Scope from "effect/Scope"; +import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http"; import * as TestClock from "effect/testing/TestClock"; @@ -40,6 +41,7 @@ const makeHarness = Effect.fn("test.makeCodexInstallation")(function* ( local?: { version: string; appServerFails?: boolean; versionFails?: boolean }; platform?: NodeJS.Platform; spawner?: ChildProcessSpawner.ChildProcessSpawner["Service"]; + useDefaultValidation?: boolean; runtimeRenameFailures?: number; /** Served by URL instead of the pinned package. */ responses?: Readonly Response>>; @@ -89,7 +91,7 @@ const makeHarness = Effect.fn("test.makeCodexInstallation")(function* ( const installation = yield* CodexInstallation.makeCodexInstallation({ baseDir, releaseAsset: { ...asset, sha256 }, - validate: () => Effect.void, + ...(input.useDefaultValidation ? {} : { validate: () => Effect.void }), ...input.options, }).pipe( Effect.provideService(ModelManifest.ModelManifest, { @@ -461,9 +463,13 @@ it.effect( }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); -it.effect.each(["cached update", "reverify"] as const)( - "reports a verification timeout during %s and preserves the managed runtime", - (operation) => +it.effect.each([ + { operation: "cached update", exits: false }, + { operation: "reverify", exits: false }, + { operation: "cached update", exits: true }, +])( + "reports verification diagnostics during $operation (exited: $exits) and preserves the managed runtime", + ({ operation, exits }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-codex-verify-test-" }); @@ -490,9 +496,34 @@ it.effect.each(["cached update", "reverify"] as const)( const pointer = JSON.stringify({ version: installedVersion }); yield* fs.writeFileString(`${managed}/active.json`, pointer); const entered = yield* Deferred.make(); + const spawner = ChildProcessSpawner.make(() => + Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: exits + ? Deferred.await(entered).pipe(Effect.as(ChildProcessSpawner.ExitCode(3))) + : Effect.never, + isRunning: Effect.succeed(true), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.empty, + stderr: Stream.succeed( + new TextEncoder().encode( + "Startup library missing\nAuthorization: Bearer private-fixture-token\n", + ), + ).pipe(Stream.tap(() => Deferred.succeed(entered, undefined))), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ), + ); const h = yield* makeHarness({ baseDir, platform: "win32", + spawner, + useDefaultValidation: true, manifestCurrent: Effect.succeed({ ...ModelManifest.BUNDLED_MODEL_MANIFEST, compatibility: [ @@ -505,21 +536,33 @@ it.effect.each(["cached update", "reverify"] as const)( }), options: { releaseAsset: operation === "reverify" ? null : { ...asset, target, sha256: checksum }, - validate: () => Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)), }, }); yield* h.installation.start; yield* Deferred.await(entered); - yield* TestClock.adjust("90 seconds"); + if (!exits) yield* TestClock.adjust("90 seconds"); expect(yield* terminalState(h.installation)).toMatchObject({ phase: "failed", installedVersion, - message: - "Codex runtime verification timed out after 90 seconds. Check this environment's server logs for startup errors, then retry installation.", + message: exits + ? "Could not unpack or start the downloaded Codex package." + : expect.stringContaining("Codex runtime verification timed out after 90 seconds."), + diagnostics: { + step: "Codex version check", + stderr: "Startup library missing\nAuthorization: [redacted]", + exitCode: exits ? 3 : null, + signal: null, + truncated: false, + }, }); expect(yield* fs.readFileString(`${managed}/active.json`)).toBe(pointer); expect((yield* h.installation.resolve()).version).toBe(installedVersion); expect(h.downloads()).toBe(0); + if (!exits) { + expect((yield* h.installation.start).diagnostics).toBeNull(); + const running = yield* h.installation.state; + expect((yield* h.installation.cancel(running.operationId!)).diagnostics).toBeNull(); + } }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); diff --git a/apps/server/src/provider/CodexInstallation.ts b/apps/server/src/provider/CodexInstallation.ts index 7a9c80ebbf007..e98fab166b981 100644 --- a/apps/server/src/provider/CodexInstallation.ts +++ b/apps/server/src/provider/CodexInstallation.ts @@ -1,6 +1,7 @@ // @effect-diagnostics nodeBuiltinImport:off - Effect has no incremental digest. import { ProviderDriverKind, + ProviderInstallDiagnostics, type ProviderInstallState, type ServerProviderCompatibilityAdvisory, } from "@t3tools/contracts"; @@ -32,6 +33,7 @@ import * as ServerConfig from "../config.ts"; import { selectManagedRelease } from "./managedRelease.ts"; import * as ModelManifest from "./ModelManifest.ts"; import { resolveProviderCompatibility } from "./providerCompatibility.ts"; +import { makeInstallationDiagnostics } from "./installationDiagnostics.ts"; const DRIVER = ProviderDriverKind.make("codex"); const STAGING_STALE_AFTER_MS = 24 * 60 * 60_000; @@ -117,6 +119,7 @@ export class CodexInstallationError extends Schema.TaggedError, + commandSpawner: ChildProcessSpawner.ChildProcessSpawner["Service"] = spawner, ) { const resolved = yield* resolveSpawnCommand(command, args).pipe( Effect.provideService(HostProcess.Platform, platform), ); - const child = yield* spawner.spawn( + const child = yield* commandSpawner.spawn( ChildProcess.make(resolved.command, resolved.args, { shell: resolved.shell }), ); const [output, , exitCode] = yield* Effect.all( @@ -539,25 +544,32 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function downloadedBytes: 0, totalBytes: null, message: null, + diagnostics: null, }) satisfies ProviderInstallState, ); return true; }); + const validationTimeoutError = installationError( + "verify", + "Codex runtime verification timed out after 90 seconds. Review Installation details or this environment's server log, then retry installation.", + ); const withValidationTimeout = Effect.timeoutOrElse({ duration: "90 seconds", - orElse: () => - Effect.fail( - installationError( - "verify", - "Codex runtime verification timed out after 90 seconds. Check this environment's server logs for startup errors, then retry installation.", - ), - ), + orElse: () => Effect.fail(validationTimeoutError), }); - const validate = + const validateRuntime: ( + executable: CodexExecutable, + version: string, + ) => Effect.Effect< + void, + CodexInstallationError, + Scope.Scope | ChildProcessSpawner.ChildProcessSpawner + > = options.validate ?? Effect.fn("CodexInstallation.validate")( function* (executable: CodexExecutable, version: string) { - const output = yield* runCommand(executable.executablePath, ["--version"]); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* runCommand(executable.executablePath, ["--version"], spawner); if (output.trim() !== `codex-cli ${version}`) return yield* installationError( "verify", @@ -566,6 +578,43 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function }, Effect.mapError(wrapFailure("verify", "The downloaded Codex runtime could not start.")), ); + const validate = Effect.fn("CodexInstallation.verify")(function* ( + executable: CodexExecutable, + version: string, + ) { + const capture = yield* makeInstallationDiagnostics("Codex version check").pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(HostProcess.Environment, environment), + Effect.provideService(HostProcess.HomeDirectory, homeDirectory), + ); + return yield* validateRuntime(executable, version).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, capture.spawner), + Effect.scoped, + withValidationTimeout, + Effect.mapError( + (error) => + new CodexInstallationError({ + ...error, + diagnostics: { + ...capture.snapshot(), + ...(error === validationTimeoutError ? { exitCode: null, signal: null } : {}), + }, + }), + ), + Effect.tapError((error) => + SubscriptionRef.get(state).pipe( + Effect.flatMap(({ operationId }) => + Effect.logWarning("Managed runtime verification failed", { + driver: DRIVER, + operationId, + diagnostics: error.diagnostics, + }), + ), + ), + ), + ); + }); + const install = Effect.fn("CodexInstallation.install")( function* (release: CodexReleaseAsset) { const ensureEligible = Effect.fn("CodexInstallation.ensureEligible")(function* () { @@ -623,7 +672,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function "The existing managed Codex release differs from the official package. Remove it and reinstall.", ); yield* ensureEligible(); - yield* validate(existing, release.version).pipe(Effect.scoped, withValidationTimeout); + yield* validate(existing, release.version); yield* activate(); return; } @@ -717,7 +766,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function ); const executable = yield* fromDirectory(runtime, release.version, release.target); yield* ensureEligible(); - yield* validate(executable, release.version).pipe(Effect.scoped, withValidationTimeout); + yield* validate(executable, release.version); yield* fs.writeFileString( path.join(runtime, ".install-complete.json"), yield* encodeRecord({ @@ -762,7 +811,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function function* () { const active = yield* readRecord(activePath, ActiveRelease); const executable = yield* completedRelease(active.version); - yield* validate(executable, executable.version).pipe(Effect.scoped, withValidationTimeout); + yield* validate(executable, executable.version); const kept = yield* keptInstall(executable.version); if (kept === null) { yield* SubscriptionRef.update(state, (current) => ({ ...current, ...offer(kept) })); @@ -814,6 +863,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function downloadedBytes: 0, ...offer(installedVersion), message: release ? "Downloading Codex." : "Checking Codex.", + diagnostics: null, }; yield* SubscriptionRef.set(state, next); const work = (release ? install(release) : reverify()).pipe( @@ -827,6 +877,8 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function return { ...value, phase: cancelled ? "cancelled" : "failed", + diagnostics: + cancelled || Option.isNone(error) ? null : (error.value.diagnostics ?? null), message: cancelled ? "Installation cancelled. The previous runtime is unchanged." : Option.isSome(error) @@ -905,6 +957,7 @@ export const makeCodexInstallation = Effect.fn("makeCodexInstallation")(function source: null, canRemove: false, message: null, + diagnostics: null, }) satisfies ProviderInstallState, ); yield* reuseLocal(); diff --git a/apps/server/src/provider/installationDiagnostics.test.ts b/apps/server/src/provider/installationDiagnostics.test.ts new file mode 100644 index 0000000000000..eecb2abb89f63 --- /dev/null +++ b/apps/server/src/provider/installationDiagnostics.test.ts @@ -0,0 +1,135 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as PlatformError from "effect/PlatformError"; +import * as Result from "effect/Result"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { makeInstallationDiagnostics } from "./installationDiagnostics.ts"; + +it.effect("bounds diagnostic retention while draining and redacting split startup output", () => + Effect.gen(function* () { + const footer = new TextEncoder().encode( + "\u001b[31mStartup failed: café\u001b[0m\nC:\\Users\\Private\\runtime\nc:/users/private/runtime\n" + + "Authorization: Bearer secret-fixture\npassword=private-value\n" + + "opaque-environment-secret\nC:\\Users\\Private\\credential-file\nhttps://example.test/auth?code=private-code#private-fragment\n" + + 'Authorization: Token alternate-secret\nsecret="unfinished secret value', + ); + const chunks = [ + new TextEncoder().encode("earlier log\n".repeat(500)), + new TextEncoder().encode("x".repeat(100_000) + "\n"), + ...Array.from(footer, (byte) => new Uint8Array([byte])), + ]; + let drained = 0; + const underlying = ChildProcessSpawner.make(() => + Effect.succeed( + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(3)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + stdin: Sink.drain, + stdout: Stream.empty, + stderr: Stream.fromIterable(chunks).pipe( + Stream.tap(() => + Effect.sync(() => { + drained++; + }), + ), + ), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }), + ), + ); + const capture = yield* makeInstallationDiagnostics("Runtime check").pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, underlying), + Effect.provideService(HostProcess.HomeDirectory, "C:\\Users\\Private"), + Effect.provideService(HostProcess.Environment, { + API_KEY: "opaque-environment-secret", + SECRET_PATH: "C:\\Users\\Private\\credential-file", + }), + ); + const child = yield* capture.spawner.spawn(ChildProcess.make("fixture")); + yield* child.stderr.pipe(Stream.runDrain); + yield* child.exitCode; + const diagnostics = capture.snapshot(); + expect(drained).toBe(chunks.length); + expect(diagnostics.stderr.length).toBeLessThanOrEqual(2_000); + expect(diagnostics).toMatchObject({ step: "Runtime check", exitCode: 3, truncated: true }); + expect(diagnostics.stderr).toContain("Startup failed: café"); + expect(diagnostics.stderr).toContain("~\\runtime"); + expect(diagnostics.stderr).toContain("[oversized diagnostic line omitted]"); + for (const privateText of [ + "secret-fixture", + "private-value", + "opaque-environment-secret", + "private-code", + "Private", + "c:/users/private", + "credential-file", + "\u001b", + "alternate-secret", + "unfinished secret value", + ]) { + expect(diagnostics.stderr).not.toContain(privateText); + } + }), +); + +it.effect.each(["NotFound", "PermissionDenied"] as const)( + "reports a %s spawn failure even when no startup stream exists", + (reason) => + Effect.gen(function* () { + const spawner = ChildProcessSpawner.make(() => + Effect.fail( + PlatformError.systemError({ + _tag: reason, + module: "ChildProcess", + method: "spawn", + }), + ), + ); + const capture = yield* makeInstallationDiagnostics("Runtime check").pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const result = yield* capture.spawner.spawn(ChildProcess.make("fixture")).pipe(Effect.result); + expect(Result.isFailure(result)).toBe(true); + expect(capture.snapshot()).toMatchObject({ + stderr: "", + exitCode: null, + systemError: + reason === "NotFound" + ? "The runtime executable could not be found." + : "The operating system denied permission to start the runtime.", + }); + }), +); + +it.live("captures a real failed child process without mixing stdout into diagnostics", () => + Effect.gen(function* () { + const executable = yield* HostProcess.ExecutablePath; + const capture = yield* makeInstallationDiagnostics("Runtime check"); + const child = yield* capture.spawner.spawn( + ChildProcess.make(executable, [ + "-e", + "process.stdout.write('private-protocol-payload'); process.stderr.write('Missing startup library\\n'); process.exitCode = 3;", + ]), + ); + yield* Effect.all( + [child.stdout.pipe(Stream.runDrain), child.stderr.pipe(Stream.runDrain), child.exitCode], + { concurrency: "unbounded" }, + ); + expect(capture.snapshot()).toEqual({ + step: "Runtime check", + stderr: "Missing startup library", + exitCode: 3, + signal: null, + truncated: false, + }); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/provider/installationDiagnostics.ts b/apps/server/src/provider/installationDiagnostics.ts new file mode 100644 index 0000000000000..9a31f53beffce --- /dev/null +++ b/apps/server/src/provider/installationDiagnostics.ts @@ -0,0 +1,133 @@ +// @effect-diagnostics nodeBuiltinImport:off -- terminal escape sequences must not reach setup details. +import type { ProviderInstallDiagnostics } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { sanitizeAcpStderrExcerpt } from "@t3tools/provider-acp/server/stderr"; +import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as NodeUtil from "node:util"; + +const MAX_DIAGNOSTIC_CHARS = 2_000; +const MAX_LINE_CHARS = 8_192; + +export const makeInstallationDiagnostics = Effect.fn("InstallationDiagnostics.make")(function* ( + step: string, +) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const environment = yield* HostProcess.Environment; + const homeDirectory = yield* HostProcess.HomeDirectory; + const homePatterns = [...new Set([homeDirectory, environment.HOME, environment.USERPROFILE])] + .filter((home): home is string => typeof home === "string" && home.length > 1) + .map( + (home) => + new RegExp( + home + .split(/[\\/]/u) + .map((part) => part.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")) + .join("[\\\\/]"), + "giu", + ), + ); + const secrets = Object.entries(environment).flatMap(([name, value]) => + /token|password|secret|credential|api[_-]?key|authorization/i.test(name) && value + ? value.split(/\r?\n/u).filter(Boolean) + : [], + ); + const sanitize = (text: string) => { + let result = text; + for (const secret of secrets) result = result.split(secret).join("[redacted]"); + result = NodeUtil.stripVTControlCharacters(result).replaceAll("\r", ""); + for (const pattern of homePatterns) result = result.replace(pattern, "~"); + return sanitizeAcpStderrExcerpt(result, homeDirectory, environment) + .replace(/\bauthorization["']?\s*[:=].*/giu, "Authorization: [redacted]") + .replace(/https?:\/\/[^\s"'<>]+/giu, "[url redacted]") + .replace( + /\b(access[_-]?token|refresh[_-]?token|id[_-]?token|api[_-]?key|password|secret|token)(["']?\s*[:=]\s*)(?:"[^"]*(?:"|$)|'[^']*(?:'|$)|[^\s,;]+)/giu, + "$1$2[redacted]", + ); + }; + let tail = ""; + let pending = ""; + let skippingLine = false; + let truncated = false; + let exitCode: number | null = null; + let systemError: string | undefined; + const append = (text: string) => { + const lines = text.split("\n"); + for (let index = 0; index < lines.length; index++) { + const part = lines[index]!; + if (!skippingLine) { + if (pending.length + part.length > MAX_LINE_CHARS) { + pending = ""; + skippingLine = true; + truncated = true; + } else pending += part; + } + if (index < lines.length - 1) { + const line = skippingLine ? "[oversized diagnostic line omitted]" : sanitize(pending); + tail += `${line}\n`; + if (tail.length > MAX_DIAGNOSTIC_CHARS) { + tail = tail.slice(-MAX_DIAGNOSTIC_CHARS); + truncated = true; + } + pending = ""; + skippingLine = false; + } + } + }; + return { + spawner: ChildProcessSpawner.make( + Effect.fn("InstallationDiagnostics.spawn")(function* (command) { + exitCode = null; + systemError = undefined; + const child = yield* spawner.spawn(command).pipe( + Effect.tapError((error) => + Effect.sync(() => { + systemError = + error.reason._tag === "NotFound" + ? "The runtime executable could not be found." + : error.reason._tag === "PermissionDenied" + ? "The operating system denied permission to start the runtime." + : `The operating system could not start the runtime (${error.reason._tag}).`; + }), + ), + ); + const decoder = new TextDecoder(); + return ChildProcessSpawner.makeHandle({ + ...child, + stderr: child.stderr.pipe( + Stream.tap((chunk) => + Effect.sync(() => append(decoder.decode(chunk, { stream: true }))), + ), + Stream.ensuring(Effect.sync(() => append(decoder.decode()))), + ), + exitCode: child.exitCode.pipe( + Effect.tap((code) => + Effect.sync(() => { + exitCode = Number(code); + }), + ), + ), + }); + }), + ), + snapshot: (exit?: { + readonly code?: number | undefined; + readonly signal?: string | undefined; + }): ProviderInstallDiagnostics => { + const lastLine = skippingLine ? "[oversized diagnostic line omitted]" : sanitize(pending); + const text = `${tail}${lastLine}`.trim(); + return { + step, + stderr: text.slice(-MAX_DIAGNOSTIC_CHARS), + exitCode: exit?.code ?? exitCode, + signal: + exit?.signal && exit.signal.length <= 32 && /^SIG[A-Z0-9]+$/u.test(exit.signal) + ? exit.signal + : null, + ...(systemError === undefined ? {} : { systemError }), + truncated: truncated || text.length > MAX_DIAGNOSTIC_CHARS, + }; + }, + }; +}); diff --git a/apps/web/src/components/settings/CodexSetupSection.tsx b/apps/web/src/components/settings/CodexSetupSection.tsx index a09b1c0b25d8c..4960844aa4b45 100644 --- a/apps/web/src/components/settings/CodexSetupSection.tsx +++ b/apps/web/src/components/settings/CodexSetupSection.tsx @@ -41,6 +41,7 @@ import { waitForQuerySettled } from "../../state/waitForQuerySettled"; import { Button } from "../ui/button"; import { ChatGptConnectionButton } from "./ChatGptConnectionButton"; import { ChatGptUsageButton } from "./ChatGptUsageButton"; +import { ProviderInstallationDetails } from "./ProviderInstallationDetails"; import { ChatGptAccountPicker } from "./ChatGptAccountPicker"; import { Input } from "../ui/input"; import { OpenAI } from "../Icons"; @@ -929,6 +930,12 @@ function ManagedCodexSetup({ {setupError}

) : null} + {installation?.phase === "failed" ? ( + + ) : null} ); @@ -1074,6 +1081,14 @@ function ManagedCodexSetup({ : "Could not read Codex setup status. Reconnect and try again.")}

) : null} + {installation?.phase === "failed" ? ( +
+ +
+ ) : null} ); } diff --git a/apps/web/src/components/settings/ProviderInstallationDetails.tsx b/apps/web/src/components/settings/ProviderInstallationDetails.tsx new file mode 100644 index 0000000000000..7c98ac27645fc --- /dev/null +++ b/apps/web/src/components/settings/ProviderInstallationDetails.tsx @@ -0,0 +1,27 @@ +import type { ProviderInstallDiagnostics } from "@t3tools/contracts"; + +export function ProviderInstallationDetails({ + diagnostics, + operationId, +}: { + readonly diagnostics: ProviderInstallDiagnostics | null | undefined; + readonly operationId: string | null; +}) { + if (!diagnostics) return null; + return ( +
+ Installation details +
+

{diagnostics.step}

+ {operationId ?

Attempt: {operationId}

: null} + {diagnostics.systemError ?

{diagnostics.systemError}

: null} + {diagnostics.exitCode !== null ?

Exit code: {diagnostics.exitCode}

: null} + {diagnostics.signal !== null ?

Exit signal: {diagnostics.signal}

: null} +
+          {diagnostics.stderr || "No startup error output was captured for this check."}
+        
+ {diagnostics.truncated ?

Some startup output was omitted or truncated.

: null} +
+
+ ); +} diff --git a/apps/web/src/components/settings/ProviderSetupSection.browser.test.tsx b/apps/web/src/components/settings/ProviderSetupSection.browser.test.tsx index e9404773f731f..f68d236212e89 100644 --- a/apps/web/src/components/settings/ProviderSetupSection.browser.test.tsx +++ b/apps/web/src/components/settings/ProviderSetupSection.browser.test.tsx @@ -181,6 +181,28 @@ function resetSetup() { } describe("Antigravity setup", () => { + it("shows the failed environment's startup details and clears them on a new attempt", async () => { + resetSetup(); + setup.installation = { + ...setup.installation!, + phase: "failed", + message: "Runtime verification timed out.", + diagnostics: { + step: "Antigravity initialization", + stderr: "Local harness failed to initialize", + exitCode: null, + signal: null, + truncated: false, + }, + }; + await using screen = await renderSetup(); + await userEvent.click(screen.getByText("Installation details")); + await expect.element(screen.getByText("Local harness failed to initialize")).toBeVisible(); + setup.installation = { ...setup.installation!, phase: "verifying", diagnostics: null }; + await publish(); + await expect.element(screen.getByText("Installation details")).not.toBeInTheDocument(); + }); + it("can start installation before the first install snapshot arrives", async () => { resetSetup(); setup.auth = authState({ phase: "idle", flowId: null, authorizationUrl: null }); diff --git a/apps/web/src/components/settings/ProviderSetupSection.tsx b/apps/web/src/components/settings/ProviderSetupSection.tsx index 03591b64ca265..3de6b888a70ed 100644 --- a/apps/web/src/components/settings/ProviderSetupSection.tsx +++ b/apps/web/src/components/settings/ProviderSetupSection.tsx @@ -22,6 +22,7 @@ import { Button } from "../ui/button"; import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; import { SettingsRow } from "./settingsLayout"; import { ProviderAuthenticationSection } from "./ProviderAuthenticationSection"; +import { ProviderInstallationDetails } from "./ProviderInstallationDetails"; interface ProviderSetupSectionProps { readonly environmentId: EnvironmentId; @@ -238,6 +239,12 @@ function ProviderSetupActions({ {installation.message}

) : null} + {installation?.phase === "failed" ? ( + + ) : null}
{installActive && installation.operationId ? ( diff --git a/docs/internals/effect-services.md b/docs/internals/effect-services.md index 13e05dc926f67..6d510ed740f2b 100644 --- a/docs/internals/effect-services.md +++ b/docs/internals/effect-services.md @@ -135,7 +135,10 @@ optional-service layer. Don't route around the layer with an imperative runtime. lives only in `cause`. Expose a category, length, count, or a URL's protocol and host instead. `DeviceBootError.diagnostics` is a deliberate exception: authorized device users and agents need the device hub's launch command and failure text to diagnose host setup. It is capped at - 2,000 characters and included in the error message; other service errors keep the rule above. + 2,000 characters and included in the error message. `ProviderInstallDiagnostics` is another + deliberate exception: installation failures return at most 2,000 characters of redacted startup + stderr to authorized setup clients, captured from that validation attempt rather than server-wide logs. + Other service errors keep the rule above. - **Translation.** Construct the error where the failure happens, and map it to a transport error only in the transport. A translation boundary passes through domain errors already in the target channel and wraps only unknown or lower-level failures. Map each failure where its context is diff --git a/docs/user/providers-antigravity.md b/docs/user/providers-antigravity.md index 8a284a11e782e..44c74e2d62416 100644 --- a/docs/user/providers-antigravity.md +++ b/docs/user/providers-antigravity.md @@ -156,6 +156,11 @@ T3 does not support Antigravity goal status, pause, resume, or clear commands. ## Check access and troubleshoot +If runtime installation fails, expand **Installation details** in provider settings +to see the failed check, any reported exit status, and redacted startup output from +that environment. Mobile **Settings > Providers** can also show a failed installation's +details. Retry clears the previous attempt's output. + Open **Usage > Limits** to see the remaining Gemini and shared Claude/GPT allowances for Google sign-ins. Each group has separate five-hour and weekly limits and reset times. Limits refresh with provider status and include usage outside T3 Code. diff --git a/docs/user/providers-codex.md b/docs/user/providers-codex.md index f77dff53dabbe..6b1daa3e009db 100644 --- a/docs/user/providers-codex.md +++ b/docs/user/providers-codex.md @@ -137,6 +137,12 @@ over to new threads. Custom models do not offer it. If Codex rejects a turn because your login lacks the program, switch Cyber Access back to Standard or sign in to Codex with an account that has it. +## Troubleshoot runtime installation + +If the managed runtime fails to install, open **Installation details** in Codex setup +to see the failed check, any reported exit status, and redacted startup output from +the environment running Codex. The same failure is visible in mobile **Settings > Providers**. + ## Codex says I hit a usage limit When Codex stops on a usage limit, the thread names the window that ran out and diff --git a/packages/contracts/src/providerSetup.test.ts b/packages/contracts/src/providerSetup.test.ts index 1fc3046fb047b..f2b80c5b1ffba 100644 --- a/packages/contracts/src/providerSetup.test.ts +++ b/packages/contracts/src/providerSetup.test.ts @@ -1,9 +1,25 @@ import { describe, expect, it } from "vite-plus/test"; import * as Schema from "effect/Schema"; -import { ProviderAuthState } from "./providerSetup.ts"; +import { ProviderAuthState, ProviderInstallState } from "./providerSetup.ts"; const decodeState = Schema.decodeUnknownSync(ProviderAuthState); +const decodeInstallState = Schema.decodeUnknownSync(ProviderInstallState); + +it("accepts installation state from servers that predate startup diagnostics", () => { + const state = { + driver: "codex", + operationId: "attempt", + phase: "failed", + downloadedBytes: 0, + totalBytes: null, + version: null, + installedVersion: null, + canRemove: false, + message: "Verification failed.", + }; + expect(decodeInstallState(state)).toEqual(state); +}); describe("provider auth state", () => { it("drops auth variants from newer servers instead of rejecting the state", () => { diff --git a/packages/contracts/src/providerSetup.ts b/packages/contracts/src/providerSetup.ts index 2693b8aa7a918..adb2cfb4eb164 100644 --- a/packages/contracts/src/providerSetup.ts +++ b/packages/contracts/src/providerSetup.ts @@ -151,6 +151,18 @@ export type ProviderAuthCancelInput = typeof ProviderAuthCancelInput.Type; const ByteCount = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)); +export const ProviderInstallDiagnostics = Schema.Struct({ + step: TrimmedNonEmptyString.check(Schema.isMaxLength(128)), + stderr: Schema.String.check(Schema.isMaxLength(2_000)), + exitCode: Schema.NullOr(Schema.Int), + signal: Schema.NullOr( + Schema.String.check(Schema.isMaxLength(32), Schema.isPattern(/^SIG[A-Z0-9]+$/u)), + ), + systemError: Schema.optionalKey(Schema.String.check(Schema.isMaxLength(128))), + truncated: Schema.Boolean, +}); +export type ProviderInstallDiagnostics = typeof ProviderInstallDiagnostics.Type; + export const ProviderInstallState = Schema.Struct({ driver: ProviderDriverKind, operationId: Schema.NullOr(SetupOperationId), @@ -171,6 +183,7 @@ export const ProviderInstallState = Schema.Struct({ source: Schema.optionalKey(Schema.NullOr(Schema.Literals(["managed", "local"]))), canRemove: Schema.Boolean, message: Schema.NullOr(Schema.String), + diagnostics: Schema.optionalKey(Schema.NullOr(ProviderInstallDiagnostics)), }); export type ProviderInstallState = typeof ProviderInstallState.Type;