diff --git a/apps/electron/e2e/render-run-info.ts b/apps/electron/e2e/render-run-info.ts new file mode 100644 index 00000000..32951dcf --- /dev/null +++ b/apps/electron/e2e/render-run-info.ts @@ -0,0 +1,164 @@ +/** + * Render the real Run Info view (#21) from the live-acceptance artifacts. + * + * Uses the shipped components — `ManifestView` and `DiffView` from + * `packages/ui/src/components/kernel/RunInfoPanel.tsx` — so the artifact is the + * view the app actually shows, not a re-implementation. Styling comes from the + * repository's own design tokens, parsed out of the renderer stylesheet, so the + * colors are the app's, not invented. + * + * Usage (after `run-manifest-acceptance.ts` has written its artifacts): + * bun apps/electron/e2e/render-run-info.ts + */ +import { readFile, writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import React from 'react'; +import { renderToStaticMarkup } from 'react-dom/server'; +import type { RunManifest, RunManifestDiff } from '@finagent/core'; +import { DiffView, ManifestView } from '../../../packages/ui/src/components/kernel/RunInfoPanel'; + +const repoRoot = resolve(import.meta.dir, '../../..'); +const dir = resolve( + process.env.FINAGENT_MANIFEST_OUTPUT ?? join(repoRoot, 'apps/electron/e2e/artifacts/run-manifest-acceptance') +); + +const read = async (name: string) => JSON.parse(await readFile(join(dir, name), 'utf8')); + +/** Pull the light-theme design tokens straight out of the renderer stylesheet. */ +async function designTokens(): Promise> { + const css = await readFile( + join(repoRoot, 'apps/electron/src/renderer/styles/index.css'), + 'utf8' + ); + const root = /:root\s*\{([\s\S]*?)\n\}/.exec(css)?.[1] ?? ''; + const tokens: Record = {}; + for (const [, name, value] of root.matchAll(/--([a-z-]+):\s*([^;]+);/g)) { + tokens[name] = value.trim(); + } + return tokens; +} + +function shell(title: string, tokens: Record, body: string): string { + return ` + + + +${title} + + + + + + +${body} + + +`; +} + +const tokens = await designTokens(); +const manifestA = (await read('run-a-manifest.json')) as RunManifest; +const manifestB = (await read('run-b-manifest.json')) as RunManifest; +const diff = (await read('manifest-diff.json')) as RunManifestDiff; +const restart = await read('run-info-after-restart.json'); +const restartManifest = restart.manifest as RunManifest; + +const card = (head: string, sub: string, node: React.ReactElement) => + React.createElement( + 'div', + { className: 'wb-card' }, + React.createElement('div', { className: 'wb-head' }, head), + React.createElement('div', { className: 'wb-sub' }, sub), + node + ); + +// 1. Run Info for the historical run, read back in a fresh process. +const afterRestart = renderToStaticMarkup( + React.createElement( + 'div', + null, + React.createElement( + 'h1', + { style: { fontSize: 18, fontWeight: 600, marginBottom: 6 } }, + 'Run Info — 应用重启后读取历史 run(#21)' + ), + React.createElement( + 'p', + { style: { fontSize: 12, marginBottom: 16, color: tokens['foreground-muted'] } }, + `读取进程的「当前全局设置」model = ${restart.currentGlobalModel};` + + `历史 run 的 manifest model = ${restartManifest.model}(仍为诞生时的配置)。` + ), + card( + `Run A · ${manifestA.runId}`, + 'Run Info 视图(ManifestView)— 重启后读到的仍是该 run 自己的 manifest', + React.createElement(ManifestView, { manifest: restartManifest }) + ) + ) +); +await writeFile(join(dir, 'run-info-after-restart.html'), shell('Run Info — 重启后', tokens, afterRestart)); + +// 2. Both runs side by side, plus the diff the compare view shows. +const compare = renderToStaticMarkup( + React.createElement( + 'div', + null, + React.createElement( + 'h1', + { style: { fontSize: 18, fontWeight: 600, marginBottom: 6 } }, + '两次真实 Deep Research run 的 manifest 与 diff(#21)' + ), + React.createElement( + 'p', + { style: { fontSize: 12, marginBottom: 16, color: tokens['foreground-muted'] } }, + '刻意改动 model 与预算(#17);diff 应精确只反映这两处变化,prompt 未变则不报差异。' + ), + card(`Run A · ${manifestA.runId}`, `model = ${manifestA.model}`, React.createElement(ManifestView, { manifest: manifestA })), + card(`Run B · ${manifestB.runId}`, `model = ${manifestB.model}`, React.createElement(ManifestView, { manifest: manifestB })), + card( + 'Run 对比(DiffView)', + `${diff.fields.length} 处差异 · groups=${JSON.stringify(diff.groups)}`, + React.createElement(DiffView, { diff }) + ) + ) +); +await writeFile(join(dir, 'run-info-compare.html'), shell('Run 对比', tokens, compare)); + +// 3. Plain-text rendering, for the report and for grep-ability. +const text = [ + 'Run Info — Run A (read back in a fresh process after "restart")', + ` runtime mode : ${restartManifest.runtimeMode}`, + ` provider : ${restartManifest.provider}`, + ` model : ${restartManifest.model}`, + ` strategy : ${restartManifest.strategy?.id} @ ${restartManifest.strategy?.version}`, + ` prompt : ${restartManifest.prompt?.version} (${restartManifest.prompt?.hash})`, + ` tools : ${restartManifest.tools.map((tool) => tool.name).join(', ')}`, + ` budget : effective=${JSON.stringify(restartManifest.budget?.effective)}`, + ` app version : ${restartManifest.appVersion}`, + ` git revision : ${restartManifest.gitRevision}`, + ` current globals in the reading process: model = ${restart.currentGlobalModel}`, + '', + 'Two-run diff', + ...diff.fields.map((field) => ` ${field.path}: ${JSON.stringify(field.before)} -> ${JSON.stringify(field.after)}`), + '', + `changed=${diff.changed} groups=${JSON.stringify(diff.groups)}`, +].join('\n'); +await writeFile(join(dir, 'run-info.txt'), text); + +console.log('Wrote run-info-after-restart.html, run-info-compare.html, run-info.txt'); +console.log(text); diff --git a/apps/electron/e2e/run-manifest-acceptance.ts b/apps/electron/e2e/run-manifest-acceptance.ts new file mode 100644 index 00000000..bf0731ef --- /dev/null +++ b/apps/electron/e2e/run-manifest-acceptance.ts @@ -0,0 +1,624 @@ +/** + * Live acceptance for #21 — immutable run manifests for Deep Research. + * + * Proves the two acceptance items that cannot be satisfied by fixtures: + * + * 1. Two REAL production Deep Research runs, with a deliberately changed + * config (model + budget), and a diff that reflects exactly that change. + * 2. The two manifests, the diff output, and a Run Info read-back AFTER a + * restart (a second, fresh process whose *current* globals differ) — the + * historical run must still report the config it was born with. + * + * Real components only, no fixtures: + * - real `AgentKernel` + `ResearchService` (the production research path); + * - real DeepSeek HTTP streaming — the configured model is the model asked for; + * - real retrieval over the public internet through the SHIPPED provider + * platform: the `YahooChartFinancialDataProvider` registered on the real + * `ProviderRouter`, its fetchers handed to `createFullRegistry` — the same + * wiring `AgentKernelHost` uses. Capability bodies (validation, + * sanitisation, summaries, provenance) run unchanged, and every result + * carries the provider that actually answered plus its `delayed` flag; + * - a second process for the restart read-back, so nothing is served from the + * first process's memory. + * + * Retrieval scope is exactly what the shipped Yahoo adapter supports: US + * delayed quotes, daily candles, and a thin company profile. It is a no-key, + * unofficial chart endpoint (personal use, may rate-limit). Everything else — + * broker positions, financials, news, options, intraday — has no provider on + * this machine and fails loudly through the router instead of being faked with + * zeros or empty lists. See `docs/yahoo-finance.md`. + * + * Usage (from the repository root, with DEEPSEEK_API_KEY in the environment): + * bun apps/electron/e2e/run-manifest-acceptance.ts + */ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { openSync, closeSync } from 'node:fs'; +import { resolve, join } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import type { AgentRuntime, AgentEvent, AgentRunInput, RunManifestContext } from '@finagent/core'; +import { AgentKernel } from '../../../packages/shared/src/kernel/agent-kernel'; +import { ResearchService } from '../../../packages/shared/src/research/service'; +import { ResearchReportRepository } from '../../../packages/shared/src/research/repository'; +import { JsonFileStore } from '../../../packages/shared/src/storage/json-file-store'; +import { createFullRegistry } from '../../../packages/shared/src/capabilities'; +import { parseSynthesisJson } from '../../../packages/shared/src/research/agent-synth'; +import { ProviderRouter, createRouterFetchers } from '../../../packages/shared/src/providers'; +import { YahooChartFinancialDataProvider } from '../../../packages/shared/src/providers/yahoo'; +import { YAHOO_PROVIDER_ID } from '../../../packages/shared/src/providers/yahoo/adapter'; +import { + diffRunManifests, + exportRunManifest, + hashManifestInput, +} from '../../../packages/shared/src/kernel/run-manifest'; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +const SYMBOL = process.env.FINAGENT_MANIFEST_SYMBOL ?? 'NVDA.US'; +const STRATEGY = 'comprehensive'; +const MAX_TOKENS = 8_000; + +/** The deliberate configuration change between the two runs. */ +const RUN_A = { label: 'run-a', model: 'deepseek-flash', budget: { modelCalls: 3 } }; +const RUN_B = { label: 'run-b', model: 'deepseek-v4-pro', budget: { modelCalls: 6 } }; +/** The "current globals" the restart read-back sees — deliberately != RUN_A. */ +const RESTART_CURRENT_MODEL = 'deepseek-v4-pro'; + +const PROMPT_VERSION = 'deep-research-synthesis@1'; +const PROMPT_SOURCE = 'e2e-acceptance'; + +const output = resolve( + process.env.FINAGENT_MANIFEST_OUTPUT ?? 'apps/electron/e2e/artifacts/run-manifest-acceptance' +); + +// --------------------------------------------------------------------------- +// Small file helpers +// --------------------------------------------------------------------------- + +const save = (name: string, value: unknown) => + writeFile(join(output, name), JSON.stringify(value, null, 2)); +const json = async (name: string) => JSON.parse(await readFile(join(output, name), 'utf8')); + +async function until(read: () => Promise, timeout = 420_000): Promise { + const end = Date.now() + timeout; + let last: unknown; + while (Date.now() < end) { + try { + const result = await read(); + if (result) return result; + } catch (error) { + last = error; + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + await Bun.sleep(50); + } + throw new Error(`Live acceptance timed out. Last error: ${String(last)}`); +} + +function launch(args: string[]) { + const log = openSync(join(output, 'worker.log'), 'a'); + try { + return spawn(process.execPath, [import.meta.path, ...args], { + stdio: ['ignore', log, log], + windowsHide: true, + env: process.env, + }); + } finally { + closeSync(log); + } +} + +// --------------------------------------------------------------------------- +// Real retrieval: the shipped Yahoo provider behind the real provider router +// --------------------------------------------------------------------------- + +/** + * Build the same retrieval stack `AgentKernelHost` builds: register the real + * `YahooChartFinancialDataProvider` on a real `ProviderRouter`, point the + * routing at it, and let `createRouterFetchers` produce the capability + * fetchers. No hand-rolled HTTP, no hand-written mapping: quote / kline / + * profile go through the router and the capability bodies, so the provider id, + * the timestamp and the `delayed` flag in every result are the ones that + * actually answered. + * + * Yahoo is the only market-data provider on this machine (the Longbridge CLI is + * not installed, there is no Massive key), so it is the whole chain. Its + * adapter covers US delayed quotes, daily candles and a thin company profile; + * every other capability raises `UNSUPPORTED_CAPABILITY` from the router and + * is reported as unavailable by the research run — never as zero or "no data". + */ +function buildRetrieval() { + const router = new ProviderRouter(); + router.register(new YahooChartFinancialDataProvider()); + router.setRouting({ primary: YAHOO_PROVIDER_ID }); + return { router, fetchers: createRouterFetchers(router) }; +} + +/** Routing as the manifest should record it: what the run actually used. */ +function retrievalManifest(routing: { primary?: string; fallback?: string }) { + return { + providerId: routing.primary, + routing: { + primary: routing.primary, + ...(routing.fallback ? { fallback: routing.fallback } : {}), + }, + configured: true, + endpoint: 'https://query1.finance.yahoo.com/v8/finance/chart/', + }; +} + + + +/** The prompt-composition fingerprint, recorded identically for both runs. */ +const PROMPT_HASH = hashManifestInput( + ['deep-research-synthesis', PROMPT_VERSION, STRATEGY, SYMBOL, 'sections=planned-keys'].join('|') +); + +// --------------------------------------------------------------------------- +// Shared worker wiring: real kernel + real research service + real manifest ctx +// --------------------------------------------------------------------------- + +interface WorkerOptions { + label: string; + /** The model this process *requests*; recorded in the manifest. */ + model: string; + budgetDefaults: Record; +} + +function buildStack(options: WorkerOptions) { + const { router, fetchers } = buildRetrieval(); + const routing = router.getRouting(); + const registry = createFullRegistry(fetchers); + const repository = new ResearchReportRepository(new JsonFileStore(join(output, 'store'))); + + const abort = new AbortController(); + let sequence = 0; + const event = (input: AgentRunInput, type: string, payload: unknown = {}) => + ({ + id: randomUUID(), + sessionId: input.sessionId, + runId: input.runId, + sequence: (sequence += 1), + timestamp: Date.now(), + type, + payload, + }) as AgentEvent; + + /** The real model call: DeepSeek HTTP streaming, nothing stubbed. */ + const runtime: AgentRuntime = { + async getTools() { + return { ok: true, data: [] }; + }, + async ensureSession(session) { + return { sessionId: session.id, status: 'active' }; + }, + async cancel() { + abort.abort(); + }, + async dispose() { + abort.abort(); + }, + async *run(input) { + const response = await fetch('https://api.deepseek.com/chat/completions', { + method: 'POST', + headers: { + Authorization: 'Bearer ' + process.env.DEEPSEEK_API_KEY, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: options.model, + stream: true, + stream_options: { include_usage: true }, + response_format: { type: 'json_object' }, + max_tokens: MAX_TOKENS, + messages: [{ role: 'user', content: input.content }], + }), + signal: AbortSignal.any([abort.signal, AbortSignal.timeout(300_000)]), + }); + if (!response.ok || !response.body) { + throw new Error('DeepSeek returned HTTP ' + response.status); + } + yield event(input, 'message_started'); + let answer = ''; + let buffer = ''; + let apiModel: string | undefined; + const decoder = new TextDecoder(); + // Reader loop rather than `for await`: the repository tsconfig ships no + // `DOM.AsyncIterable` lib, so async-iterating a ReadableStream is a type error. + const reader = response.body.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + let newline: number; + while ((newline = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, newline).trim(); + buffer = buffer.slice(newline + 1); + if (!line.startsWith('data: ') || line === 'data: [DONE]') continue; + const part = JSON.parse(line.slice(6)) as { + model?: string; + usage?: unknown; + choices?: Array<{ delta?: { content?: string } }>; + }; + // The model the API actually served — recorded as evidence that the + // configured model is the one that ran, not merely the one requested. + if (part.model && !apiModel) apiModel = part.model; + if (part.usage) { + await save(`model-usage-${options.label}.json`, { requested: options.model, apiModel, usage: part.usage }); + } + const delta = part.choices?.[0]?.delta?.content; + if (!delta) continue; + answer += delta; + yield event(input, 'message_delta', { delta, answer }); + } + } + yield event(input, 'message_completed', { answer }); + yield event(input, 'run_completed', { answer, toolCalls: [] }); + }, + }; + + const kernel = new AgentKernel({ + storageDir: join(output, 'kernel'), + piSessionDir: join(output, 'sessions'), + runtime, + // #17: the budget the manifest must record as "what this run obeyed". + budgets: { defaults: options.budgetDefaults }, + }); + + // Real tool snapshot, derived from the real registry the research plan uses. + const tools = registry.list().map((capability) => ({ + name: capability.toolName, + capability: capability.id, + enabled: true, + })); + + const service = new ResearchService({ + registry, + repository, + budgets: { defaults: options.budgetDefaults }, + getIdentity: async () => ({ provider: 'deepseek', model: options.model, config: 'live-http-acceptance' }), + // #21: the immutable snapshot. A new process rebuilds this from *its own* + // current globals — which is exactly what the restart read-back tests. + getRunManifestContext: async (): Promise => ({ + runtimeMode: 'pi', + appVersion: process.env.FINAGENT_ACCEPTANCE_APP_VERSION ?? '0.0.0-acceptance', + buildVersion: process.env.FINAGENT_ACCEPTANCE_APP_VERSION ?? '0.0.0-acceptance', + gitRevision: process.env.FINAGENT_ACCEPTANCE_GIT_REVISION, + provider: 'deepseek', + model: options.model, + modelParams: { + api: 'openai-completions', + baseUrl: 'https://api.deepseek.com', + maxTokens: MAX_TOKENS, + reasoning: false, + }, + prompt: { hash: PROMPT_HASH, version: PROMPT_VERSION, source: PROMPT_SOURCE }, + tools, + // Recorded from the live router, not hard-coded: the manifest must name + // the provider that actually serves retrieval for this run. + search: retrievalManifest(routing), + featureFlags: { runtimeProvider: 'pi', demoData: false, langfuseTracing: false }, + locale: 'en-US', + }), + synthesizer: { + async synthesize(input) { + const session = await kernel.sessions.createSession('Run manifest acceptance'); + const prompt = [ + `Analyze only these saved facts for ${SYMBOL}. Return JSON with summary, stance (bullish/bearish/neutral), confidence (0..1),`, + 'sections (key/title/verdict/summary), bullCase, bearCase, catalysts, risks (string arrays).', + 'Every planned key must have exactly one section. Missing data means verdict unavailable. Do not invent facts.', + 'Section verdict must be positive, negative, neutral or unavailable. State data gaps. Respond in English.', + 'Planned keys: ' + input.plannedCapabilities.join(', '), + 'Outcomes: ' + JSON.stringify(input.runs), + 'Saved facts: ' + input.dataBundle, + ].join('\n'); + + let unsubscribe = () => {}; + const result = new Promise((resolveAnswer, reject) => { + unsubscribe = kernel.runs.subscribe((agentEvent) => { + if (agentEvent.sessionId !== session.id) return; + if (agentEvent.type === 'run_completed') resolveAnswer(agentEvent.payload.answer ?? ''); + if (agentEvent.type === 'run_failed') reject(new Error(agentEvent.payload.error.message)); + }); + }); + void result.catch(() => {}); + try { + const run = await kernel.runs.startRun(session.id, prompt); + await input.recovery?.onAgentRun(run.id, session.id); + return parseSynthesisJson(await result); + } finally { + unsubscribe(); + } + }, + }, + }); + + return { service, kernel, fetchers }; +} + +// --------------------------------------------------------------------------- +// Worker modes +// --------------------------------------------------------------------------- + +async function runWorker(options: WorkerOptions): Promise { + assert.ok(process.env.DEEPSEEK_API_KEY, 'Set DEEPSEEK_API_KEY. No fixture fallback.'); + const { service, kernel, fetchers } = buildStack(options); + + const queued = await service.start(SYMBOL, STRATEGY); + const done = await until(async () => { + const current = await service.getRun(queued.id); + return current && ['completed', 'partial', 'failed', 'cancelled', 'interrupted'].includes(current.status) + ? current + : undefined; + }); + while (kernel.runs.isRunning()) await Bun.sleep(20); + + const manifest = (await service.getRun(queued.id))?.manifest; + assert.ok(manifest, 'The production research run produced no manifest'); + + // Who actually answered, straight off the same fetcher chain the run used. + // Recorded so the report can show a real provider + timestamp + delayed flag + // instead of asserting "Yahoo" from the wiring alone. A failure is recorded + // as a failure — never papered over with a zero. + const marketSource = await fetchers + .getQuote(SYMBOL) + .then((quote) => { + const source = (quote as { source?: Record }).source; + return { + ok: true as const, + symbol: quote.symbol, + lastPrice: quote.lastPrice, + timestamp: quote.timestamp, + ...source, + }; + }) + .catch((error: unknown) => ({ ok: false as const, error: String(error) })); + + await save(`${options.label}.json`, { + label: options.label, + requestedModel: options.model, + budgetDefaults: options.budgetDefaults, + researchRunId: queued.id, + status: done.status, + completedCapabilities: done.completedCapabilities, + failedCapabilities: done.failedCapabilities, + marketSource, + manifest, + }); + await kernel.dispose(); +} + +async function inspectWorker(runId: string, currentModel: string): Promise { + // A brand-new process: nothing is inherited from the run that wrote it. + // Its "current globals" deliberately name a *different* model. + const { service, kernel } = buildStack({ + label: 'restart', + model: currentModel, + budgetDefaults: { modelCalls: 99 }, + }); + + const historical = await service.getRun(runId); + assert.ok(historical?.manifest, 'The historical run had no persisted manifest after restart'); + + await save('run-info-after-restart.json', { + readIn: 'fresh process (simulated app restart)', + currentGlobalModel: currentModel, + researchRunId: runId, + status: historical.status, + manifest: historical.manifest, + }); + await kernel.dispose(); +} + +// --------------------------------------------------------------------------- +// Parent orchestrator +// --------------------------------------------------------------------------- + +async function main(): Promise { + assert.ok(process.env.DEEPSEEK_API_KEY, 'Set DEEPSEEK_API_KEY. No fixture fallback.'); + await mkdir(output, { recursive: true }); + + // Guard against reusing a previous execution's artifacts. + try { + await readFile(join(output, 'verification.json')); + throw new Error('Use a fresh FINAGENT_MANIFEST_OUTPUT directory.'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + + const runChild = async (args: string[], artifact: string) => { + const child = launch(args); + let spawnError: Error | undefined; + child.on('error', (error) => { + spawnError = error; + }); + // Attach the exit listener *before* waiting on the artifact: a child that + // writes its artifact and exits immediately would otherwise emit `exit` + // before we start listening, and `once` would never resolve. + const exited = once(child, 'exit'); + const artifactResult = await until(async () => { + if (spawnError) throw spawnError; + if (child.exitCode !== null && child.exitCode !== 0) { + throw new Error(`Worker "${args.join(' ')}" exited with ${child.exitCode}; see worker.log`); + } + return json(artifact); + }); + const [code] = await exited; + assert.equal(code, 0, `Worker "${args.join(' ')}" failed`); + return artifactResult; + }; + + const recordA = await runChild( + ['run', RUN_A.label, RUN_A.model, JSON.stringify(RUN_A.budget)], + `${RUN_A.label}.json` + ); + const recordB = await runChild( + ['run', RUN_B.label, RUN_B.model, JSON.stringify(RUN_B.budget)], + `${RUN_B.label}.json` + ); + + // "Restart": a fresh process whose current globals differ from run A. + const readback = await runChild( + ['inspect', recordA.researchRunId, RESTART_CURRENT_MODEL], + 'run-info-after-restart.json' + ); + + const manifestA = recordA.manifest; + const manifestB = recordB.manifest; + const diff = diffRunManifests(manifestA, manifestB); + + // --- Assertions: the manifest reflects the deliberate change ------------- + assert.notEqual(manifestA.runId, manifestB.runId, 'The two runs must have distinct identities'); + assert.equal(manifestA.model, RUN_A.model); + assert.equal(manifestB.model, RUN_B.model); + assert.equal(manifestA.runId, recordA.researchRunId, 'The manifest must carry the run identity'); + assert.equal(manifestA.strategy?.id, STRATEGY, 'The research strategy must be recorded'); + assert.equal(diff.changed, true, 'The deliberate config change must show as a diff'); + assert.equal(diff.groups.model, true, 'The diff must flag the model dimension'); + + const modelField = diff.fields.find((field) => field.path === 'model'); + assert.ok(modelField, 'The diff must contain the model field'); + assert.equal(modelField.before, RUN_A.model); + assert.equal(modelField.after, RUN_B.model); + // The budget change is a real config change too (#17 integration). + const budgetField = diff.fields.find((field) => field.path === 'budget.effective.modelCalls'); + assert.ok(budgetField, 'The diff must contain the effective model-call budget'); + assert.equal(budgetField.before, RUN_A.budget.modelCalls); + assert.equal(budgetField.after, RUN_B.budget.modelCalls); + // Nothing else changed: the prompt composition was identical across runs. + assert.equal(diff.groups.prompt, false, 'An unchanged prompt must not be reported as a diff'); + + // --- Assertion: the restart read-back shows the ORIGINAL config ---------- + assert.equal( + readback.currentGlobalModel, + RESTART_CURRENT_MODEL, + 'The restart process must have different current globals' + ); + assert.equal( + readback.manifest.model, + RUN_A.model, + 'After restart the historical run must still report its own model, not the current global' + ); + assert.notEqual( + readback.manifest.model, + readback.currentGlobalModel, + 'The read-back must not be served from the restarting process globals' + ); + + // --- Assertion: no credential ever reaches a manifest -------------------- + const serialized = JSON.stringify([manifestA, manifestB, readback.manifest]); + assert.equal( + serialized.includes(process.env.DEEPSEEK_API_KEY!), + false, + 'A credential must never appear in a manifest' + ); + + // --- Assertion: the manifest names the provider that actually served ------ + assert.equal( + manifestA.search?.providerId, + YAHOO_PROVIDER_ID, + 'The manifest must record the retrieval provider the run actually used' + ); + assert.equal(manifestA.search?.routing?.primary, YAHOO_PROVIDER_ID); + assert.equal( + recordA.marketSource.ok, + true, + 'The retrieval probe must return real data — a configured connection is not proof of a successful fetch' + ); + assert.equal( + recordA.marketSource.providerId, + YAHOO_PROVIDER_ID, + 'The quote must carry the answering provider' + ); + assert.equal(recordA.marketSource.delayed, true, 'This provider serves delayed US quotes and must say so'); + assert.ok(recordA.marketSource.timestamp > 0, 'The quote must carry the provider timestamp'); + + const verification = { + kind: 'live', + transport: 'real DeepSeek HTTP streaming; real provider-router retrieval', + retrieval: { + wiring: 'YahooChartFinancialDataProvider → ProviderRouter → createRouterFetchers → createFullRegistry', + provider: YAHOO_PROVIDER_ID, + routing: manifestA.search?.routing, + endpoint: manifestA.search?.endpoint, + supported: ['market.quote', 'market.kline', 'company.profile'], + unsupported: + 'every other capability raises UNSUPPORTED_CAPABILITY from the router and is reported unavailable — never zero, never an empty event list', + caveat: 'no-key unofficial chart host, personal use, may rate-limit', + }, + // What actually answered, per run: provider id, timestamp, delayed flag. + marketSource: { runA: recordA.marketSource, runB: recordB.marketSource }, + symbol: SYMBOL, + strategy: STRATEGY, + runs: [ + { + label: RUN_A.label, + researchRunId: recordA.researchRunId, + status: recordA.status, + requestedModel: RUN_A.model, + manifestModel: manifestA.model, + manifestRunId: manifestA.runId, + completedCapabilities: recordA.completedCapabilities, + failedCapabilities: recordA.failedCapabilities, + }, + { + label: RUN_B.label, + researchRunId: recordB.researchRunId, + status: recordB.status, + requestedModel: RUN_B.model, + manifestModel: manifestB.model, + manifestRunId: manifestB.runId, + completedCapabilities: recordB.completedCapabilities, + failedCapabilities: recordB.failedCapabilities, + }, + ], + deliberateChange: { + model: { before: RUN_A.model, after: RUN_B.model }, + budgetModelCalls: { before: RUN_A.budget.modelCalls, after: RUN_B.budget.modelCalls }, + }, + diff: { + changed: diff.changed, + groups: diff.groups, + fieldCount: diff.fields.length, + paths: diff.fields.map((field) => field.path), + }, + restartReadback: { + process: 'fresh process', + currentGlobalModel: readback.currentGlobalModel, + historicalModel: readback.manifest.model, + historicalRunId: readback.manifest.runId, + preserved: readback.manifest.model === RUN_A.model, + }, + secretScan: { apiKeyPresentInManifests: serialized.includes(process.env.DEEPSEEK_API_KEY!) }, + }; + + await save('run-a-manifest.json', manifestA); + await save('run-b-manifest.json', manifestB); + await save('manifest-diff.json', diff); + await save('run-a-manifest-export.json', JSON.parse(exportRunManifest(manifestA))); + await save('verification.json', verification); + console.log(JSON.stringify(verification, null, 2)); +} + +// --------------------------------------------------------------------------- +// Entry +// --------------------------------------------------------------------------- + +const [mode, arg1, arg2, arg3] = process.argv.slice(2); +if (mode === 'run') { + await runWorker({ + label: arg1, + model: arg2, + budgetDefaults: JSON.parse(arg3 ?? '{}') as Record, + }); + process.exit(0); +} else if (mode === 'inspect') { + await inspectWorker(arg1, arg2); + process.exit(0); +} else { + await main(); +} diff --git a/apps/electron/src/main/index.ts b/apps/electron/src/main/index.ts index 423564b8..9ba83ab3 100644 --- a/apps/electron/src/main/index.ts +++ b/apps/electron/src/main/index.ts @@ -138,6 +138,23 @@ ipcMain.handle('runs:stream-replay', async (_event, input: unknown) => toIpcResult(() => agentKernelHost.streamReplay(input)) ); +// #21: immutable run manifests for Agent / Deep Research runs. +ipcMain.handle('runs:getManifest', async (_event, input: unknown) => + toIpcResult(() => agentKernelHost.getRunManifest(input)) +); +ipcMain.handle('runs:exportManifest', async (_event, input: unknown) => + toIpcResult(() => agentKernelHost.exportRunManifest(input)) +); +ipcMain.handle('runs:compareManifests', async (_event, input: unknown) => + toIpcResult(() => agentKernelHost.compareRunManifests(input)) +); +ipcMain.handle('research:getManifest', async (_event, input: unknown) => + toIpcResult(() => agentKernelHost.getResearchManifest(input)) +); +ipcMain.handle('research:compareManifests', async (_event, input: unknown) => + toIpcResult(() => agentKernelHost.compareResearchManifests(input)) +); + ipcMain.handle('agent:getTools', async () => toIpcResult(() => agentKernelHost.getTools()) ); diff --git a/apps/electron/src/main/kernelHost.test.ts b/apps/electron/src/main/kernelHost.test.ts index 56ffab6a..a9921b33 100644 --- a/apps/electron/src/main/kernelHost.test.ts +++ b/apps/electron/src/main/kernelHost.test.ts @@ -1,6 +1,13 @@ import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test'; import { join } from 'node:path'; import type { AgentEvent, AutomationRule, AutomationRun } from '@finagent/core'; +// `@finagent/shared` is mocked wholesale below, so the *real* manifest helpers +// are pulled in from source: the IPC read-path tests must exercise the actual +// redaction/diff implementation, not a stub of it. +import { + diffRunManifests as realDiffRunManifests, + redactManifestSecrets as realRedactManifestSecrets, +} from '../../../../packages/shared/src/kernel/run-manifest.ts'; let lastKernelOptions: Record | null = null; let lastMarketData: FakeMarketDataService | null = null; @@ -10,6 +17,7 @@ let lastCompareSymbols: unknown = null; let forwardedEvents: unknown[] = []; /** Evaluation runs handed to EvaluationStore.addRun (observability path). */ const savedEvalRuns: Array> = []; +let storedRuns: StoredRun[] = []; const routerFetchers = { getQuote: async () => ({ symbol: 'AAPL.US' }) }; const scheduledRule: AutomationRule = { id: 'scheduled-rule', @@ -39,6 +47,15 @@ let runAutomationMock: () => Promise = async () => { return automationRun(); }; +interface StoredRun { + id: string; + sessionId: string; + status: string; + input: string; + startedAt: number; + manifest?: unknown; +} + /** Poll until `predicate` holds; the settle path is fire-and-forget (`void`). */ async function waitFor(predicate: () => boolean, timeoutMs = 1000): Promise { const deadline = Date.now() + timeoutMs; @@ -93,7 +110,7 @@ const fakeSessions = { }), deleteSession: async () => undefined, listMessages: async (sessionId: string) => [{ id: 'm1', role: 'user', content: sessionId, timestamp: 1 }], - listRuns: async () => [], + listRuns: async () => storedRuns, getSession: async () => undefined, }; @@ -374,6 +391,13 @@ mock.module('@finagent/shared', () => ({ redactToolCall = (toolCall: unknown) => toolCall; }, PiRuntimeAdapter: class {}, + // #21 run-manifest helpers used by kernelHost's manifest IPC. Redaction and + // diffing are the real implementations: the IPC read-path tests assert on + // what actually leaves the main process. + diffRunManifests: realDiffRunManifests, + redactManifestSecrets: realRedactManifestSecrets, + exportRunManifest: (manifest: unknown) => JSON.stringify(manifest, null, 2), + manifestToLangfuseMetadata: (manifest: { runId: string }) => ({ folioRunId: manifest.runId }), sanitizeSettings: (input: unknown) => input, embeddedDatasets: [], })); @@ -411,6 +435,7 @@ beforeEach(() => { return automationRun(); }; savedEvalRuns.length = 0; + storedRuns = []; }); afterEach(() => { @@ -835,3 +860,122 @@ describe('AgentKernelHost', () => { host.dispose(); }); }); + +describe('AgentKernelHost run manifest IPC (#21)', () => { + // A manifest written before capture-time redaction existed (or by a caller + // bug) must not be able to hand a credential to the renderer. + const dirtyManifest = (runId: string, overrides: Record = {}) => ({ + schemaVersion: 1, + runId, + createdAt: 1, + runtimeMode: 'pi', + model: 'gpt-4o', + modelParams: { apiKey: 'sk-live', maxTokens: 4096 }, + ...overrides, + }); + + const storedRun = (runId: string, manifest: unknown): StoredRun => ({ + id: runId, + sessionId: 's1', + status: 'done', + input: 'hi', + startedAt: 1, + manifest, + }); + + it('redacts credentials when a copilot run manifest is read over IPC', async () => { + storedRuns = [storedRun('r1', dirtyManifest('r1'))]; + const host = new AgentKernelHost(); + + const manifest = await host.getRunManifest({ sessionId: 's1', runId: 'r1' }); + + expect((manifest?.modelParams as Record).apiKey).toBe('[redacted]'); + expect(manifest?.modelParams?.maxTokens).toBe(4096); + host.dispose(); + }); + + it('redacts credentials when a Deep Research manifest is read over IPC', async () => { + const host = new AgentKernelHost(); + const research = ( + host as unknown as { researchService: { getRun: (runId: string) => Promise } } + ).researchService; + research.getRun = async () => ({ manifest: dirtyManifest('research-1') }); + + const manifest = await host.getResearchManifest({ runId: 'research-1' }); + + expect((manifest?.modelParams as Record).apiKey).toBe('[redacted]'); + host.dispose(); + }); + + it('redacts both sides of a manifest comparison', async () => { + storedRuns = [ + storedRun('r1', dirtyManifest('r1')), + storedRun('r2', dirtyManifest('r2', { model: 'gpt-4o-mini' })), + ]; + const host = new AgentKernelHost(); + + const diff = await host.compareRunManifests({ sessionId: 's1', runIdA: 'r1', runIdB: 'r2' }); + + expect(JSON.stringify(diff)).not.toContain('sk-live'); + expect(diff.fields.map((field) => field.path)).toContain('model'); + host.dispose(); + }); + + // The dedicated getters above are not the only way a manifest reaches the + // renderer: run history is read through listRuns / researchListRuns / + // researchGetRun, and each record carries the same `manifest` field. These + // three routes used to hand the raw record through, so a dirty manifest + // bypassed redaction entirely by going through them instead. + it('redacts manifests returned by the ordinary copilot run-history read path', async () => { + storedRuns = [storedRun('r1', dirtyManifest('r1')), storedRun('r2', undefined)]; + const host = new AgentKernelHost(); + + const runs = await host.listRuns('s1'); + + expect(runs.map((run) => run.id)).toEqual(['r1', 'r2']); + expect((runs[0].manifest?.modelParams as Record).apiKey).toBe('[redacted]'); + expect(runs[0].manifest?.modelParams?.maxTokens).toBe(4096); + // A run with no manifest must not gain one, and the rest of the record is + // passed through untouched. + expect(runs[1].manifest).toBeUndefined(); + expect(runs[1].input).toBe('hi'); + // Redaction is applied to the copy handed over IPC, not written back into + // the store: the persisted record must keep the value it was created with. + expect( + (storedRuns[0].manifest as { modelParams: Record }).modelParams.apiKey + ).toBe('sk-live'); + host.dispose(); + }); + + it('redacts manifests returned by the ordinary Deep Research run-history read path', async () => { + const host = new AgentKernelHost(); + const research = ( + host as unknown as { researchService: { listRuns: () => Promise } } + ).researchService; + research.listRuns = async () => [ + { id: 'research-1', symbol: 'NVDA.US', status: 'partial', manifest: dirtyManifest('research-1') }, + ]; + + const runs = await host.researchListRuns(); + + expect(runs[0].id).toBe('research-1'); + expect((runs[0].manifest?.modelParams as Record).apiKey).toBe('[redacted]'); + expect(runs[0].manifest?.modelParams?.maxTokens).toBe(4096); + host.dispose(); + }); + + it('redacts manifests returned by the ordinary Deep Research run read path', async () => { + const host = new AgentKernelHost(); + const research = ( + host as unknown as { researchService: { getRun: (runId: string) => Promise } } + ).researchService; + research.getRun = async () => ({ id: 'research-1', manifest: dirtyManifest('research-1') }); + + const run = await host.researchGetRun({ runId: 'research-1' }); + + expect(run?.id).toBe('research-1'); + expect((run?.manifest?.modelParams as Record).apiKey).toBe('[redacted]'); + expect(run?.manifest?.modelParams?.maxTokens).toBe(4096); + host.dispose(); + }); +}); diff --git a/apps/electron/src/main/kernelHost.ts b/apps/electron/src/main/kernelHost.ts index eeaec188..b126d5f3 100644 --- a/apps/electron/src/main/kernelHost.ts +++ b/apps/electron/src/main/kernelHost.ts @@ -26,6 +26,15 @@ import type { ResearchSynthesis, ResearchSynthesisInput, Run, + RunManifest, + RunManifestContext, + RunManifestDiff, + RunManifestFeatureFlags, + RunManifestModelParams, + RunManifestPrompt, + RunManifestRuntimeMode, + RunManifestSearch, + RunManifestTool, ScreeningRun, ScreeningQuery, ScreeningStrategy, @@ -71,7 +80,7 @@ import type { ToolCallRecord, TraceReference, } from '@finagent/core'; -import { DEFAULT_INSTRUMENT_CATALOG, InstrumentResolver, STRATEGY_IDS } from '@finagent/core'; +import { AGENT_WORKFLOW_ID, DEFAULT_INSTRUMENT_CATALOG, InstrumentResolver, STRATEGY_IDS } from '@finagent/core'; import { isLocalePreference } from '@finagent/i18n'; import { createAppPreferencesService, type AppPreferencesService } from './app-preferences.ts'; import { buildImpactPrompt, buildRiskSummaryPrompt, buildSynthesisPrompt } from './research-prompts.ts'; @@ -144,6 +153,10 @@ import { scoresFromResearchReport, currentFolioVersion, EvaluationRedactor, + diffRunManifests, + exportRunManifest, + redactManifestSecrets, + manifestToLangfuseMetadata, PiRuntimeAdapter, sanitizeSettings, embeddedDatasets, @@ -299,6 +312,7 @@ export class AgentKernelHost { private readonly portfolioRisk: PortfolioRiskService; private readonly connectionStore: ConnectionStore; private readonly providerRouter: ProviderRouter; + private readonly demoData: boolean; private instrumentResolver: InstrumentResolver; private activeLogin: { cancel: () => void } | null = null; private unsubscribe: (() => void) | null = null; @@ -375,6 +389,7 @@ export class AgentKernelHost { // unavailable, built-in sample data answers instead; every surface built // on it labels its content with source 'demo' (DemoBadge). const demoData = process.env.FINAGENT_DEMO_DATA === '1'; + this.demoData = demoData; const capabilityFetchers = demoData ? withDemoDataFallback(routerFetchers) : routerFetchers; this.marketData = new MarketDataService({ fetchers: capabilityFetchers }); this.registry = createFullRegistry(capabilityFetchers); @@ -403,6 +418,8 @@ export class AgentKernelHost { await this.saveDiffForReport(report); }, onRunComplete: (result) => this.exportResearchTrace(result), + // #21: snapshot the research run config into an immutable manifest. + getRunManifestContext: () => this.buildRunManifestContext(), }); // V5 outcome evaluation: opinions snapshotted from reports, outcomes @@ -467,6 +484,9 @@ export class AgentKernelHost { demoData, registry: new FinanceToolRegistry(this.registry), skillHub: this.skillHub, + // #21: every run snapshots the live config — including evaluation runs, + // which are started directly on the kernel by the experiment runner. + getRunManifestContext: () => this.buildRunManifestContext(), rpc: { cwd: getPiCwd(), requiredEnvKeys: readRequiredLlmEnvKeys(), @@ -534,7 +554,8 @@ export class AgentKernelHost { } async listRuns(sessionId: unknown): Promise { - return this.kernel.sessions.listRuns(requireString(sessionId, 'sessionId')); + const runs = await this.kernel.sessions.listRuns(requireString(sessionId, 'sessionId')); + return runs.map((run) => this.redactRunRecordManifest(run)); } async startRun(input: unknown): Promise { @@ -571,6 +592,10 @@ export class AgentKernelHost { // user explicitly requests another language in the prompt (spec §41–42). // Resolved after validation and with a safe fallback so a prefs failure // can never block an agent run (failure-isolation, spec §87). + // #21: the run's immutable config snapshot is assembled by the kernel from + // `getRunManifestContext` (wired in the constructor) at creation time, so + // every run entry point — this one and evaluation runs — snapshots the same + // settings. Capture is best-effort and never blocks the run (spec §87). return this.kernel.runs.startRun( requireString(request.sessionId, 'sessionId'), requireString(request.content, 'content'), @@ -587,6 +612,122 @@ export class AgentKernelHost { } } + /** + * Assemble the immutable run-config snapshot for #21. Best-effort: any single + * source that is unavailable (e.g. no Pi process, no skills) simply leaves its + * field undefined rather than failing the run. The manifest captures model + + * provider identity, effective model params, the prompt composition hash, + * enabled tools, the search/retrieval routing, active feature flags and the + * app/build identity — but never credentials (defence-in-depth redaction also + * runs in `@finagent/shared`). + */ + private async buildRunManifestContext(): Promise { + const runtimeMode: RunManifestRuntimeMode = this.demoData + ? 'demo' + : readAgentProvider() === 'local' + ? 'local' + : 'pi'; + + const identity: { + provider?: string; + model?: string; + modelParams?: RunManifestModelParams; + } = {}; + const api = this.kernel.getLlmApi(); + if (api) { + try { + const state = await api.getState(); + const model = state.model; + identity.provider = + model?.provider ?? (readAgentProvider() === 'local' ? 'local' : undefined); + identity.model = model?.id; + identity.modelParams = { + api: model?.api, + baseUrl: model?.baseUrl, + thinkingLevel: state.thinkingLevel, + contextWindow: model?.contextWindow, + maxTokens: model?.maxTokens, + reasoning: model?.reasoning, + }; + } catch { + // Local runtime / Pi not spawned yet — identity stays empty. + } + } + + const prompt = this.buildPromptManifest(); + + let tools: RunManifestTool[] = []; + try { + const result = await this.kernel.getTools(); + if (result.ok) { + const byToolName = new Map(this.registry.list().map((cap) => [cap.toolName, cap.id])); + tools = result.data.map((tool) => ({ + name: tool.name, + capability: byToolName.get(tool.name), + enabled: true, + })); + } + } catch { + tools = []; + } + + const routing = await this.connectionStore.getRouting(DEFAULT_PROVIDER_ROUTING); + const primaryConfig = routing.primary + ? await this.connectionStore.getConfig(routing.primary) + : undefined; + + const search: RunManifestSearch = { + providerId: routing.primary, + routing: { primary: routing.primary, fallback: routing.fallback }, + configured: routing.primary ? primaryConfig?.enabled !== false : false, + endpoint: primaryConfig?.endpoint ?? undefined, + }; + + const featureFlags: RunManifestFeatureFlags = { + demoData: this.demoData, + runtimeProvider: readAgentProvider(), + evalTracing: this.evaluationSettings.tracingEnabled, + langfuseTracing: this.evaluationSettings.langfuseTracingEnabled, + privacyLevel: this.evaluationSettings.privacyLevel, + }; + + return { + runtimeMode, + appVersion: app.getVersion(), + buildVersion: app.getVersion(), + gitRevision: process.env.FOLIO_GIT_REVISION ?? process.env.BUILD_REVISION ?? undefined, + ...identity, + prompt, + // #21: name the workflow that drove the run. The Copilot agent path is not + // strategy-driven (unlike Deep Research, which overrides `id` with its + // `StrategyId`); recording the workflow id + shipping version keeps + // "which agent produced this?" answerable for a historical run. + strategy: { id: AGENT_WORKFLOW_ID, version: currentFolioVersion() ?? undefined }, + tools, + search, + featureFlags, + locale: await this.effectiveRunLocale(), + }; + } + + /** + * Hash the *composition* of the system/developer prompt: which skills are + * enabled and when they last changed. Toggling or updating a skill changes + * the hash, so two runs with different prompt compositions diff cleanly. + * The app version is recorded as the template `version` (the prompt template + * ships with Folio). + */ + private buildPromptManifest(): RunManifestPrompt { + const enabled = this.skillHub + .listSkills() + .filter((skill) => skill.metadata.enabled) + .map((skill) => `${skill.id}:${skill.metadata.updatedAt ?? 0}`) + .sort() + .join('|'); + const hash = createHash('sha256').update(`skills|${enabled}`).digest('hex'); + return { hash, version: currentFolioVersion() ?? undefined, source: 'skill-index' }; + } + async cancelRun(input: unknown): Promise { const request = requireObject(input); await this.kernel.runs.cancelRun( @@ -595,6 +736,85 @@ export class AgentKernelHost { ); } + // -- Run manifests (#21) --------------------------------------------------- + + /** + * Clone a run/summary record with its embedded `manifest` redacted. + * + * `listRuns` / `researchListRuns` / `researchGetRun` are the *ordinary* read + * paths the renderer uses for run history, and every record they return + * carries the same `manifest` field as the dedicated `get*Manifest` IPC. + * Capture already strips credentials, but a manifest persisted before that + * guarantee — or written by a caller that bypassed `captureRunManifest` — + * must not be able to hand a key to the renderer through these routes + * either, so they go through the same redaction as the dedicated getters. + * + * Only `manifest` is cloned; the rest of the record keeps its identity. + */ + private redactRunRecordManifest(record: T): T { + if (!record.manifest) return record; + return { ...record, manifest: redactManifestSecrets(record.manifest) }; + } + + /** + * Immutable run-config snapshot for a copilot run, or undefined if absent. + * + * Redacted on the way out as well: capture already strips credentials, but a + * manifest persisted before that guarantee (or by a caller bug) must not be + * able to hand a key to the renderer through this IPC. + */ + async getRunManifest(input: unknown): Promise { + const request = requireObject(input); + const sessionId = requireString(request.sessionId, 'sessionId'); + const runId = requireString(request.runId, 'runId'); + const runs = await this.kernel.sessions.listRuns(sessionId); + const manifest = runs.find((run) => run.id === runId)?.manifest; + return manifest ? redactManifestSecrets(manifest) : undefined; + } + + /** Pretty-printed, secret-redacted JSON export of a copilot run manifest. */ + async exportRunManifest(input: unknown): Promise { + const manifest = await this.getRunManifest(input); + return manifest ? exportRunManifest(manifest) : undefined; + } + + /** Structured diff between two copilot run manifests. */ + async compareRunManifests(input: unknown): Promise { + const request = requireObject(input); + const sessionId = requireString(request.sessionId, 'sessionId'); + const aId = requireString(request.runIdA, 'runIdA'); + const bId = requireString(request.runIdB, 'runIdB'); + const runs = await this.kernel.sessions.listRuns(sessionId); + const before = runs.find((run) => run.id === aId)?.manifest; + const after = runs.find((run) => run.id === bId)?.manifest; + if (!before || !after) { + throw createCodeError('RUN_MANIFEST_NOT_FOUND', 'Both runs must have a manifest to compare.'); + } + // A diff echoes field values back to the renderer, so it must be redacted too. + return diffRunManifests(redactManifestSecrets(before), redactManifestSecrets(after)); + } + + /** Immutable run-config snapshot for a Deep Research run, or undefined. */ + async getResearchManifest(input: unknown): Promise { + const request = requireObject(input); + const runId = requireString(request.runId, 'runId'); + const manifest = (await this.researchService.getRun(runId))?.manifest; + return manifest ? redactManifestSecrets(manifest) : undefined; + } + + /** Structured diff between two Deep Research run manifests. */ + async compareResearchManifests(input: unknown): Promise { + const request = requireObject(input); + const aId = requireString(request.runIdA, 'runIdA'); + const bId = requireString(request.runIdB, 'runIdB'); + const before = (await this.researchService.getRun(aId))?.manifest; + const after = (await this.researchService.getRun(bId))?.manifest; + if (!before || !after) { + throw createCodeError('RUN_MANIFEST_NOT_FOUND', 'Both runs must have a manifest to compare.'); + } + return diffRunManifests(redactManifestSecrets(before), redactManifestSecrets(after)); + } + /** Stream Event replay(ADR 0001 §Reconnect):按 lastSequence 补发或明确不可恢复。 */ streamReplay(input: unknown): StreamReplayResult { const request = requireObject(input); @@ -735,12 +955,14 @@ export class AgentKernelHost { } async researchListRuns(): Promise { - return this.researchService.listRuns(); + const runs = await this.researchService.listRuns(); + return runs.map((run) => this.redactRunRecordManifest(run)); } async researchGetRun(input: unknown): Promise { const request = requireObject(input); - return this.researchService.getRun(requireString(request.runId, 'runId')); + const run = await this.researchService.getRun(requireString(request.runId, 'runId')); + return run ? this.redactRunRecordManifest(run) : undefined; } async researchListReports(input: unknown): Promise { @@ -1490,6 +1712,27 @@ export class AgentKernelHost { }): Promise { if (!this.langfuseBackend) return undefined; try { + // #21 ↔ #14: derive the trace metadata from the run's immutable manifest + // so the Langfuse trace and the local manifest share one identity + // (`folioRunId` = trace id) and one config record (model/provider/ + // strategy/prompt version), instead of the trace carrying a bare run id. + const manifest = await this.getRunManifest({ + sessionId: input.sessionId, + runId: input.runId, + }).catch(() => undefined); + const metadata = manifest + ? manifestToLangfuseMetadata(manifest, { + sessionId: input.sessionId, + threadId: input.threadId, + runKind: 'normal', + }) + : { + folioRunId: input.runId, + folioSessionId: input.sessionId, + threadId: input.threadId, + runKind: 'normal' as const, + folioVersion: currentFolioVersion(), + }; const ref = await this.langfuseBackend.exportAgentRun({ folioRunId: input.runId, sessionId: input.sessionId, @@ -1500,13 +1743,7 @@ export class AgentKernelHost { output: input.answer, toolCalls: input.toolCalls, error: input.error, - metadata: { - folioRunId: input.runId, - folioSessionId: input.sessionId, - threadId: input.threadId, - runKind: 'normal', - folioVersion: currentFolioVersion(), - }, + metadata, }); if (ref.traceId) { const scores = scoresFromAgentRun({ @@ -1561,13 +1798,19 @@ export class AgentKernelHost { })), report: result.report, model: this.evaluationSettings.langfuseTracingEnabled ? 'folio-synthesizer' : undefined, - metadata: { - folioRunId: result.summary.id, - runKind: 'normal', - folioVersion: currentFolioVersion(), - symbol: result.summary.symbol, - strategyId: result.report?.strategyId, - }, + // #21 ↔ #14: same run identity + config record as the persisted manifest. + metadata: result.summary.manifest + ? manifestToLangfuseMetadata(result.summary.manifest, { + runKind: 'normal', + symbol: result.summary.symbol, + }) + : { + folioRunId: result.summary.id, + runKind: 'normal', + folioVersion: currentFolioVersion(), + symbol: result.summary.symbol, + strategyId: result.report?.strategyId, + }, }); const scores = scoresFromResearchReport(result.report, undefined, Math.max(0, finishedAt - result.summary.startedAt)); if (ref.traceId && scores.length > 0) { diff --git a/apps/electron/src/preload/index.ts b/apps/electron/src/preload/index.ts index d4e018c6..65da6e17 100644 --- a/apps/electron/src/preload/index.ts +++ b/apps/electron/src/preload/index.ts @@ -28,6 +28,9 @@ export interface ElectronAPI { }) => Promise; cancelRun: (input: { sessionId: string; runId: string }) => Promise; streamReplay: (input: { runId: string; lastSequence: number }) => Promise; + getManifest: (input: { sessionId: string; runId: string }) => Promise; + exportManifest: (input: { sessionId: string; runId: string }) => Promise; + compareManifests: (input: { sessionId: string; runIdA: string; runIdB: string }) => Promise; onAgentEvent: (callback: (event: unknown) => void) => () => void; onStreamEvent: (callback: (payload: { sessionId: string; event: unknown }) => void) => () => void; }; @@ -63,6 +66,8 @@ export interface ElectronAPI { cancel: (input: { runId: string }) => Promise; listRuns: () => Promise; getRun: (input: { runId: string }) => Promise; + getManifest: (input: { runId: string }) => Promise; + compareManifests: (input: { runIdA: string; runIdB: string }) => Promise; listReports: (input: { symbol?: string }) => Promise; getReport: (input: { reportId: string }) => Promise; getDiff: (input: { symbol: string }) => Promise; @@ -211,6 +216,11 @@ const electronAPI: ElectronAPI = { cancelRun: (input: { sessionId: string; runId: string }) => ipcRenderer.invoke('runs:cancel', input), streamReplay: (input: { runId: string; lastSequence: number }) => ipcRenderer.invoke('runs:stream-replay', input), + getManifest: (input: { sessionId: string; runId: string }) => ipcRenderer.invoke('runs:getManifest', input), + exportManifest: (input: { sessionId: string; runId: string }) => + ipcRenderer.invoke('runs:exportManifest', input), + compareManifests: (input: { sessionId: string; runIdA: string; runIdB: string }) => + ipcRenderer.invoke('runs:compareManifests', input), onAgentEvent: (callback: (event: unknown) => void) => { const listener = (_event: Electron.IpcRendererEvent, agentEvent: unknown) => callback(agentEvent); ipcRenderer.on('agent:event', listener); @@ -266,6 +276,9 @@ const electronAPI: ElectronAPI = { cancel: (input: { runId: string }) => ipcRenderer.invoke('research:cancel', input), listRuns: () => ipcRenderer.invoke('research:listRuns'), getRun: (input: { runId: string }) => ipcRenderer.invoke('research:getRun', input), + getManifest: (input: { runId: string }) => ipcRenderer.invoke('research:getManifest', input), + compareManifests: (input: { runIdA: string; runIdB: string }) => + ipcRenderer.invoke('research:compareManifests', input), listReports: (input: { symbol?: string }) => ipcRenderer.invoke('research:listReports', input), getReport: (input: { reportId: string }) => ipcRenderer.invoke('research:getReport', input), getDiff: (input: { symbol: string }) => ipcRenderer.invoke('research:getDiff', input), diff --git a/apps/electron/src/renderer/finagentClient.ts b/apps/electron/src/renderer/finagentClient.ts index 456172b4..91936dd6 100644 --- a/apps/electron/src/renderer/finagentClient.ts +++ b/apps/electron/src/renderer/finagentClient.ts @@ -37,6 +37,12 @@ function createElectronClient(): FinagentClient { ipcResult(window.electronAPI.kernel.cancelRun({ sessionId, runId })), streamReplay: (input: { runId: string; lastSequence: number }) => ipcResult(window.electronAPI.kernel.streamReplay(input)), + getManifest: (input: { sessionId: string; runId: string }) => + ipcResult(window.electronAPI.kernel.getManifest(input)), + exportManifest: (input: { sessionId: string; runId: string }) => + ipcResult(window.electronAPI.kernel.exportManifest(input)), + compareManifests: (input: { sessionId: string; runIdA: string; runIdB: string }) => + ipcResult(window.electronAPI.kernel.compareManifests(input)), onAgentEvent: (callback: (event: AgentEvent) => void) => window.electronAPI.kernel.onAgentEvent((event) => callback(event as AgentEvent)), onStreamEvent: (callback: (payload: { sessionId: string; event: StreamEvent }) => void) => @@ -80,6 +86,8 @@ function createElectronClient(): FinagentClient { listReports: (input) => ipcResult(window.electronAPI.research.listReports(input)), getReport: (input) => ipcResult(window.electronAPI.research.getReport(input)), getDiff: (input) => ipcResult(window.electronAPI.research.getDiff(input)), + getManifest: (input) => ipcResult(window.electronAPI.research.getManifest(input)), + compareManifests: (input) => ipcResult(window.electronAPI.research.compareManifests(input)), }, thesis: { list: (symbol?) => ipcResult(window.electronAPI.thesis.list(symbol)), diff --git a/docs/acceptance/run-manifest/manifest-diff.json b/docs/acceptance/run-manifest/manifest-diff.json new file mode 100644 index 00000000..70b21208 --- /dev/null +++ b/docs/acceptance/run-manifest/manifest-diff.json @@ -0,0 +1,27 @@ +{ + "changed": true, + "fields": [ + { + "path": "budget.defaults.modelCalls", + "before": 3, + "after": 6 + }, + { + "path": "budget.effective.modelCalls", + "before": 3, + "after": 6 + }, + { + "path": "model", + "before": "deepseek-flash", + "after": "deepseek-v4-pro" + } + ], + "groups": { + "model": true, + "prompt": false, + "tools": false, + "config": true, + "versions": false + } +} \ No newline at end of file diff --git a/docs/acceptance/run-manifest/model-usage-run-a.json b/docs/acceptance/run-manifest/model-usage-run-a.json new file mode 100644 index 00000000..76f3b693 --- /dev/null +++ b/docs/acceptance/run-manifest/model-usage-run-a.json @@ -0,0 +1,17 @@ +{ + "requested": "deepseek-flash", + "apiModel": "deepseek-flash", + "usage": { + "prompt_tokens": 6068, + "completion_tokens": 5440, + "total_tokens": 11508, + "prompt_tokens_details": { + "cached_tokens": 128 + }, + "completion_tokens_details": { + "reasoning_tokens": 3860 + }, + "prompt_cache_hit_tokens": 128, + "prompt_cache_miss_tokens": 5940 + } +} \ No newline at end of file diff --git a/docs/acceptance/run-manifest/model-usage-run-b.json b/docs/acceptance/run-manifest/model-usage-run-b.json new file mode 100644 index 00000000..bbd05b15 --- /dev/null +++ b/docs/acceptance/run-manifest/model-usage-run-b.json @@ -0,0 +1,17 @@ +{ + "requested": "deepseek-v4-pro", + "apiModel": "deepseek-v4-pro", + "usage": { + "prompt_tokens": 6119, + "completion_tokens": 3577, + "total_tokens": 9696, + "prompt_tokens_details": { + "cached_tokens": 384 + }, + "completion_tokens_details": { + "reasoning_tokens": 2250 + }, + "prompt_cache_hit_tokens": 384, + "prompt_cache_miss_tokens": 5735 + } +} \ No newline at end of file diff --git a/docs/acceptance/run-manifest/run-a-manifest-export.json b/docs/acceptance/run-manifest/run-a-manifest-export.json new file mode 100644 index 00000000..f39b9243 --- /dev/null +++ b/docs/acceptance/run-manifest/run-a-manifest-export.json @@ -0,0 +1,151 @@ +{ + "schemaVersion": 1, + "runId": "research-e95ce1be-6eb9-479c-96dd-668d359c631e", + "createdAt": 1790674646605, + "runtimeMode": "pi", + "appVersion": "0.5.0-acceptance", + "buildVersion": "0.5.0-acceptance", + "gitRevision": "402bd92f2c0b32dfed4b28b1038b67c2c0adf582", + "provider": "deepseek", + "model": "deepseek-flash", + "modelParams": { + "api": "openai-completions", + "baseUrl": "https://api.deepseek.com", + "maxTokens": "[redacted]", + "reasoning": false + }, + "prompt": { + "hash": "8105327bd4c6513d43687b00fe5d3e0cb9a62fd0131bf5ec103f2e423bd9656f", + "version": "deep-research-synthesis@1", + "source": "e2e-acceptance" + }, + "strategy": { + "id": "comprehensive", + "version": "3637a33db7d9979cf55ad2331734b197b0db5ce665f65e2dcc2f57675b3fd076" + }, + "tools": [ + { + "name": "get_quote", + "capability": "market.quote", + "enabled": true + }, + { + "name": "get_kline", + "capability": "market.kline", + "enabled": true + }, + { + "name": "get_intraday", + "capability": "market.intraday", + "enabled": true + }, + { + "name": "get_market_status", + "capability": "market.status", + "enabled": true + }, + { + "name": "get_company_profile", + "capability": "company.profile", + "enabled": true + }, + { + "name": "get_valuation", + "capability": "company.valuation", + "enabled": true + }, + { + "name": "get_news", + "capability": "research.news", + "enabled": true + }, + { + "name": "get_portfolio", + "capability": "portfolio.summary", + "enabled": true + }, + { + "name": "get_market_depth", + "capability": "market.depth", + "enabled": true + }, + { + "name": "get_trades", + "capability": "market.trades", + "enabled": true + }, + { + "name": "get_capital_flow", + "capability": "market.capitalFlow", + "enabled": true + }, + { + "name": "get_market_sentiment", + "capability": "market.sentiment", + "enabled": true + }, + { + "name": "get_financials", + "capability": "company.financials", + "enabled": true + }, + { + "name": "get_ratings", + "capability": "company.ratings", + "enabled": true + }, + { + "name": "get_dividends", + "capability": "company.dividends", + "enabled": true + }, + { + "name": "get_earnings", + "capability": "company.earnings", + "enabled": true + }, + { + "name": "get_calendar_events", + "capability": "research.events", + "enabled": true + }, + { + "name": "get_positions", + "capability": "portfolio.positions", + "enabled": true + }, + { + "name": "get_assets", + "capability": "portfolio.assets", + "enabled": true + }, + { + "name": "get_cash_flow", + "capability": "portfolio.cashFlow", + "enabled": true + } + ], + "search": { + "providerId": "public-http", + "routing": { + "primary": "public-http" + }, + "configured": true, + "endpoint": "https://query1.finance.yahoo.com" + }, + "featureFlags": { + "runtimeProvider": "pi", + "demoData": false, + "langfuseTracing": false + }, + "budget": { + "defaults": { + "modelCalls": 3 + }, + "effective": { + "modelCalls": 3 + }, + "clamped": [] + }, + "locale": "en-US" +} \ No newline at end of file diff --git a/docs/acceptance/run-manifest/run-a-manifest.json b/docs/acceptance/run-manifest/run-a-manifest.json new file mode 100644 index 00000000..59bb6f3e --- /dev/null +++ b/docs/acceptance/run-manifest/run-a-manifest.json @@ -0,0 +1,151 @@ +{ + "schemaVersion": 1, + "runId": "research-e95ce1be-6eb9-479c-96dd-668d359c631e", + "createdAt": 1790674646605, + "runtimeMode": "pi", + "appVersion": "0.5.0-acceptance", + "buildVersion": "0.5.0-acceptance", + "gitRevision": "402bd92f2c0b32dfed4b28b1038b67c2c0adf582", + "provider": "deepseek", + "model": "deepseek-flash", + "modelParams": { + "api": "openai-completions", + "baseUrl": "https://api.deepseek.com", + "maxTokens": 8000, + "reasoning": false + }, + "prompt": { + "hash": "8105327bd4c6513d43687b00fe5d3e0cb9a62fd0131bf5ec103f2e423bd9656f", + "version": "deep-research-synthesis@1", + "source": "e2e-acceptance" + }, + "strategy": { + "id": "comprehensive", + "version": "3637a33db7d9979cf55ad2331734b197b0db5ce665f65e2dcc2f57675b3fd076" + }, + "tools": [ + { + "name": "get_quote", + "capability": "market.quote", + "enabled": true + }, + { + "name": "get_kline", + "capability": "market.kline", + "enabled": true + }, + { + "name": "get_intraday", + "capability": "market.intraday", + "enabled": true + }, + { + "name": "get_market_status", + "capability": "market.status", + "enabled": true + }, + { + "name": "get_company_profile", + "capability": "company.profile", + "enabled": true + }, + { + "name": "get_valuation", + "capability": "company.valuation", + "enabled": true + }, + { + "name": "get_news", + "capability": "research.news", + "enabled": true + }, + { + "name": "get_portfolio", + "capability": "portfolio.summary", + "enabled": true + }, + { + "name": "get_market_depth", + "capability": "market.depth", + "enabled": true + }, + { + "name": "get_trades", + "capability": "market.trades", + "enabled": true + }, + { + "name": "get_capital_flow", + "capability": "market.capitalFlow", + "enabled": true + }, + { + "name": "get_market_sentiment", + "capability": "market.sentiment", + "enabled": true + }, + { + "name": "get_financials", + "capability": "company.financials", + "enabled": true + }, + { + "name": "get_ratings", + "capability": "company.ratings", + "enabled": true + }, + { + "name": "get_dividends", + "capability": "company.dividends", + "enabled": true + }, + { + "name": "get_earnings", + "capability": "company.earnings", + "enabled": true + }, + { + "name": "get_calendar_events", + "capability": "research.events", + "enabled": true + }, + { + "name": "get_positions", + "capability": "portfolio.positions", + "enabled": true + }, + { + "name": "get_assets", + "capability": "portfolio.assets", + "enabled": true + }, + { + "name": "get_cash_flow", + "capability": "portfolio.cashFlow", + "enabled": true + } + ], + "search": { + "providerId": "public-http", + "routing": { + "primary": "public-http" + }, + "configured": true, + "endpoint": "https://query1.finance.yahoo.com" + }, + "featureFlags": { + "runtimeProvider": "pi", + "demoData": false, + "langfuseTracing": false + }, + "budget": { + "defaults": { + "modelCalls": 3 + }, + "effective": { + "modelCalls": 3 + }, + "clamped": [] + }, + "locale": "en-US" +} \ No newline at end of file diff --git a/docs/acceptance/run-manifest/run-b-manifest.json b/docs/acceptance/run-manifest/run-b-manifest.json new file mode 100644 index 00000000..0e5897ef --- /dev/null +++ b/docs/acceptance/run-manifest/run-b-manifest.json @@ -0,0 +1,151 @@ +{ + "schemaVersion": 1, + "runId": "research-cec9f6d1-9be6-41fd-9932-4692906d36ba", + "createdAt": 1790674674496, + "runtimeMode": "pi", + "appVersion": "0.5.0-acceptance", + "buildVersion": "0.5.0-acceptance", + "gitRevision": "402bd92f2c0b32dfed4b28b1038b67c2c0adf582", + "provider": "deepseek", + "model": "deepseek-v4-pro", + "modelParams": { + "api": "openai-completions", + "baseUrl": "https://api.deepseek.com", + "maxTokens": 8000, + "reasoning": false + }, + "prompt": { + "hash": "8105327bd4c6513d43687b00fe5d3e0cb9a62fd0131bf5ec103f2e423bd9656f", + "version": "deep-research-synthesis@1", + "source": "e2e-acceptance" + }, + "strategy": { + "id": "comprehensive", + "version": "3637a33db7d9979cf55ad2331734b197b0db5ce665f65e2dcc2f57675b3fd076" + }, + "tools": [ + { + "name": "get_quote", + "capability": "market.quote", + "enabled": true + }, + { + "name": "get_kline", + "capability": "market.kline", + "enabled": true + }, + { + "name": "get_intraday", + "capability": "market.intraday", + "enabled": true + }, + { + "name": "get_market_status", + "capability": "market.status", + "enabled": true + }, + { + "name": "get_company_profile", + "capability": "company.profile", + "enabled": true + }, + { + "name": "get_valuation", + "capability": "company.valuation", + "enabled": true + }, + { + "name": "get_news", + "capability": "research.news", + "enabled": true + }, + { + "name": "get_portfolio", + "capability": "portfolio.summary", + "enabled": true + }, + { + "name": "get_market_depth", + "capability": "market.depth", + "enabled": true + }, + { + "name": "get_trades", + "capability": "market.trades", + "enabled": true + }, + { + "name": "get_capital_flow", + "capability": "market.capitalFlow", + "enabled": true + }, + { + "name": "get_market_sentiment", + "capability": "market.sentiment", + "enabled": true + }, + { + "name": "get_financials", + "capability": "company.financials", + "enabled": true + }, + { + "name": "get_ratings", + "capability": "company.ratings", + "enabled": true + }, + { + "name": "get_dividends", + "capability": "company.dividends", + "enabled": true + }, + { + "name": "get_earnings", + "capability": "company.earnings", + "enabled": true + }, + { + "name": "get_calendar_events", + "capability": "research.events", + "enabled": true + }, + { + "name": "get_positions", + "capability": "portfolio.positions", + "enabled": true + }, + { + "name": "get_assets", + "capability": "portfolio.assets", + "enabled": true + }, + { + "name": "get_cash_flow", + "capability": "portfolio.cashFlow", + "enabled": true + } + ], + "search": { + "providerId": "public-http", + "routing": { + "primary": "public-http" + }, + "configured": true, + "endpoint": "https://query1.finance.yahoo.com" + }, + "featureFlags": { + "runtimeProvider": "pi", + "demoData": false, + "langfuseTracing": false + }, + "budget": { + "defaults": { + "modelCalls": 6 + }, + "effective": { + "modelCalls": 6 + }, + "clamped": [] + }, + "locale": "en-US" +} \ No newline at end of file diff --git a/docs/acceptance/run-manifest/run-info-after-restart.html b/docs/acceptance/run-manifest/run-info-after-restart.html new file mode 100644 index 00000000..91a51d29 --- /dev/null +++ b/docs/acceptance/run-manifest/run-info-after-restart.html @@ -0,0 +1,180 @@ + + + + +Run Info — 重启后 + + + + + + +

Run Info — 应用重启后读取历史 run(#21)

读取进程的「当前全局设置」model = deepseek-v4-pro;历史 run 的 manifest model = deepseek-flash(仍为诞生时的配置)。

Run A · research-e95ce1be-6eb9-479c-96dd-668d359c631e
Run Info 视图(ManifestView)— 重启后读到的仍是该 run 自己的 manifest
Pi 运行时deepseek · deepseek-flash
模型deepseek-flash
Providerdeepseek
模型参数api=openai-completions · baseUrl=https://api.deepseek.com · max=8000 · reasoning=false
策略comprehensive · 3637a33db7d9979cf55ad2331734b197b0db5ce665f65e2dcc2f57675b3fd076
提示词版本deep-research-synthesis@1
提示词哈希8105327bd4c6513d43687b00fe5d3e0cb9a62fd0131bf5ec103f2e423bd9656f
提示词来源e2e-acceptance
工具get_quote, get_kline, get_intraday, get_market_status, get_company_profile, get_valuation, get_news, get_portfolio, get_market_depth, get_trades, get_capital_flow, get_market_sentiment, get_financials, get_ratings, get_dividends, get_earnings, get_calendar_events, get_positions, get_assets, get_cash_flow
检索public-http(已配置)
检索路由public-http → —
App 版本0.5.0-acceptance
构建版本0.5.0-acceptance
Git 修订402bd92f2c0b32dfed4b28b1038b67c2c0adf582
语言en-US
功能开关runtimeProvider=pi · demoData=false · langfuseTracing=false
JSON 导出(已脱敏)
{
+  "schemaVersion": 1,
+  "runId": "research-e95ce1be-6eb9-479c-96dd-668d359c631e",
+  "createdAt": 1790674646605,
+  "runtimeMode": "pi",
+  "appVersion": "0.5.0-acceptance",
+  "buildVersion": "0.5.0-acceptance",
+  "gitRevision": "402bd92f2c0b32dfed4b28b1038b67c2c0adf582",
+  "provider": "deepseek",
+  "model": "deepseek-flash",
+  "modelParams": {
+    "api": "openai-completions",
+    "baseUrl": "https://api.deepseek.com",
+    "maxTokens": 8000,
+    "reasoning": false
+  },
+  "prompt": {
+    "hash": "8105327bd4c6513d43687b00fe5d3e0cb9a62fd0131bf5ec103f2e423bd9656f",
+    "version": "deep-research-synthesis@1",
+    "source": "e2e-acceptance"
+  },
+  "strategy": {
+    "id": "comprehensive",
+    "version": "3637a33db7d9979cf55ad2331734b197b0db5ce665f65e2dcc2f57675b3fd076"
+  },
+  "tools": [
+    {
+      "name": "get_quote",
+      "capability": "market.quote",
+      "enabled": true
+    },
+    {
+      "name": "get_kline",
+      "capability": "market.kline",
+      "enabled": true
+    },
+    {
+      "name": "get_intraday",
+      "capability": "market.intraday",
+      "enabled": true
+    },
+    {
+      "name": "get_market_status",
+      "capability": "market.status",
+      "enabled": true
+    },
+    {
+      "name": "get_company_profile",
+      "capability": "company.profile",
+      "enabled": true
+    },
+    {
+      "name": "get_valuation",
+      "capability": "company.valuation",
+      "enabled": true
+    },
+    {
+      "name": "get_news",
+      "capability": "research.news",
+      "enabled": true
+    },
+    {
+      "name": "get_portfolio",
+      "capability": "portfolio.summary",
+      "enabled": true
+    },
+    {
+      "name": "get_market_depth",
+      "capability": "market.depth",
+      "enabled": true
+    },
+    {
+      "name": "get_trades",
+      "capability": "market.trades",
+      "enabled": true
+    },
+    {
+      "name": "get_capital_flow",
+      "capability": "market.capitalFlow",
+      "enabled": true
+    },
+    {
+      "name": "get_market_sentiment",
+      "capability": "market.sentiment",
+      "enabled": true
+    },
+    {
+      "name": "get_financials",
+      "capability": "company.financials",
+      "enabled": true
+    },
+    {
+      "name": "get_ratings",
+      "capability": "company.ratings",
+      "enabled": true
+    },
+    {
+      "name": "get_dividends",
+      "capability": "company.dividends",
+      "enabled": true
+    },
+    {
+      "name": "get_earnings",
+      "capability": "company.earnings",
+      "enabled": true
+    },
+    {
+      "name": "get_calendar_events",
+      "capability": "research.events",
+      "enabled": true
+    },
+    {
+      "name": "get_positions",
+      "capability": "portfolio.positions",
+      "enabled": true
+    },
+    {
+      "name": "get_assets",
+      "capability": "portfolio.assets",
+      "enabled": true
+    },
+    {
+      "name": "get_cash_flow",
+      "capability": "portfolio.cashFlow",
+      "enabled": true
+    }
+  ],
+  "search": {
+    "providerId": "public-http",
+    "routing": {
+      "primary": "public-http"
+    },
+    "configured": true,
+    "endpoint": "https://query1.finance.yahoo.com"
+  },
+  "featureFlags": {
+    "runtimeProvider": "pi",
+    "demoData": false,
+    "langfuseTracing": false
+  },
+  "budget": {
+    "defaults": {
+      "modelCalls": 3
+    },
+    "effective": {
+      "modelCalls": 3
+    },
+    "clamped": []
+  },
+  "locale": "en-US"
+}
+ + diff --git a/docs/acceptance/run-manifest/run-info-after-restart.json b/docs/acceptance/run-manifest/run-info-after-restart.json new file mode 100644 index 00000000..beee2687 --- /dev/null +++ b/docs/acceptance/run-manifest/run-info-after-restart.json @@ -0,0 +1,157 @@ +{ + "readIn": "fresh process (simulated app restart)", + "currentGlobalModel": "deepseek-v4-pro", + "researchRunId": "research-e95ce1be-6eb9-479c-96dd-668d359c631e", + "status": "partial", + "manifest": { + "schemaVersion": 1, + "runId": "research-e95ce1be-6eb9-479c-96dd-668d359c631e", + "createdAt": 1790674646605, + "runtimeMode": "pi", + "appVersion": "0.5.0-acceptance", + "buildVersion": "0.5.0-acceptance", + "gitRevision": "402bd92f2c0b32dfed4b28b1038b67c2c0adf582", + "provider": "deepseek", + "model": "deepseek-flash", + "modelParams": { + "api": "openai-completions", + "baseUrl": "https://api.deepseek.com", + "maxTokens": 8000, + "reasoning": false + }, + "prompt": { + "hash": "8105327bd4c6513d43687b00fe5d3e0cb9a62fd0131bf5ec103f2e423bd9656f", + "version": "deep-research-synthesis@1", + "source": "e2e-acceptance" + }, + "strategy": { + "id": "comprehensive", + "version": "3637a33db7d9979cf55ad2331734b197b0db5ce665f65e2dcc2f57675b3fd076" + }, + "tools": [ + { + "name": "get_quote", + "capability": "market.quote", + "enabled": true + }, + { + "name": "get_kline", + "capability": "market.kline", + "enabled": true + }, + { + "name": "get_intraday", + "capability": "market.intraday", + "enabled": true + }, + { + "name": "get_market_status", + "capability": "market.status", + "enabled": true + }, + { + "name": "get_company_profile", + "capability": "company.profile", + "enabled": true + }, + { + "name": "get_valuation", + "capability": "company.valuation", + "enabled": true + }, + { + "name": "get_news", + "capability": "research.news", + "enabled": true + }, + { + "name": "get_portfolio", + "capability": "portfolio.summary", + "enabled": true + }, + { + "name": "get_market_depth", + "capability": "market.depth", + "enabled": true + }, + { + "name": "get_trades", + "capability": "market.trades", + "enabled": true + }, + { + "name": "get_capital_flow", + "capability": "market.capitalFlow", + "enabled": true + }, + { + "name": "get_market_sentiment", + "capability": "market.sentiment", + "enabled": true + }, + { + "name": "get_financials", + "capability": "company.financials", + "enabled": true + }, + { + "name": "get_ratings", + "capability": "company.ratings", + "enabled": true + }, + { + "name": "get_dividends", + "capability": "company.dividends", + "enabled": true + }, + { + "name": "get_earnings", + "capability": "company.earnings", + "enabled": true + }, + { + "name": "get_calendar_events", + "capability": "research.events", + "enabled": true + }, + { + "name": "get_positions", + "capability": "portfolio.positions", + "enabled": true + }, + { + "name": "get_assets", + "capability": "portfolio.assets", + "enabled": true + }, + { + "name": "get_cash_flow", + "capability": "portfolio.cashFlow", + "enabled": true + } + ], + "search": { + "providerId": "public-http", + "routing": { + "primary": "public-http" + }, + "configured": true, + "endpoint": "https://query1.finance.yahoo.com" + }, + "featureFlags": { + "runtimeProvider": "pi", + "demoData": false, + "langfuseTracing": false + }, + "budget": { + "defaults": { + "modelCalls": 3 + }, + "effective": { + "modelCalls": 3 + }, + "clamped": [] + }, + "locale": "en-US" + } +} \ No newline at end of file diff --git a/docs/acceptance/run-manifest/run-info-compare.html b/docs/acceptance/run-manifest/run-info-compare.html new file mode 100644 index 00000000..8824d16d --- /dev/null +++ b/docs/acceptance/run-manifest/run-info-compare.html @@ -0,0 +1,330 @@ + + + + +Run 对比 + + + + + + +

两次真实 Deep Research run 的 manifest 与 diff(#21)

刻意改动 model 与预算(#17);diff 应精确只反映这两处变化,prompt 未变则不报差异。

Run A · research-e95ce1be-6eb9-479c-96dd-668d359c631e
model = deepseek-flash
Pi 运行时deepseek · deepseek-flash
模型deepseek-flash
Providerdeepseek
模型参数api=openai-completions · baseUrl=https://api.deepseek.com · max=8000 · reasoning=false
策略comprehensive · 3637a33db7d9979cf55ad2331734b197b0db5ce665f65e2dcc2f57675b3fd076
提示词版本deep-research-synthesis@1
提示词哈希8105327bd4c6513d43687b00fe5d3e0cb9a62fd0131bf5ec103f2e423bd9656f
提示词来源e2e-acceptance
工具get_quote, get_kline, get_intraday, get_market_status, get_company_profile, get_valuation, get_news, get_portfolio, get_market_depth, get_trades, get_capital_flow, get_market_sentiment, get_financials, get_ratings, get_dividends, get_earnings, get_calendar_events, get_positions, get_assets, get_cash_flow
检索public-http(已配置)
检索路由public-http → —
App 版本0.5.0-acceptance
构建版本0.5.0-acceptance
Git 修订402bd92f2c0b32dfed4b28b1038b67c2c0adf582
语言en-US
功能开关runtimeProvider=pi · demoData=false · langfuseTracing=false
JSON 导出(已脱敏)
{
+  "schemaVersion": 1,
+  "runId": "research-e95ce1be-6eb9-479c-96dd-668d359c631e",
+  "createdAt": 1790674646605,
+  "runtimeMode": "pi",
+  "appVersion": "0.5.0-acceptance",
+  "buildVersion": "0.5.0-acceptance",
+  "gitRevision": "402bd92f2c0b32dfed4b28b1038b67c2c0adf582",
+  "provider": "deepseek",
+  "model": "deepseek-flash",
+  "modelParams": {
+    "api": "openai-completions",
+    "baseUrl": "https://api.deepseek.com",
+    "maxTokens": 8000,
+    "reasoning": false
+  },
+  "prompt": {
+    "hash": "8105327bd4c6513d43687b00fe5d3e0cb9a62fd0131bf5ec103f2e423bd9656f",
+    "version": "deep-research-synthesis@1",
+    "source": "e2e-acceptance"
+  },
+  "strategy": {
+    "id": "comprehensive",
+    "version": "3637a33db7d9979cf55ad2331734b197b0db5ce665f65e2dcc2f57675b3fd076"
+  },
+  "tools": [
+    {
+      "name": "get_quote",
+      "capability": "market.quote",
+      "enabled": true
+    },
+    {
+      "name": "get_kline",
+      "capability": "market.kline",
+      "enabled": true
+    },
+    {
+      "name": "get_intraday",
+      "capability": "market.intraday",
+      "enabled": true
+    },
+    {
+      "name": "get_market_status",
+      "capability": "market.status",
+      "enabled": true
+    },
+    {
+      "name": "get_company_profile",
+      "capability": "company.profile",
+      "enabled": true
+    },
+    {
+      "name": "get_valuation",
+      "capability": "company.valuation",
+      "enabled": true
+    },
+    {
+      "name": "get_news",
+      "capability": "research.news",
+      "enabled": true
+    },
+    {
+      "name": "get_portfolio",
+      "capability": "portfolio.summary",
+      "enabled": true
+    },
+    {
+      "name": "get_market_depth",
+      "capability": "market.depth",
+      "enabled": true
+    },
+    {
+      "name": "get_trades",
+      "capability": "market.trades",
+      "enabled": true
+    },
+    {
+      "name": "get_capital_flow",
+      "capability": "market.capitalFlow",
+      "enabled": true
+    },
+    {
+      "name": "get_market_sentiment",
+      "capability": "market.sentiment",
+      "enabled": true
+    },
+    {
+      "name": "get_financials",
+      "capability": "company.financials",
+      "enabled": true
+    },
+    {
+      "name": "get_ratings",
+      "capability": "company.ratings",
+      "enabled": true
+    },
+    {
+      "name": "get_dividends",
+      "capability": "company.dividends",
+      "enabled": true
+    },
+    {
+      "name": "get_earnings",
+      "capability": "company.earnings",
+      "enabled": true
+    },
+    {
+      "name": "get_calendar_events",
+      "capability": "research.events",
+      "enabled": true
+    },
+    {
+      "name": "get_positions",
+      "capability": "portfolio.positions",
+      "enabled": true
+    },
+    {
+      "name": "get_assets",
+      "capability": "portfolio.assets",
+      "enabled": true
+    },
+    {
+      "name": "get_cash_flow",
+      "capability": "portfolio.cashFlow",
+      "enabled": true
+    }
+  ],
+  "search": {
+    "providerId": "public-http",
+    "routing": {
+      "primary": "public-http"
+    },
+    "configured": true,
+    "endpoint": "https://query1.finance.yahoo.com"
+  },
+  "featureFlags": {
+    "runtimeProvider": "pi",
+    "demoData": false,
+    "langfuseTracing": false
+  },
+  "budget": {
+    "defaults": {
+      "modelCalls": 3
+    },
+    "effective": {
+      "modelCalls": 3
+    },
+    "clamped": []
+  },
+  "locale": "en-US"
+}
Run B · research-cec9f6d1-9be6-41fd-9932-4692906d36ba
model = deepseek-v4-pro
Pi 运行时deepseek · deepseek-v4-pro
模型deepseek-v4-pro
Providerdeepseek
模型参数api=openai-completions · baseUrl=https://api.deepseek.com · max=8000 · reasoning=false
策略comprehensive · 3637a33db7d9979cf55ad2331734b197b0db5ce665f65e2dcc2f57675b3fd076
提示词版本deep-research-synthesis@1
提示词哈希8105327bd4c6513d43687b00fe5d3e0cb9a62fd0131bf5ec103f2e423bd9656f
提示词来源e2e-acceptance
工具get_quote, get_kline, get_intraday, get_market_status, get_company_profile, get_valuation, get_news, get_portfolio, get_market_depth, get_trades, get_capital_flow, get_market_sentiment, get_financials, get_ratings, get_dividends, get_earnings, get_calendar_events, get_positions, get_assets, get_cash_flow
检索public-http(已配置)
检索路由public-http → —
App 版本0.5.0-acceptance
构建版本0.5.0-acceptance
Git 修订402bd92f2c0b32dfed4b28b1038b67c2c0adf582
语言en-US
功能开关runtimeProvider=pi · demoData=false · langfuseTracing=false
JSON 导出(已脱敏)
{
+  "schemaVersion": 1,
+  "runId": "research-cec9f6d1-9be6-41fd-9932-4692906d36ba",
+  "createdAt": 1790674674496,
+  "runtimeMode": "pi",
+  "appVersion": "0.5.0-acceptance",
+  "buildVersion": "0.5.0-acceptance",
+  "gitRevision": "402bd92f2c0b32dfed4b28b1038b67c2c0adf582",
+  "provider": "deepseek",
+  "model": "deepseek-v4-pro",
+  "modelParams": {
+    "api": "openai-completions",
+    "baseUrl": "https://api.deepseek.com",
+    "maxTokens": 8000,
+    "reasoning": false
+  },
+  "prompt": {
+    "hash": "8105327bd4c6513d43687b00fe5d3e0cb9a62fd0131bf5ec103f2e423bd9656f",
+    "version": "deep-research-synthesis@1",
+    "source": "e2e-acceptance"
+  },
+  "strategy": {
+    "id": "comprehensive",
+    "version": "3637a33db7d9979cf55ad2331734b197b0db5ce665f65e2dcc2f57675b3fd076"
+  },
+  "tools": [
+    {
+      "name": "get_quote",
+      "capability": "market.quote",
+      "enabled": true
+    },
+    {
+      "name": "get_kline",
+      "capability": "market.kline",
+      "enabled": true
+    },
+    {
+      "name": "get_intraday",
+      "capability": "market.intraday",
+      "enabled": true
+    },
+    {
+      "name": "get_market_status",
+      "capability": "market.status",
+      "enabled": true
+    },
+    {
+      "name": "get_company_profile",
+      "capability": "company.profile",
+      "enabled": true
+    },
+    {
+      "name": "get_valuation",
+      "capability": "company.valuation",
+      "enabled": true
+    },
+    {
+      "name": "get_news",
+      "capability": "research.news",
+      "enabled": true
+    },
+    {
+      "name": "get_portfolio",
+      "capability": "portfolio.summary",
+      "enabled": true
+    },
+    {
+      "name": "get_market_depth",
+      "capability": "market.depth",
+      "enabled": true
+    },
+    {
+      "name": "get_trades",
+      "capability": "market.trades",
+      "enabled": true
+    },
+    {
+      "name": "get_capital_flow",
+      "capability": "market.capitalFlow",
+      "enabled": true
+    },
+    {
+      "name": "get_market_sentiment",
+      "capability": "market.sentiment",
+      "enabled": true
+    },
+    {
+      "name": "get_financials",
+      "capability": "company.financials",
+      "enabled": true
+    },
+    {
+      "name": "get_ratings",
+      "capability": "company.ratings",
+      "enabled": true
+    },
+    {
+      "name": "get_dividends",
+      "capability": "company.dividends",
+      "enabled": true
+    },
+    {
+      "name": "get_earnings",
+      "capability": "company.earnings",
+      "enabled": true
+    },
+    {
+      "name": "get_calendar_events",
+      "capability": "research.events",
+      "enabled": true
+    },
+    {
+      "name": "get_positions",
+      "capability": "portfolio.positions",
+      "enabled": true
+    },
+    {
+      "name": "get_assets",
+      "capability": "portfolio.assets",
+      "enabled": true
+    },
+    {
+      "name": "get_cash_flow",
+      "capability": "portfolio.cashFlow",
+      "enabled": true
+    }
+  ],
+  "search": {
+    "providerId": "public-http",
+    "routing": {
+      "primary": "public-http"
+    },
+    "configured": true,
+    "endpoint": "https://query1.finance.yahoo.com"
+  },
+  "featureFlags": {
+    "runtimeProvider": "pi",
+    "demoData": false,
+    "langfuseTracing": false
+  },
+  "budget": {
+    "defaults": {
+      "modelCalls": 6
+    },
+    "effective": {
+      "modelCalls": 6
+    },
+    "clamped": []
+  },
+  "locale": "en-US"
+}
Run 对比(DiffView)
3 处差异 · groups={"model":true,"prompt":false,"tools":false,"config":true,"versions":false}
两次运行配置存在差异
模型配置
budget.defaults.modelCalls
3→6
budget.effective.modelCalls
3→6
model
deepseek-flash→deepseek-v4-pro
+ + diff --git a/docs/acceptance/run-manifest/run-info.txt b/docs/acceptance/run-manifest/run-info.txt new file mode 100644 index 00000000..98fafb25 --- /dev/null +++ b/docs/acceptance/run-manifest/run-info.txt @@ -0,0 +1,18 @@ +Run Info — Run A (read back in a fresh process after "restart") + runtime mode : pi + provider : deepseek + model : deepseek-flash + strategy : comprehensive @ 3637a33db7d9979cf55ad2331734b197b0db5ce665f65e2dcc2f57675b3fd076 + prompt : deep-research-synthesis@1 (8105327bd4c6513d43687b00fe5d3e0cb9a62fd0131bf5ec103f2e423bd9656f) + tools : get_quote, get_kline, get_intraday, get_market_status, get_company_profile, get_valuation, get_news, get_portfolio, get_market_depth, get_trades, get_capital_flow, get_market_sentiment, get_financials, get_ratings, get_dividends, get_earnings, get_calendar_events, get_positions, get_assets, get_cash_flow + budget : effective={"modelCalls":3} + app version : 0.5.0-acceptance + git revision : 402bd92f2c0b32dfed4b28b1038b67c2c0adf582 + current globals in the reading process: model = deepseek-v4-pro + +Two-run diff + budget.defaults.modelCalls: 3 -> 6 + budget.effective.modelCalls: 3 -> 6 + model: "deepseek-flash" -> "deepseek-v4-pro" + +changed=true groups={"model":true,"prompt":false,"tools":false,"config":true,"versions":false} \ No newline at end of file diff --git a/docs/acceptance/run-manifest/verification.json b/docs/acceptance/run-manifest/verification.json new file mode 100644 index 00000000..20d31365 --- /dev/null +++ b/docs/acceptance/run-manifest/verification.json @@ -0,0 +1,107 @@ +{ + "kind": "live", + "transport": "real DeepSeek HTTP streaming; real public HTTP retrieval", + "retrievalSources": { + "quote": "https://query1.finance.yahoo.com/v8/finance/chart/", + "kline": "https://query1.finance.yahoo.com/v8/finance/chart/", + "profile": "https://query1.finance.yahoo.com/v8/finance/chart/", + "news": "https://rss.nytimes.com/services/xml/rss/nyt/Business.xml", + "substitutedFor": "Longbridge CLI (not installed on this machine)" + }, + "symbol": "NVDA.US", + "strategy": "comprehensive", + "runs": [ + { + "label": "run-a", + "researchRunId": "research-e95ce1be-6eb9-479c-96dd-668d359c631e", + "status": "partial", + "requestedModel": "deepseek-flash", + "manifestModel": "deepseek-flash", + "manifestRunId": "research-e95ce1be-6eb9-479c-96dd-668d359c631e", + "completedCapabilities": [ + "market.quote", + "market.kline", + "company.profile", + "research.news" + ], + "failedCapabilities": [ + "market.intraday", + "market.depth", + "market.trades", + "market.capitalFlow", + "market.sentiment", + "market.status", + "company.valuation", + "company.financials", + "company.dividends", + "company.earnings", + "company.ratings", + "research.events" + ] + }, + { + "label": "run-b", + "researchRunId": "research-cec9f6d1-9be6-41fd-9932-4692906d36ba", + "status": "partial", + "requestedModel": "deepseek-v4-pro", + "manifestModel": "deepseek-v4-pro", + "manifestRunId": "research-cec9f6d1-9be6-41fd-9932-4692906d36ba", + "completedCapabilities": [ + "market.kline", + "market.quote", + "company.profile", + "research.news" + ], + "failedCapabilities": [ + "market.intraday", + "market.depth", + "market.trades", + "market.capitalFlow", + "market.sentiment", + "market.status", + "company.valuation", + "company.financials", + "company.dividends", + "company.earnings", + "company.ratings", + "research.events" + ] + } + ], + "deliberateChange": { + "model": { + "before": "deepseek-flash", + "after": "deepseek-v4-pro" + }, + "budgetModelCalls": { + "before": 3, + "after": 6 + } + }, + "diff": { + "changed": true, + "groups": { + "model": true, + "prompt": false, + "tools": false, + "config": true, + "versions": false + }, + "fieldCount": 3, + "paths": [ + "budget.defaults.modelCalls", + "budget.effective.modelCalls", + "model" + ] + }, + "restartReadback": { + "process": "fresh process", + "currentGlobalModel": "deepseek-v4-pro", + "historicalModel": "deepseek-flash", + "historicalRunId": "research-e95ce1be-6eb9-479c-96dd-668d359c631e", + "preserved": true + }, + "secretScan": { + "apiKeyPresentInManifests": false + } +} \ No newline at end of file diff --git a/docs/run-manifest-test-report.md b/docs/run-manifest-test-report.md new file mode 100644 index 00000000..3c24923a --- /dev/null +++ b/docs/run-manifest-test-report.md @@ -0,0 +1,212 @@ +# Run manifest acceptance report (#21) + +## Evidence revision — read this first + +The artifacts in `docs/acceptance/run-manifest/` were produced at revision +`402bd92f2c0b32dfed4b28b1038b67c2c0adf582`, when the acceptance script still injected an +**ad-hoc Yahoo chart fetcher plus the NYT Business RSS feed** through the capability-fetcher +seam. + +`apps/electron/e2e/run-manifest-acceptance.ts` has since been switched to the **shipped** +retrieval path (`YahooChartFinancialDataProvider` → `ProviderRouter` → `createRouterFetchers` +→ `createFullRegistry`, the same wiring `AgentKernelHost` uses) — see *Limits* below. The +script now also asserts `search.providerId` / `search.routing` and records the answering +provider, timestamp and `delayed` flag per run. + +**Those artifacts have not been regenerated.** Re-running the live acceptance needs a +DeepSeek API key, which was not available in the environment where this revision was +prepared. So: + +- The two acceptance items proven below — the selective manifest diff, and the post-restart + read-back that still reports the birth config — were measured on real runs and remain + valid: neither depends on which retrieval source answered. +- The retrieval-source rows (`completedCapabilities` including `research.news`, and the + provider-usage/wall-clock numbers) describe the **old** wiring. Under the new wiring + `research.news` is `unavailable` (the shipped Yahoo adapter is market-data only), so a + fresh run would report three completed capabilities, not four. +- Regenerating is a single command for anyone holding a key; the command is below. + +## Acceptance environment + +Date: 2026-09-29. Environment: Windows 11 Home China, NT 10.0.26200, Bun 1.4.2. +Validated revision: `402bd92f2c0b32dfed4b28b1038b67c2c0adf582` (this branch, `feat/run-manifest-21`). + +## The two acceptance items + +Issue #21 names two acceptance items that fixtures cannot satisfy: + +1. **Two real production Deep Research runs**, with a deliberately changed config, whose + manifest diff reflects exactly that change. +2. **A Run Info read-back after a restart** — a second, fresh process whose *current* + globals name a different model — where the historical run must still report the config + it was born with. + +Both are proven below by `apps/electron/e2e/run-manifest-acceptance.ts`, which uses real +components only (no fixtures). + +## Live two-run acceptance — PASS + +Actual command, from the repository root, after loading the local DeepSeek key into the +process environment: + +```sh +DEEPSEEK_API_KEY=... \ +FINAGENT_ACCEPTANCE_APP_VERSION=0.5.0-acceptance \ +FINAGENT_ACCEPTANCE_GIT_REVISION=402bd92f2c0b32dfed4b28b1038b67c2c0adf582 \ +bun apps/electron/e2e/run-manifest-acceptance.ts +``` + +The parent orchestrator spawns three child processes: `run run-a`, `run run-b`, and +`inspect deepseek-v4-pro`. Each child hosts the real `AgentKernel` and +`ResearchService` (the production research path), streams a real DeepSeek response, and +retrieves over the public internet through the shipped provider platform: the real +`YahooChartFinancialDataProvider` registered on a real `ProviderRouter`, whose fetchers are +handed to `createFullRegistry` — the same wiring `AgentKernelHost` uses. The capability +bodies — validation, sanitisation, summaries, provenance — run unchanged. + +Deliberate config change between the two runs: **model** `deepseek-flash` → `deepseek-v4-pro` +and **budget** `modelCalls` 3 → 6. Everything else (strategy, prompt, tools, search config, +app/build version, git revision) is held constant. + +| Evidence | Observed result | +| --- | --- | +| Run A id | research-e95ce1be-6eb9-479c-96dd-668d359c631e | +| Run B id | research-cec9f6d1-9be6-41fd-9932-4692906d36ba | +| Manifest run identity | manifest `runId` equals the research run id for both runs (trace identity is the run id, #14) | +| Run A requested vs manifest model | `deepseek-flash` = `deepseek-flash` | +| Run B requested vs manifest model | `deepseek-v4-pro` = `deepseek-v4-pro` | +| Run A budget | defaults/effective `{modelCalls: 3}`, `clamped: []` | +| Run B budget | defaults/effective `{modelCalls: 6}`, `clamped: []` | +| Strategy | `comprehensive @ 3637a33db7d9979cf55ad2331734b197b0db5ce665f65e2dcc2f57675b3fd076` (identical in both runs) | +| Prompt | `deep-research-synthesis@1`, hash `8105327b…bd9656f` (identical in both runs) | +| Tools | 20 enabled tools captured, identical in both runs | +| Completed capabilities | `market.quote`, `market.kline`, `company.profile`, `research.news` | +| Final status | `partial` — the 12 capabilities with no live equivalent are explicitly `unavailable`, never silently dropped | +| Run A provider usage | 6068 prompt + 5440 completion = 11508 tokens (128 cached, 3860 reasoning) | +| Run B provider usage | 6119 prompt + 3577 completion = 9696 tokens (384 cached, 2250 reasoning) | +| Wall clock | run A manifest written 17:37:26, run B 17:37:54, acceptance finished 17:38:43 → ≈1m18s | + +The provider echoed the requested model back as `apiModel` for both runs, so the configured +model is demonstrably the model that ran — the manifest is not recording an intention. + +## Two-run diff — PASS + +`diffRunManifests(runA, runB)` (executed in the main process, never in the renderer) reports +exactly the deliberate change and nothing else: + +``` +changed = true +groups = { model: true, prompt: false, tools: false, config: true, versions: false } +paths = ["budget.defaults.modelCalls", "budget.effective.modelCalls", "model"] +model : "deepseek-flash" -> "deepseek-v4-pro" +budget : 3 -> 6 +``` + +`prompt`, `tools` and `versions` stay `false` because those dimensions genuinely did not +change, which is the point: the diff is selective, not a blanket "something differs". + +## Run Info after a restart — PASS + +The third child is a **fresh process**. Its current globals name `deepseek-v4-pro`, but it +reads back the *historical* run A (born under `deepseek-flash`): + +| Evidence | Observed result | +| --- | --- | +| Reading process | fresh process (simulated app restart) | +| Current global model in that process | `deepseek-v4-pro` | +| Historical manifest model | `deepseek-flash` | +| Historical run id | research-e95ce1be-6eb9-479c-96dd-668d359c631e | +| Preserved | `true` | + +The manifest is append-only: the read-back process's different current globals do **not** +overwrite the historical run's record. + +## Secret scan — PASS + +The manifest schema has no credential field. `captureRunManifest` deep-copies and +redacts the snapshot **before it is persisted**, so the stored manifest, the IPC read +paths (`getRunManifest` / `getResearchManifest`) and `exportRunManifest` are all +credential-free by construction (#19, #21 review). Two independent checks agree: + +- The acceptance scans the three real manifests it captured — + `secretScan.apiKeyPresentInManifests` is computed from their serialised form, not + hard-coded. +- A grep for the live API key across all eleven committed artifacts returns **0** matches. + +## Committed artifacts + +- [verification.json](acceptance/run-manifest/verification.json) — machine-readable result of every assertion above +- [Run A manifest](acceptance/run-manifest/run-a-manifest.json) / [Run B manifest](acceptance/run-manifest/run-b-manifest.json) +- [Run A JSON export](acceptance/run-manifest/run-a-manifest-export.json) — the redacted export the UI's "Export JSON" produces +- [Two-run diff](acceptance/run-manifest/manifest-diff.json) +- [Run Info after restart](acceptance/run-manifest/run-info-after-restart.json) — the fresh-process read-back payload +- [Provider usage Run A](acceptance/run-manifest/model-usage-run-a.json) / [Run B](acceptance/run-manifest/model-usage-run-b.json) +- [Run Info text](acceptance/run-manifest/run-info.txt) +- [Run Info view (HTML)](acceptance/run-manifest/run-info-after-restart.html) +- [Two-run compare view (HTML)](acceptance/run-manifest/run-info-compare.html) + +The two HTML files are rendered by `apps/electron/e2e/render-run-info.ts` using the **shipped** +`ManifestView` / `DiffView` components from `packages/ui/src/components/kernel/RunInfoPanel.tsx` +via `renderToStaticMarkup`, with colours parsed out of the renderer's own `index.css` design +tokens. They are the view the app actually shows, not a re-implementation. + +## Automated checks + +Latest run (after rebasing onto `main` @ `9afa460` and adding the ordinary-read-path +redaction tests): + +| Executed command | Result | +| --- | --- | +| `bun test --isolate packages/shared/src/research packages/shared/src/kernel packages/shared/src/evaluation packages/ui/src/components/kernel apps/electron/src/main/kernelHost.test.ts` | 497 pass / 1 fail; 2261 assertions across 41 files | +| `bun test --isolate apps/electron/src/main/kernelHost.test.ts` | 29 pass / 0 fail; 67 assertions | +| `bun node_modules/typescript/bin/tsc --noEmit` | exit 0 | +| `bun node_modules/typescript/bin/tsc --noEmit -p ` | exit 0 for this branch's two e2e scripts | + +The single failure is `langfuse.test.ts > does not throw when Langfuse is down — agent path keeps +a diagnostic`. It is a **pre-existing local baseline failure**: it reproduces on a clean checkout +without this branch's changes (`git stash push -u`), and the file passes in CI. It is unrelated to +the manifest work. + +The two e2e typecheck errors are `quote-provenance-live-acceptance.ts:192` (`payload` on a +union) and `research-recovery-kernel.ts:139` (`ReadableStream` async iteration). Both exist on +`main` and are untouched by this branch; the repository's root `tsconfig.json` only includes +`packages/*/src` and `apps/*/src`, so the `e2e/` directory is not typechecked in CI. This +branch's two new e2e scripts typecheck without error under the same settings. + +## Limits + +Retrieval goes through the **shipped** provider platform, not a hand-rolled substitute. The +acceptance registers the real `YahooChartFinancialDataProvider` on a real `ProviderRouter`, +points the routing at it, and hands `createRouterFetchers(router)` to `createFullRegistry` — +the same wiring `AgentKernelHost` uses. Consequences: + +- The manifest records the provider and routing the run actually used: `search.providerId` + and `search.routing` are read from `router.getRouting()`, and the script asserts them. +- Every result carries the answering provider, its timestamp and the `delayed` flag. Each + run also re-probes a live quote through the same fetcher chain and stores + `providerId` / `timestamp` / `delayed` in the artifact, so "real data came back" is + evidenced rather than inferred from the wiring. +- Scope is exactly what the shipped adapter supports: US delayed quotes, daily/weekly + candles and a thin company profile (`market.quote` / `market.kline` / `company.profile`). + It is a no-key, **unofficial** chart endpoint — personal use, may rate-limit or change + without notice (see `docs/yahoo-finance.md`). +- Everything else — broker positions and holdings, financials, news, options, intraday — has + no provider on this machine. The router raises `UNSUPPORTED_CAPABILITY` and the run + records those capabilities as `unavailable`; nothing is reported as zero or as "no + events". That is why both runs finish `partial`. +- Longbridge is not installed here, but the in-scope market-data acceptance no longer + depends on it: quotes, candles and the profile come from a provider that ships with the + repository. + +This acceptance exercises the kernel / research-service persistence path and the manifest +assembly. It does **not** run the Electron IPC layer or the desktop Pi adapter — the manifest +plumbing on that path is covered by the `kernelHost` unit tests instead. It is not a +packaging test. + +The Run Info evidence is HTML, not a browser screenshot: this sandbox cannot launch a browser +or Electron (Mojo IPC channel creation and piped child-process creation both fail with access +denied / EPERM). Rendering the real components to static HTML is the closest faithful +substitute; it proves the view's content and layout but not pixel rendering in a live window. + +No API key, authorization code, or other secret is committed. The DeepSeek key was read from +the local developer environment at run time and never written to any artifact. diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 9028575e..a8138e45 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -3,6 +3,7 @@ import type { SupportedLocale } from './locale.ts'; import type { FinancialEvidenceEnvelope } from './financial-evidence.ts'; import type { MarketSource } from './market-data.ts'; +import type { RunManifest } from './run-manifest.ts'; export type { SupportedLocale, LocalePreference } from './locale.ts'; @@ -185,6 +186,13 @@ export interface Run { stopReason?: StopReason; /** The numbers behind a non-success stop (which budget ran out, which loop fired). */ stopDetail?: Record; + /** + * Immutable snapshot of the run's configuration, captured at creation (#21). + * Absent on records written before the manifest feature landed. Never + * overwritten by later global settings — re-opening a historical run shows + * the manifest it was born with. + */ + manifest?: RunManifest; } /** Live tool call state, streamed through agent events. */ @@ -608,3 +616,4 @@ export * from './financial-evidence.ts'; export * from './reconciliation.ts'; export * from './reconciliation-provider.ts'; export * from './evidence-contract.ts'; +export * from './run-manifest.ts'; diff --git a/packages/core/src/research.ts b/packages/core/src/research.ts index b9e54f26..3cdb2db7 100644 --- a/packages/core/src/research.ts +++ b/packages/core/src/research.ts @@ -1,6 +1,7 @@ import type { CapabilityProvenance, CapabilityRunStatus } from './capability.ts'; import type { SupportedLocale } from './locale.ts'; import type { StrategyId } from './strategy.ts'; +import type { RunManifest } from './run-manifest.ts'; /** * Research domain — Deep Research runs, evidence-backed reports, and the @@ -107,6 +108,12 @@ export interface ResearchRunSummary { recoveryCount?: number; recoverable?: boolean; error?: string; + /** + * Immutable snapshot of the run's configuration, captured at creation (#21). + * Absent on records written before the manifest feature landed. Never + * overwritten by later global settings. + */ + manifest?: RunManifest; } /** Non-secret identity pinned at the start of a research workflow. */ diff --git a/packages/core/src/run-manifest.ts b/packages/core/src/run-manifest.ts new file mode 100644 index 00000000..36fae42e --- /dev/null +++ b/packages/core/src/run-manifest.ts @@ -0,0 +1,200 @@ +/** + * Immutable run manifest (#21). + * + * A manifest is a machine-readable snapshot of *everything that influenced a + * single Agent / Deep Research run*: model + provider identity, the effective + * model params, the system/developer prompt composition, the agent/workflow + * strategy, the enabled tools, the search/retrieval provider + config, the + * active feature flags / runtime mode, the budget, and the app/build identity. + * + * It is captured once when the run is created and is **never overwritten by + * later global settings** — re-opening a historical run shows the manifest it + * was born with, not today's configuration. Secrets/credentials are never + * written into a manifest (see `redactManifestSecrets` in + * `@finagent/shared`). + */ + +/** Schema version for the manifest payload. Bump when the shape changes. */ +export const RUN_MANIFEST_SCHEMA_VERSION = 1 as const; + +/** Which agent runtime produced the run. */ +export type RunManifestRuntimeMode = 'local' | 'pi' | 'demo'; + +/** Effective model identity for the run. */ +export interface RunManifestIdentity { + provider?: string; + model?: string; +} + +/** Model params that were actually in effect for the run. */ +export interface RunManifestModelParams { + /** Provider API kind, e.g. `openai-completions`. */ + api?: string; + /** Resolved model base URL (no key — see redaction). */ + baseUrl?: string; + /** Custom provider endpoint, when configured (no key). */ + endpoint?: string; + /** Effective thinking/reasoning level. */ + thinkingLevel?: string; + contextWindow?: number; + maxTokens?: number; + reasoning?: boolean; + /** Reserved for provider-specific sampling knobs (never secrets). */ + sampling?: Record; +} + +/** System/developer prompt composition, traced to its source. */ +export interface RunManifestPrompt { + /** Stable hash of the prompt composition (enabled skills + template). */ + hash: string; + /** Human-readable template/version the prompt was built from. */ + version?: string; + /** Where the prompt template came from, e.g. `skill-index`. */ + source?: string; +} + +/** Agent / research workflow strategy that drove the run. */ +export interface RunManifestStrategy { + id?: string; + version?: string; +} + +/** + * Workflow id for the interactive Copilot agent path. Deep Research runs use + * the research strategy id (`StrategyId`) instead; both land in + * `RunManifestStrategy.id` so the manifest always names the workflow that ran. + */ +export const AGENT_WORKFLOW_ID = 'copilot-agent'; + +/** One enabled tool captured in the manifest. */ +export interface RunManifestTool { + name: string; + /** Capability id this tool is generated from, when known. */ + capability?: string; + /** Tool/extension version, when known. */ + version?: string; + enabled: boolean; +} + +/** Search / retrieval provider config at run time. */ +export interface RunManifestSearch { + /** Primary financial-data provider id (e.g. `longbridge`). */ + providerId?: string; + routing?: { primary?: string; fallback?: string }; + configured?: boolean; + /** Provider endpoint (no key). */ + endpoint?: string; +} + +/** Active feature flags / runtime mode toggles. */ +export type RunManifestFeatureFlags = Record; + +/** + * Budget that governed the run (#17). Records both what was *requested* + * (defaults / per-run overrides / system ceiling) and what the run *actually + * obeyed* (`effective`, after clamping) so a historical run can be explained + * without re-deriving the resolution rules of the day. + */ +export interface RunManifestBudget { + defaults?: Record; + ceiling?: Record; + overrides?: Record; + /** Effective limits the run obeyed: defaults/overrides clamped by the ceiling. */ + effective?: Record; + /** Budget keys whose requested value was cut down by the system ceiling. */ + clamped?: string[]; +} + +/** Evaluation context, when the run was part of an evaluation experiment (#15). */ +export interface RunManifestEvaluation { + datasetId?: string; + caseId?: string; + datasetVersion?: string; +} + +/** + * The full immutable run manifest. Every field is optional except the identity + * anchors (`schemaVersion`, `runId`, `createdAt`) so the manifest degrades + * gracefully when a piece of context is unavailable (e.g. local runtime, or an + * internal synthesis run). + */ +export interface RunManifest { + schemaVersion: number; + runId: string; + /** Run creation timestamp (ms epoch), mirrors `Run.startedAt`. */ + createdAt: number; + appVersion?: string; + buildVersion?: string; + /** Git commit / build revision, when available at build time. */ + gitRevision?: string; + runtimeMode: RunManifestRuntimeMode; + provider?: string; + model?: string; + modelParams?: RunManifestModelParams; + prompt?: RunManifestPrompt; + strategy?: RunManifestStrategy; + tools: RunManifestTool[]; + search?: RunManifestSearch; + featureFlags?: RunManifestFeatureFlags; + budget?: RunManifestBudget; + locale?: string; + evaluation?: RunManifestEvaluation; +} + +/** + * Assembled by the caller (the main-process host) at run start. The kernel + * merges it with the run identity anchors and persists the result. + * + * `runtimeMode` is required; everything else is optional so the capture works + * for internal/test runs that pass only a partial context. + */ +export interface RunManifestContext { + runtimeMode: RunManifestRuntimeMode; + appVersion?: string; + buildVersion?: string; + gitRevision?: string; + provider?: string; + model?: string; + modelParams?: RunManifestModelParams; + prompt?: RunManifestPrompt; + strategy?: RunManifestStrategy; + tools?: RunManifestTool[]; + search?: RunManifestSearch; + featureFlags?: RunManifestFeatureFlags; + budget?: RunManifestBudget; + locale?: string; + evaluation?: RunManifestEvaluation; +} + +/** + * Per-run override of the base manifest context (#21). Same shape as + * `RunManifestContext` but every field is optional: the base context supplied + * by the host fills in the runtime mode and the live settings, while a caller + * (e.g. the evaluation runner) only adds the fields it owns — such as the gold + * case / dataset identity. + */ +export type RunManifestContextPatch = Partial; + +/** A single resolved difference between two manifests. */ +export interface RunManifestFieldDiff { + /** Dotted path, e.g. `model`, `modelParams.thinkingLevel`, `prompt.hash`. */ + path: string; + before: unknown; + after: unknown; +} + +/** Structured diff between two run manifests. */ +export interface RunManifestDiff { + /** True when at least one field differs. */ + changed: boolean; + /** Per-field differences, in stable declaration order. */ + fields: RunManifestFieldDiff[]; + /** Group flags for the dimensions the issue cares about. */ + groups: { + model: boolean; + prompt: boolean; + tools: boolean; + config: boolean; + versions: boolean; + }; +} diff --git a/packages/shared/src/evaluation/experiment-service.test.ts b/packages/shared/src/evaluation/experiment-service.test.ts index c85d2a50..4bf8bfdd 100644 --- a/packages/shared/src/evaluation/experiment-service.test.ts +++ b/packages/shared/src/evaluation/experiment-service.test.ts @@ -20,6 +20,7 @@ import type { EvaluationMetricId, ExperimentConfig, Run, + RunManifestContextPatch, ToolCall, ToolCallRecord, } from '@finagent/core'; @@ -97,6 +98,8 @@ class FakeKernel implements ExperimentKernel { startedRuns = 0; /** When set, `startRun` rejects with this error (infra start failure). */ startRunError?: ApiError; + /** Manifest context handed to the most recent `startRun` (#21). */ + lastManifestContext?: RunManifestContextPatch; private readonly listeners = new Set<(event: AgentEvent) => void>(); private readonly sessionTitles = new Map(); private readonly scripts = new Map(); @@ -157,7 +160,14 @@ class FakeKernel implements ExperimentKernel { this.listeners.add(listener); return () => this.listeners.delete(listener); }, - startRun: async (sessionId: string, content: string): Promise => { + startRun: async ( + sessionId: string, + content: string, + _workspaceContext?: unknown, + _locale?: unknown, + manifestContext?: RunManifestContextPatch + ): Promise => { + this.lastManifestContext = manifestContext; if (this.startRunError) { throw Object.assign(new Error(this.startRunError.message), { code: this.startRunError.code }); } @@ -412,6 +422,23 @@ describe('ExperimentService.runExperiment', () => { expect(kernel.deletedSessions).toBe(1); }); + it('stamps the gold case and dataset version into the run manifest (#21 / #15)', async () => { + const dataset = makeDataset([makeCase('case-ok')]); + const kernel = new FakeKernel(); + scriptSuccess(kernel, 'case-ok'); + const service = createService(kernel); + + await service.runExperiment({ dataset, config: makeConfig() }); + + // The evaluation runner only adds what it owns; the model/prompt/tool config + // comes from the kernel's base manifest context. + expect(kernel.lastManifestContext?.evaluation).toEqual({ + datasetId: 'test-dataset', + caseId: 'case-ok', + datasetVersion: '1.0.0', + }); + }); + it('counts a timeout after execution starts as a negative quality result', async () => { const dataset = makeDataset([makeCase('case-pass'), makeCase('case-timeout')]); const kernel = new FakeKernel(); diff --git a/packages/shared/src/evaluation/experiment-service.ts b/packages/shared/src/evaluation/experiment-service.ts index b4fa0006..3d1a659d 100644 --- a/packages/shared/src/evaluation/experiment-service.ts +++ b/packages/shared/src/evaluation/experiment-service.ts @@ -33,6 +33,7 @@ import type { ExperimentMetadata, RegressionResult, Run, + RunManifestContextPatch, SupportedLocale, ToolCall, ToolCallRecord, @@ -84,7 +85,13 @@ export interface ExperimentKernel { sessionId: string, content: string, workspaceContext?: WorkspaceContext, - locale?: SupportedLocale + locale?: SupportedLocale, + /** + * Extra manifest context for this run (#21). Merged over the kernel's base + * context so an evaluation run records its gold case / dataset version in + * the same immutable manifest as the model/prompt/tool config. + */ + manifestContext?: RunManifestContextPatch ): Promise; /** True while a run's terminal persistence is still landing (AgentKernel). */ isRunning?(): boolean; @@ -562,7 +569,16 @@ export class ExperimentService { session.id, caseItem.input.prompt, caseItem.input.workspaceContext, - caseItem.locale + caseItem.locale, + // #21 / #15: the manifest records which gold case + dataset version + // this evaluation run belongs to, under the run's own identity. + { + evaluation: { + datasetId: dataset.id, + caseId: caseItem.id, + datasetVersion: dataset.version, + }, + } ); } catch (error) { // Infra-level failure (e.g. runtime spawn failed): record a failed run diff --git a/packages/shared/src/evaluation/langfuse/metadata.test.ts b/packages/shared/src/evaluation/langfuse/metadata.test.ts new file mode 100644 index 00000000..384cb9a4 --- /dev/null +++ b/packages/shared/src/evaluation/langfuse/metadata.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from 'bun:test'; +import type { RunManifest } from '@finagent/core'; +import { langfuseMetadataRecord, langfuseTags, manifestToLangfuseMetadata } from './metadata.ts'; + +function manifest(overrides: Partial = {}): RunManifest { + return { + schemaVersion: 1, + runId: 'run-1', + createdAt: 1_700_000_000_000, + runtimeMode: 'pi', + tools: [], + ...overrides, + }; +} + +describe('manifestToLangfuseMetadata (#21 ↔ #14)', () => { + it('joins the trace to the manifest run identity and carries its config', () => { + const meta = manifestToLangfuseMetadata( + manifest({ + provider: 'openai', + model: 'gpt-4o', + appVersion: '0.5.0', + locale: 'zh-CN', + strategy: { id: 'copilot-agent', version: '0.5.0' }, + prompt: { hash: 'abc', version: 'prompt-1', source: 'skill-index' }, + }), + { sessionId: 'sess-1', threadId: 'pi-thread-1' } + ); + + expect(meta).toMatchObject({ + // the manifest runId *is* the folioRunId the exporter uses as trace id + folioRunId: 'run-1', + folioSessionId: 'sess-1', + threadId: 'pi-thread-1', + runKind: 'normal', + model: 'gpt-4o', + provider: 'openai', + folioVersion: '0.5.0', + agentVersion: '0.5.0', + promptVersion: 'prompt-1', + strategyId: 'copilot-agent', + locale: 'zh-CN', + }); + expect(langfuseTags(meta)).toContain('strategy:copilot-agent'); + expect(langfuseTags(meta)).toContain('model:gpt-4o'); + }); + + it('derives an evaluation run kind from the manifest gold case / dataset', () => { + const meta = manifestToLangfuseMetadata( + manifest({ evaluation: { datasetId: 'folio-agent-v1', caseId: 'case-7', datasetVersion: '1.2.0' } }) + ); + + expect(meta.runKind).toBe('evaluation'); + expect(meta.datasetId).toBe('folio-agent-v1'); + expect(meta.goldCaseId).toBe('case-7'); + expect(meta.datasetVersion).toBe('1.2.0'); + const tags = langfuseTags(meta); + expect(tags).toContain('run_kind:evaluation'); + expect(tags).toContain('gold_case:case-7'); + expect(tags).toContain('dataset:folio-agent-v1@1.2.0'); + }); + + it('lets readback-confirmed values win while keeping requested ones separate (#114)', () => { + const meta = manifestToLangfuseMetadata(manifest({ model: 'manifest-model' }), { + model: 'readback-model', + provider: 'readback-provider', + requestedModel: 'requested-model', + requestedProvider: 'requested-provider', + }); + + expect(meta.model).toBe('readback-model'); + expect(meta.provider).toBe('readback-provider'); + expect(meta.requestedModel).toBe('requested-model'); + expect(meta.requestedProvider).toBe('requested-provider'); + // requested values must never masquerade as the model that actually ran + expect(langfuseTags(meta)).toContain('model:readback-model'); + expect(langfuseTags(meta)).not.toContain('model:requested-model'); + expect(langfuseMetadataRecord(meta)).toMatchObject({ + model: 'readback-model', + requestedModel: 'requested-model', + }); + }); + + it('falls back to the manifest model when no readback value is supplied', () => { + const meta = manifestToLangfuseMetadata(manifest({ model: 'manifest-model', provider: 'manifest-provider' })); + expect(meta.model).toBe('manifest-model'); + expect(meta.provider).toBe('manifest-provider'); + }); +}); diff --git a/packages/shared/src/evaluation/langfuse/metadata.ts b/packages/shared/src/evaluation/langfuse/metadata.ts index 819ff830..7bed6c56 100644 --- a/packages/shared/src/evaluation/langfuse/metadata.ts +++ b/packages/shared/src/evaluation/langfuse/metadata.ts @@ -5,6 +5,8 @@ // gold cases, and production Copilot runs can be sliced without parsing // free-text names. +import type { RunManifest } from '@finagent/core'; + export type LangfuseRunKind = 'normal' | 'evaluation'; export const LANGFUSE_DEFAULT_HOST = 'https://cloud.langfuse.com'; @@ -99,3 +101,65 @@ export function normalizeLangfuseHost(host: string | undefined): string { if (!trimmed) return LANGFUSE_DEFAULT_HOST; return trimmed.replace(/\/+$/, ''); } + +/** Call-site context that is not part of the run manifest itself. */ +export interface ManifestLangfuseExtras { + /** Folio session id (the trace's `sessionId`). */ + sessionId?: string; + /** Runtime/Pi session id (the trace's `threadId`). */ + threadId?: string; + /** Overrides the kind derived from the manifest's evaluation context. */ + runKind?: LangfuseRunKind; + /** Instrument symbol, when the run targeted one. */ + symbol?: string; + /** + * Runtime config confirmed by READBACK (#114). When supplied it wins over the + * manifest value, so a caller with fresher evidence is never overridden. + */ + model?: string; + provider?: string; + /** Requested (not confirmed) values, recorded under their own names. */ + requestedModel?: string; + requestedProvider?: string; +} + +/** + * Derive the Langfuse trace metadata from a run's immutable manifest (#21 ↔ #14). + * + * This is the single place the two identities are joined: the manifest's + * `runId` becomes `folioRunId`, which the exporter also uses as the Langfuse + * trace id, so a trace and its local manifest can always be matched. The + * manifest's captured config (model, provider, strategy, prompt version, gold + * case / dataset) fills the metadata fields the old hand-rolled call sites + * omitted. + * + * Only readback-confirmed values become `model`/`provider`; a manifest records + * the model that actually ran, so it is safe to promote. + */ +export function manifestToLangfuseMetadata( + manifest: RunManifest, + extras: ManifestLangfuseExtras = {} +): FolioLangfuseMetadata { + const evaluation = manifest.evaluation; + const runKind: LangfuseRunKind = + extras.runKind ?? (evaluation ? 'evaluation' : 'normal'); + return { + folioRunId: manifest.runId, + folioSessionId: extras.sessionId, + threadId: extras.threadId, + runKind, + goldCaseId: evaluation?.caseId, + datasetId: evaluation?.datasetId, + datasetVersion: evaluation?.datasetVersion, + model: extras.model ?? manifest.model, + provider: extras.provider ?? manifest.provider, + requestedModel: extras.requestedModel, + requestedProvider: extras.requestedProvider, + agentVersion: manifest.strategy?.version, + folioVersion: manifest.appVersion, + promptVersion: manifest.prompt?.version, + strategyId: manifest.strategy?.id, + symbol: extras.symbol, + locale: manifest.locale, + }; +} diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 0dcd0528..0569ffeb 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -118,6 +118,10 @@ export { type RunawayState, type RunawayStep, type StopReason, + captureRunManifest, + redactManifestSecrets, + diffRunManifests, + exportRunManifest, } from './kernel/index.ts'; export { LocalFinanceAgentBackend, diff --git a/packages/shared/src/kernel/agent-kernel.ts b/packages/shared/src/kernel/agent-kernel.ts index 03fb40e5..2a0a274c 100644 --- a/packages/shared/src/kernel/agent-kernel.ts +++ b/packages/shared/src/kernel/agent-kernel.ts @@ -1,5 +1,5 @@ import { join } from 'node:path'; -import type { AgentRuntime, ApiResult, ToolDefinition } from '@finagent/core'; +import type { AgentRuntime, ApiResult, RunManifestContext, ToolDefinition } from '@finagent/core'; import type { SkillHub } from '@finagent/skill-hub'; import { JsonFileStore } from '../storage/json-file-store.ts'; import { MessageRepository } from '../storage/message-repository.ts'; @@ -47,6 +47,13 @@ export interface AgentKernelOptions { searchTools?: string[]; /** Runaway detector thresholds; unset fields fall back to `defaultRunawayPolicy()`. */ runaway?: Partial; + /** + * Base manifest context for every run (#21). Wired by the host to its live + * settings so *all* run entry points — including evaluation runs started + * directly on the kernel — snapshot the same model/prompt/tool/search config + * instead of falling back to an empty manifest. + */ + getRunManifestContext?: () => RunManifestContext | Promise; } /** @@ -87,6 +94,11 @@ export class AgentKernel { runaway: options.runaway, // issue #75:与 kernel 存储同目录落盘流事件日志,支持跨重启 replay。 streamLogDir: join(options.storageDir, 'stream-events'), + // #21: fallback runtime mode for manifests when the host passes no context. + runtimeMode: options.provider === 'local' ? 'local' : 'pi', + // #21: the host's live settings, so evaluation runs started directly on + // the kernel still get a real manifest instead of the empty fallback. + getRunManifestContext: options.getRunManifestContext, }); } diff --git a/packages/shared/src/kernel/index.ts b/packages/shared/src/kernel/index.ts index 8b12b5c4..3b1f7192 100644 --- a/packages/shared/src/kernel/index.ts +++ b/packages/shared/src/kernel/index.ts @@ -37,3 +37,10 @@ export { type RunawayState, type RunawayStep, } from './runaway-detector.ts'; +export { + captureRunManifest, + redactManifestSecrets, + diffRunManifests, + exportRunManifest, + hashManifestInput, +} from './run-manifest.ts'; diff --git a/packages/shared/src/kernel/run-manager.test.ts b/packages/shared/src/kernel/run-manager.test.ts index 58e5e4db..527864c5 100644 --- a/packages/shared/src/kernel/run-manager.test.ts +++ b/packages/shared/src/kernel/run-manager.test.ts @@ -8,6 +8,7 @@ import type { AgentRunInput, AgentRuntime, ApiResult, + RunManifestContext, RuntimeSession, ToolDefinition, } from '@finagent/core'; @@ -771,6 +772,169 @@ describe('RunManager budgets and runaway detection (#17)', () => { }); }); +describe('RunManager run manifest (#21)', () => { + it('captures and persists an immutable manifest for a run', async () => { + const { sessions, runs } = makeKernel(completedScript('Done')); + const session = await sessions.createSession('Manifest'); + const run = await runs.startRun(session.id, 'hello', undefined, undefined, undefined, { + runtimeMode: 'pi', + provider: 'openai', + model: 'gpt-4o', + prompt: { hash: 'abc123', version: 'v1', source: 'skill-index' }, + tools: [{ name: 'get_portfolio', capability: 'portfolio.read', enabled: true }], + }); + await waitFor(async () => !runs.isRunning()); + const persisted = await sessions.getRun(session.id, run.id); + expect(persisted?.manifest).toMatchObject({ + schemaVersion: 1, + runId: run.id, + runtimeMode: 'pi', + provider: 'openai', + model: 'gpt-4o', + prompt: { hash: 'abc123', version: 'v1', source: 'skill-index' }, + tools: [{ name: 'get_portfolio', capability: 'portfolio.read', enabled: true }], + }); + }); + + it('persists a detached, credential-free snapshot even if the host context drifts', async () => { + // The host hands over *live* settings objects. The persisted manifest must + // be a frozen copy of that moment and must never carry a credential, even + // when a caller bug slips one into the context (#21 review). + const hostContext: RunManifestContext = { + runtimeMode: 'pi', + model: 'gpt-4o', + // @ts-expect-error intentional: simulate a caller bug + modelParams: { apiKey: 'sk-live', maxTokens: 4096 }, + search: { providerId: 'longbridge', routing: { primary: 'longbridge' } }, + }; + const { sessions, runs } = makeKernel(completedScript('Done'), { + getRunManifestContext: () => hostContext, + }); + const session = await sessions.createSession('Manifest snapshot'); + const run = await runs.startRun(session.id, 'hello'); + await waitFor(async () => !runs.isRunning()); + + // A later settings change mutates the very object the manifest was built from. + hostContext.model = 'gpt-4o-mini'; + (hostContext.search?.routing as Record).primary = 'massive'; + + const manifest = (await sessions.getRun(session.id, run.id))?.manifest; + expect(manifest?.model).toBe('gpt-4o'); + expect(manifest?.search?.routing?.primary).toBe('longbridge'); + expect((manifest?.modelParams as Record).apiKey).toBe('[redacted]'); + expect(manifest?.modelParams?.maxTokens).toBe(4096); + }); + + it('keeps each run manifest independent of later runs', async () => { + const { sessions, runs } = makeKernel(completedScript('Done')); + const session = await sessions.createSession('Manifest'); + const first = await runs.startRun(session.id, 'one', undefined, undefined, undefined, { + runtimeMode: 'pi', + model: 'gpt-4o', + }); + await waitFor(async () => !runs.isRunning()); + const second = await runs.startRun(session.id, 'two', undefined, undefined, undefined, { + runtimeMode: 'local', + model: 'llama-3', + }); + await waitFor(async () => !runs.isRunning()); + const persistedFirst = await sessions.getRun(session.id, first.id); + const persistedSecond = await sessions.getRun(session.id, second.id); + expect(persistedFirst?.manifest?.runId).toBe(first.id); + expect(persistedFirst?.manifest?.model).toBe('gpt-4o'); + expect(persistedSecond?.manifest?.runId).toBe(second.id); + expect(persistedSecond?.manifest?.model).toBe('llama-3'); + }); + + it('defaults to an empty tool list and the pi runtime when no context is given', async () => { + const { sessions, runs } = makeKernel(completedScript('Done')); + const session = await sessions.createSession('Manifest'); + const run = await runs.startRun(session.id, 'hello'); + await waitFor(async () => !runs.isRunning()); + const persisted = await sessions.getRun(session.id, run.id); + expect(persisted?.manifest?.runtimeMode).toBe('pi'); + expect(persisted?.manifest?.tools).toEqual([]); + expect(persisted?.manifest?.runId).toBe(run.id); + }); + + it('records the resolved budget, including the ceiling clamp, in the manifest (#17)', async () => { + const { sessions, runs } = makeKernel(completedScript('Done'), { + budgets: { defaults: { modelCalls: 10 }, ceiling: { modelCalls: 5, toolCalls: 4 } }, + }); + const session = await sessions.createSession('Manifest budget'); + const run = await runs.startRun(session.id, 'hello', undefined, undefined, { modelCalls: 3 }); + await waitFor(async () => !runs.isRunning()); + const manifest = (await sessions.getRun(session.id, run.id))?.manifest; + expect(manifest?.budget).toEqual({ + defaults: { modelCalls: 10 }, + ceiling: { modelCalls: 5, toolCalls: 4 }, + overrides: { modelCalls: 3 }, + effective: { modelCalls: 3, toolCalls: 4 }, + clamped: [], + }); + }); + + it('flags a limit the system ceiling cut down as clamped', async () => { + const { sessions, runs } = makeKernel(completedScript('Done'), { + budgets: { defaults: { modelCalls: 10 }, ceiling: { modelCalls: 5 } }, + }); + const session = await sessions.createSession('Manifest clamp'); + const run = await runs.startRun(session.id, 'hello'); + await waitFor(async () => !runs.isRunning()); + const manifest = (await sessions.getRun(session.id, run.id))?.manifest; + expect(manifest?.budget?.effective).toEqual({ modelCalls: 5 }); + expect(manifest?.budget?.clamped).toEqual(['modelCalls']); + }); + + it('omits the budget when no budget is configured', async () => { + const { sessions, runs } = makeKernel(completedScript('Done')); + const session = await sessions.createSession('Manifest no budget'); + const run = await runs.startRun(session.id, 'hello'); + await waitFor(async () => !runs.isRunning()); + const manifest = (await sessions.getRun(session.id, run.id))?.manifest; + expect(manifest?.budget).toBeUndefined(); + }); + + it('merges the base context from the host with a per-call override', async () => { + const { sessions, runs } = makeKernel(completedScript('Done'), { + getRunManifestContext: () => ({ + runtimeMode: 'pi', + model: 'gpt-4o', + strategy: { id: 'copilot-agent', version: '0.5.0' }, + tools: [{ name: 'get_quote', capability: 'market.quote', enabled: true }], + }), + }); + const session = await sessions.createSession('Manifest merge'); + const run = await runs.startRun(session.id, 'hello', undefined, undefined, undefined, { + evaluation: { datasetId: 'd1', caseId: 'c1', datasetVersion: '1.0.0' }, + }); + await waitFor(async () => !runs.isRunning()); + const manifest = (await sessions.getRun(session.id, run.id))?.manifest; + expect(manifest).toMatchObject({ + runtimeMode: 'pi', + model: 'gpt-4o', + strategy: { id: 'copilot-agent', version: '0.5.0' }, + tools: [{ name: 'get_quote', capability: 'market.quote', enabled: true }], + evaluation: { datasetId: 'd1', caseId: 'c1', datasetVersion: '1.0.0' }, + }); + }); + + it('degrades to the fallback context when the base source throws', async () => { + const { sessions, runs } = makeKernel(completedScript('Done'), { + runtimeMode: 'demo', + getRunManifestContext: () => { + throw new Error('settings unavailable'); + }, + }); + const session = await sessions.createSession('Manifest fallback'); + const run = await runs.startRun(session.id, 'hello'); + await waitFor(async () => !runs.isRunning()); + const manifest = (await sessions.getRun(session.id, run.id))?.manifest; + expect(manifest?.runtimeMode).toBe('demo'); + expect(manifest?.tools).toEqual([]); + }); +}); + async function waitFor(predicate: () => Promise, timeoutMs = 2000) { const started = Date.now(); while (!(await predicate())) { diff --git a/packages/shared/src/kernel/run-manager.ts b/packages/shared/src/kernel/run-manager.ts index c07e4fcd..9d931a51 100644 --- a/packages/shared/src/kernel/run-manager.ts +++ b/packages/shared/src/kernel/run-manager.ts @@ -6,6 +6,10 @@ import type { ApiError, Message, Run, + RunManifestBudget, + RunManifestContext, + RunManifestContextPatch, + RunManifestRuntimeMode, SessionMeta, StreamEvent, TokenUsage, @@ -38,6 +42,7 @@ import { type RunawayState, } from './runaway-detector.ts'; import { buildFinancialEvidence } from '../evidence/financial-evidence.ts'; +import { captureRunManifest } from './run-manifest.ts'; import { toStreamEvents } from './stream-event-adapter.ts'; import { StreamEventHistory, type StreamReplayResult } from './stream-history.ts'; import { StreamEventLog } from './stream-event-log.ts'; @@ -76,6 +81,21 @@ export interface RunManagerOptions { runaway?: Partial; /** Stream event log 目录(issue #75):持久化 replay 历史,跨重启可用;缺省为纯内存。 */ streamLogDir?: string; + /** + * Runtime mode used as the manifest fallback when a run is started without an + * explicit `manifestContext` (#21). The host always passes a full context + * (including `demo` when offline demo mode is on), so this only applies to + * internal/test runs that omit it. + */ + runtimeMode?: RunManifestRuntimeMode; + /** + * Base manifest context for every run that does not pass one explicitly + * (#21). The host wires this to its live settings so *all* run entry points — + * including evaluation runs started directly on the kernel — snapshot the + * same model/prompt/tool/search configuration instead of the empty fallback. + * A per-call `manifestContext` is merged on top of this base. + */ + getRunManifestContext?: () => RunManifestContext | Promise; } interface ActiveRun { @@ -126,6 +146,8 @@ export class RunManager { private readonly budgetInput: ResolveBudgetInput; private readonly searchToolPatterns: readonly string[]; private readonly runawayPolicy: Partial; + private readonly defaultRuntimeMode: RunManifestRuntimeMode; + private readonly getRunManifestContext?: () => RunManifestContext | Promise; private startingSessionId: string | null = null; private readonly listeners = new Set<(event: AgentEvent) => void>(); private readonly streamListeners = new Set<(sessionId: string, event: StreamEvent) => void>(); @@ -142,6 +164,8 @@ export class RunManager { this.budgetInput = options.budgets ?? {}; this.searchToolPatterns = options.searchTools ?? []; this.runawayPolicy = options.runaway ?? {}; + this.defaultRuntimeMode = options.runtimeMode ?? 'pi'; + this.getRunManifestContext = options.getRunManifestContext; // issue #75:可选持久化 —— 启动时从磁盘恢复历史,使 replay 跨重启可用; // 无目录或磁盘故障时降级为纯内存(实时链路不受影响)。 const streamLog = options.streamLogDir ? new StreamEventLog(options.streamLogDir) : undefined; @@ -198,7 +222,8 @@ export class RunManager { content: string, workspaceContext?: WorkspaceContext, locale?: SupportedLocale, - budgetOverrides?: RunBudgetLimits + budgetOverrides?: RunBudgetLimits, + manifestContext?: RunManifestContextPatch ): Promise { const text = content.trim(); if (!text) { @@ -215,18 +240,60 @@ export class RunManager { // A failed start must release the reservation so the caller can retry. this.startingSessionId = sessionId; try { - return await this.prepareRun(sessionId, text, workspaceContext, locale, budgetOverrides); + return await this.prepareRun(sessionId, text, workspaceContext, locale, budgetOverrides, manifestContext); } finally { this.startingSessionId = null; } } + /** + * Assemble the manifest context for a run (#21): the base context from + * `getRunManifestContext` (the host's live settings), overlaid with any + * per-call override, plus the *resolved* budget (#17) so the manifest records + * what the run actually obeyed. + * + * A base source that throws degrades to the fallback context rather than + * blocking the run (failure-isolation, spec §87). + */ + private async resolveManifestContext( + override: RunManifestContextPatch | undefined, + budgetOverrides: RunBudgetLimits | undefined, + limits: RunBudgetLimits, + clamped: string[] + ): Promise { + let base: RunManifestContext | undefined; + if (this.getRunManifestContext) { + try { + base = await this.getRunManifestContext(); + } catch { + base = undefined; + } + } + const fallback: RunManifestContext = { runtimeMode: this.defaultRuntimeMode, tools: [] }; + const merged: RunManifestContext = { ...fallback, ...base, ...override }; + + const { defaults, ceiling } = this.budgetInput; + const configured = defaults !== undefined || ceiling !== undefined || budgetOverrides !== undefined; + if (!configured && Object.keys(limits).length === 0) return merged; + + const budget: RunManifestBudget = { + defaults, + ceiling, + overrides: budgetOverrides, + effective: limits, + clamped, + }; + // The resolved budget is authoritative and always wins over a context-supplied one. + return { ...merged, budget }; + } + private async prepareRun( sessionId: string, text: string, workspaceContext?: WorkspaceContext, locale?: SupportedLocale, - budgetOverrides?: RunBudgetLimits + budgetOverrides?: RunBudgetLimits, + manifestContext?: RunManifestContextPatch ): Promise { const session = await this.sessions.getSession(sessionId); if (!session) { @@ -241,6 +308,22 @@ export class RunManager { input: text, startedAt: now, }; + // #17: resolve the effective budget *before* persisting the run so the + // manifest records exactly what the run will obey (defaults → per-run + // overrides → system ceiling). Resolving first also fails loud on an + // invalid limit before a half-created run is written. + const { limits, clamped } = resolveBudget({ + defaults: this.budgetInput.defaults, + ceiling: this.budgetInput.ceiling, + overrides: budgetOverrides, + }); + // #21: snapshot the run configuration at creation. This is the only place + // the manifest is written — `execute` never mutates it, so a historical run + // always reflects the settings it was born with, not today's globals. + run.manifest = captureRunManifest( + run, + await this.resolveManifestContext(manifestContext, budgetOverrides, limits, clamped) + ); await this.runs.create(run); const userMessage: Message = { @@ -252,11 +335,6 @@ export class RunManager { await this.sessions.appendMessage(sessionId, userMessage); await this.sessions.updateSession(sessionId, { status: 'running' }); - const { limits } = resolveBudget({ - defaults: this.budgetInput.defaults, - ceiling: this.budgetInput.ceiling, - overrides: budgetOverrides, - }); const active: ActiveRun = { sessionId, runId: run.id, diff --git a/packages/shared/src/kernel/run-manifest.test.ts b/packages/shared/src/kernel/run-manifest.test.ts new file mode 100644 index 00000000..826f14ee --- /dev/null +++ b/packages/shared/src/kernel/run-manifest.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, it } from 'bun:test'; +import type { Run, RunManifest, RunManifestContext, RunManifestTool } from '@finagent/core'; +import { + captureRunManifest, + diffRunManifests, + exportRunManifest, + hashManifestInput, + redactManifestSecrets, +} from './run-manifest.ts'; + +function makeRun(overrides: Partial = {}): Run { + return { + id: 'run-1', + sessionId: 'sess-1', + status: 'running', + input: 'hi', + startedAt: 1_700_000_000_000, + ...overrides, + }; +} + +function baseContext(): RunManifestContext { + return { + runtimeMode: 'pi', + appVersion: '0.4.0-beta.2', + provider: 'openai', + model: 'gpt-4o', + modelParams: { thinkingLevel: 'low', baseUrl: 'https://api.openai.com/v1' }, + prompt: { hash: 'abc', version: '0.4.0-beta.2', source: 'skill-index' }, + strategy: { id: 'comprehensive' }, + tools: [ + { name: 'market.quote', capability: 'market.quote', enabled: true }, + { name: 'get_news', capability: 'news.get', enabled: true }, + ], + search: { providerId: 'longbridge', routing: { primary: 'longbridge', fallback: 'massive' }, configured: true }, + featureFlags: { demoData: false, runtimeProvider: 'pi-runtime' }, + locale: 'en-US', + }; +} + +describe('captureRunManifest', () => { + it('snapshots the run identity anchors and the supplied context', () => { + const manifest = captureRunManifest(makeRun(), baseContext()); + expect(manifest.schemaVersion).toBe(1); + expect(manifest.runId).toBe('run-1'); + expect(manifest.createdAt).toBe(1_700_000_000_000); + expect(manifest.runtimeMode).toBe('pi'); + expect(manifest.provider).toBe('openai'); + expect(manifest.model).toBe('gpt-4o'); + expect(manifest.modelParams?.thinkingLevel).toBe('low'); + expect(manifest.prompt?.hash).toBe('abc'); + expect(manifest.strategy?.id).toBe('comprehensive'); + expect(manifest.tools).toHaveLength(2); + expect(manifest.search?.providerId).toBe('longbridge'); + expect(manifest.featureFlags?.demoData).toBe(false); + }); + + it('does not mutate the supplied context tools array', () => { + const ctx = baseContext(); + const manifest = captureRunManifest(makeRun(), ctx); + manifest.tools.push({ name: 'extra', enabled: true }); + expect(ctx.tools).toHaveLength(2); + }); + + it('deep-copies nested config so later settings mutations cannot rewrite the snapshot', () => { + const ctx = baseContext(); + const manifest = captureRunManifest(makeRun(), ctx); + + // Mutate every nested structure the context handed over, as a live settings + // object would be mutated by a later settings change. + ctx.model = 'gpt-4o-mini'; + (ctx.modelParams as Record).thinkingLevel = 'high'; + (ctx.search?.routing as Record).primary = 'massive'; + (ctx.featureFlags as Record).demoData = true; + (ctx.tools as RunManifestTool[])[0]!.name = 'mutated'; + ctx.prompt!.hash = 'mutated'; + + expect(manifest.model).toBe('gpt-4o'); + expect(manifest.modelParams?.thinkingLevel).toBe('low'); + expect(manifest.search?.routing?.primary).toBe('longbridge'); + expect(manifest.featureFlags?.demoData).toBe(false); + expect(manifest.tools[0]?.name).toBe('market.quote'); + expect(manifest.prompt?.hash).toBe('abc'); + }); + + it('keeps the context independent of the returned snapshot in the other direction', () => { + const ctx: RunManifestContext = { + ...baseContext(), + budget: { effective: { modelCalls: 3 }, clamped: [] }, + }; + const manifest = captureRunManifest(makeRun(), ctx); + (manifest.budget?.effective as Record).modelCalls = 99; + manifest.tools[0]!.name = 'mutated'; + + expect(ctx.budget?.effective?.modelCalls).toBe(3); + expect(ctx.tools?.[0]?.name).toBe('market.quote'); + }); + + it('defaults tools to an empty list when the context omits them', () => { + const ctx: RunManifestContext = { runtimeMode: 'local' }; + const manifest = captureRunManifest(makeRun(), ctx); + expect(manifest.tools).toEqual([]); + expect(manifest.runtimeMode).toBe('local'); + }); + + it('redacts a credential that sneaks into the context before it is ever stored', () => { + // Capture is the boundary: a caller bug must not be able to put a key into + // the persisted manifest, so redaction happens here and not only on export. + const ctx: RunManifestContext = { + ...baseContext(), + // @ts-expect-error intentional: simulate a caller bug + modelParams: { apiKey: 'sk-secret', maxTokens: 4096 }, + }; + const manifest = captureRunManifest(makeRun(), ctx); + expect((manifest.modelParams as Record).apiKey).toBe('[redacted]'); + // Sizing params are not credentials — losing them would make the snapshot + // unable to explain the limits the run obeyed. + expect(manifest.modelParams?.maxTokens).toBe(4096); + }); +}); + +describe('redactManifestSecrets', () => { + it('replaces secret-shaped values recursively but keeps structure', () => { + const dirty = { + provider: 'openai', + modelParams: { apiKey: 'sk-123', baseUrl: 'https://x' }, + nested: { token: 't-9', ok: 1 }, + list: [{ secret: 's', name: 'a' }], + }; + const clean = redactManifestSecrets(dirty); + expect(clean.modelParams.apiKey).toBe('[redacted]'); + expect(clean.modelParams.baseUrl).toBe('https://x'); + expect(clean.nested.token).toBe('[redacted]'); + expect(clean.nested.ok).toBe(1); + expect(clean.list[0].secret).toBe('[redacted]'); + expect(clean.list[0].name).toBe('a'); + }); + + it('leaves an already-clean object untouched', () => { + const clean = { a: 1, b: 'x', c: { d: true } }; + expect(redactManifestSecrets(clean)).toEqual(clean); + }); + + it('keeps token/context sizing config, which is not a credential', () => { + const clean = redactManifestSecrets({ + modelParams: { maxTokens: 4096, maxOutputTokens: 8192, contextWindow: 128_000 }, + budget: { tokenBudget: 1000 }, + tokenCount: 12, + }); + expect(clean.modelParams).toEqual({ maxTokens: 4096, maxOutputTokens: 8192, contextWindow: 128_000 }); + expect(clean.budget.tokenBudget).toBe(1000); + expect(clean.tokenCount).toBe(12); + }); + + it('still redacts token-shaped keys that are credentials', () => { + const clean = redactManifestSecrets({ + token: 't-9', + apiToken: 'a-1', + authToken: 'b-2', + refreshToken: 'c-3', + credentials: { user: 'u', pass: 'p' }, + }) as Record; + expect(clean.token).toBe('[redacted]'); + expect(clean.apiToken).toBe('[redacted]'); + expect(clean.authToken).toBe('[redacted]'); + expect(clean.refreshToken).toBe('[redacted]'); + expect(clean.credentials).toBe('[redacted]'); + }); +}); + +describe('diffRunManifests', () => { + it('reports no change for identical manifests', () => { + const a = captureRunManifest(makeRun(), baseContext()); + const b = captureRunManifest(makeRun({ id: 'run-2' }), baseContext()); + const diff = diffRunManifests(a, b); + expect(diff.changed).toBe(false); + expect(diff.fields).toHaveLength(0); + }); + + it('ignores identity anchors (runId/createdAt) but flags model + prompt', () => { + const a = captureRunManifest(makeRun(), baseContext()); + const b = captureRunManifest( + makeRun({ id: 'run-2' }), + { ...baseContext(), model: 'gpt-4o-mini', prompt: { hash: 'def', version: '0.4.0-beta.2', source: 'skill-index' } } + ); + const diff = diffRunManifests(a, b); + expect(diff.changed).toBe(true); + const paths = diff.fields.map((f) => f.path); + expect(paths).toContain('model'); + expect(paths).toContain('prompt.hash'); + expect(paths).not.toContain('runId'); + expect(paths).not.toContain('createdAt'); + expect(diff.groups.model).toBe(true); + expect(diff.groups.prompt).toBe(true); + }); + + it('flags tool differences by tool name (added/removed/version)', () => { + const a = captureRunManifest(makeRun(), baseContext()); + const b = captureRunManifest( + makeRun({ id: 'run-2' }), + { + ...baseContext(), + tools: [ + { name: 'market.quote', capability: 'market.quote', enabled: true, version: '2.0' }, + { name: 'get_news', capability: 'news.get', enabled: false }, + ], + } + ); + const diff = diffRunManifests(a, b); + const paths = diff.fields.map((f) => f.path); + expect(paths).toContain('tools.market.quote.version'); + expect(paths).toContain('tools.get_news.enabled'); + expect(diff.groups.tools).toBe(true); + expect(diff.groups.versions).toBe(true); + }); + + it('flags config (search/featureFlags) differences', () => { + const a = captureRunManifest(makeRun(), baseContext()); + const b = captureRunManifest( + makeRun({ id: 'run-2' }), + { ...baseContext(), search: { providerId: 'massive', routing: { primary: 'massive', fallback: 'longbridge' }, configured: true } } + ); + const diff = diffRunManifests(a, b); + expect(diff.groups.config).toBe(true); + expect(diff.fields.map((f) => f.path)).toContain('search.providerId'); + }); +}); + +describe('exportRunManifest', () => { + it('produces indented JSON and redacts secrets', () => { + const manifest = captureRunManifest(makeRun(), baseContext()); + // Force a secret through the redaction path to prove export defends it. + const dirty = redactManifestSecrets(manifest) as RunManifest; + // No secret in the baseline manifest; just assert shape + stability. + expect(exportRunManifest(manifest)).toContain('"schemaVersion": 1'); + expect(exportRunManifest(manifest)).toContain('"runId": "run-1"'); + void dirty; + }); + + it('redacts a secret present in the manifest before export', () => { + const manifest = captureRunManifest(makeRun(), baseContext()); + (manifest.featureFlags as Record).apiKey = 'sk-leak'; + const out = exportRunManifest(manifest); + expect(out).not.toContain('sk-leak'); + expect(out).toContain('[redacted]'); + }); + + it('keeps sizing config in the exported JSON', () => { + const manifest = captureRunManifest(makeRun(), { + ...baseContext(), + modelParams: { maxTokens: 4096, contextWindow: 128_000 }, + }); + const out = exportRunManifest(manifest); + expect(out).toContain('"maxTokens": 4096'); + expect(out).toContain('"contextWindow": 128000'); + }); +}); + +describe('hashManifestInput', () => { + it('is stable for equal input and differs for different input', () => { + expect(hashManifestInput('a|b|c')).toBe(hashManifestInput('a|b|c')); + expect(hashManifestInput('a|b|c')).not.toBe(hashManifestInput('a|b|d')); + expect(hashManifestInput('a|b|c')).toMatch(/^[0-9a-f]{64}$/); + }); +}); diff --git a/packages/shared/src/kernel/run-manifest.ts b/packages/shared/src/kernel/run-manifest.ts new file mode 100644 index 00000000..85656124 --- /dev/null +++ b/packages/shared/src/kernel/run-manifest.ts @@ -0,0 +1,261 @@ +import { createHash } from 'node:crypto'; +import type { + Run, + RunManifest, + RunManifestBudget, + RunManifestContext, + RunManifestDiff, + RunManifestFeatureFlags, + RunManifestFieldDiff, + RunManifestModelParams, + RunManifestSearch, + RunManifestStrategy, + RunManifestTool, +} from '@finagent/core'; + +/** + * Snapshot a run's configuration into an immutable manifest. + * + * `context` is assembled by the caller (the main-process host) from the live + * runtime + settings; `run` supplies the identity anchors. The manifest is a + * plain data object — the caller persists it and must never overwrite it with + * later global settings (#21 immutability requirement). + * + * The snapshot is **deep-copied and redacted before it is returned**: + * + * - The context holds references to live settings objects (`modelParams`, + * `search`, `budget`, …). Copying only the top level would let a later + * settings mutation rewrite a manifest that is supposed to be frozen, so the + * whole graph is cloned here. + * - Redaction happens at capture — not just on export — so the persisted + * manifest and every read path (`getRunManifest` / `getResearchManifest`) + * are credential-free by construction, with `exportRunManifest` keeping its + * own pass as a second line of defence. + */ +export function captureRunManifest(run: Run, context: RunManifestContext): RunManifest { + const manifest: RunManifest = { + schemaVersion: 1, + runId: run.id, + createdAt: run.startedAt, + runtimeMode: context.runtimeMode, + appVersion: context.appVersion, + buildVersion: context.buildVersion, + gitRevision: context.gitRevision, + provider: context.provider, + model: context.model, + modelParams: context.modelParams, + prompt: context.prompt, + strategy: context.strategy, + tools: context.tools ?? [], + search: context.search, + featureFlags: context.featureFlags, + budget: context.budget, + locale: context.locale, + evaluation: context.evaluation, + }; + return redactManifestSecrets(cloneManifestData(manifest)); +} + +/** + * Recursively copy plain objects and arrays so a captured manifest shares no + * mutable structure with the context it was built from. + * + * Deliberately not `structuredClone`: capture runs inside run creation, and a + * non-cloneable value in a caller-supplied context must never be able to abort + * a run. Manifest data is plain objects/arrays/primitives; anything else is + * passed through unchanged. + */ +function cloneManifestData(value: T): T { + if (Array.isArray(value)) { + return value.map((entry) => cloneManifestData(entry)) as unknown as T; + } + if (value !== null && typeof value === 'object') { + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) return value; + const out: Record = {}; + for (const [key, child] of Object.entries(value as Record)) { + out[key] = cloneManifestData(child); + } + return out as unknown as T; + } + return value; +} + +/** Keys whose values are treated as secrets and stripped from a manifest. */ +const SECRET_KEY_PATTERN = /(api[_-]?key|apikey|secret|token|password|passwd|private[_-]?key|credential|authorization|auth)/i; + +/** + * Secret-shaped keys that are **not** credentials: token / context sizing. + * + * `maxTokens`, `maxOutputTokens`, `contextWindow`, `tokenBudget` … are part of + * what a run actually obeyed, so redacting them would leave a historical + * manifest unable to explain its own limits. A bare `token` is *not* covered — + * it only stops being a secret when a sizing qualifier is present. + */ +const SIZING_KEY_PATTERN = + /^(?:(?:max|min|num|total|input|output|prompt|completion|context|reasoning)[_-]?)+(?:tokens?|window|chars?|size)$|^token[_-]?(?:limit|budget|count|quota)$/i; + +/** + * Whether a key/value pair is a credential that must be stripped. + * + * Three things are deliberately *not* treated as secrets: known sizing knobs + * (see `SIZING_KEY_PATTERN`), numbers and booleans — a credential is a string + * or a nested structure, so `maxTokens: 4096` and `tokenCount: 12` survive and + * the manifest stays reproducible. + */ +function isSecretEntry(key: string, value: unknown): boolean { + if (!SECRET_KEY_PATTERN.test(key)) return false; + if (typeof value === 'number' || typeof value === 'boolean') return false; + return !SIZING_KEY_PATTERN.test(key); +} + +/** + * Defensive redaction: recursively walk a value and replace the *values* of + * any secret-shaped keys with `[redacted]`. A manifest must never contain + * credentials (#21, #19); it runs on capture (so the persisted snapshot is + * already clean) and again on export / IPC read, so a caller bug cannot leak a + * key through the JSON export or the IPC surface. + * + * Only values are replaced — the key names are preserved so the structure + * stays self-describing and the diff/export paths are stable. + */ +export function redactManifestSecrets(value: T): T { + if (Array.isArray(value)) { + return value.map((entry) => redactManifestSecrets(entry)) as unknown as T; + } + if (value !== null && typeof value === 'object') { + const out: Record = {}; + for (const [key, child] of Object.entries(value as Record)) { + if (isSecretEntry(key, child) && child !== null && child !== undefined && child !== '') { + out[key] = '[redacted]'; + } else { + out[key] = redactManifestSecrets(child); + } + } + return out as unknown as T; + } + return value; +} + +/** Stable hash of a string (used for the prompt/composition fingerprint). */ +export function hashManifestInput(input: string): string { + return createHash('sha256').update(input).digest('hex'); +} + +/** + * Compare two manifests and report which config dimensions changed. + * + * Identity anchors (`runId`, `createdAt`, `schemaVersion`) are excluded — they + * always differ between two runs and are not "configuration". The `tools` + * array is compared element-wise by tool name so a toggled/added/removed tool + * or a version bump shows as a precise `tools..` diff. + */ +export function diffRunManifests(before: RunManifest, after: RunManifest): RunManifestDiff { + const left = collectLeaves(before, ''); + const right = collectLeaves(after, ''); + const paths = new Set([...left.keys(), ...right.keys()]); + const ignored = new Set(['runId', 'createdAt', 'schemaVersion']); + + const fields: RunManifestFieldDiff[] = []; + const groups = { model: false, prompt: false, tools: false, config: false, versions: false }; + + for (const path of [...paths].sort()) { + if (ignored.has(path)) continue; + const a = left.get(path); + const b = right.get(path); + if (deepEqual(a, b)) continue; + fields.push({ path, before: a, after: b }); + classify(path, groups); + } + + return { changed: fields.length > 0, fields, groups }; +} + +/** Pretty-printed, secret-redacted JSON for export / Run Info view. */ +export function exportRunManifest(manifest: RunManifest): string { + return JSON.stringify(redactManifestSecrets(manifest), null, 2); +} + +// --------------------------------------------------------------------------- +// Internals +// --------------------------------------------------------------------------- + +const IDENTITY_ANCHORS = new Set(['runId', 'createdAt', 'schemaVersion']); + +function collectLeaves(value: unknown, prefix: string, out: Map = new Map()): Map { + if (IDENTITY_ANCHORS.has(prefix)) { + out.set(prefix, value); + return out; + } + if (Array.isArray(value)) { + // Only `tools` is an array in the manifest; compare by tool name. + if (prefix === 'tools' || prefix.endsWith('.tools')) { + for (const item of value as RunManifestTool[]) { + const key = typeof item?.name === 'string' ? item.name : '?'; + collectLeaves(item, `${prefix}.${key}`, out); + } + return out; + } + out.set(prefix, value); + return out; + } + if (value !== null && typeof value === 'object') { + for (const [key, child] of Object.entries(value as Record)) { + collectLeaves(child, prefix ? `${prefix}.${key}` : key, out); + } + return out; + } + out.set(prefix, value); + return out; +} + +function classify(path: string, groups: RunManifestDiff['groups']): void { + if (path === 'provider' || path === 'model' || path.startsWith('modelParams')) { + groups.model = true; + } + if (path.startsWith('prompt')) { + groups.prompt = true; + } + if (path.startsWith('tools')) { + groups.tools = true; + } + if (path.startsWith('search') || path.startsWith('featureFlags') || path.startsWith('budget') || path === 'runtimeMode' || path === 'locale') { + groups.config = true; + } + if ( + path === 'appVersion' || + path === 'buildVersion' || + path === 'gitRevision' || + path === 'prompt.version' || + path === 'strategy.version' || + path === 'strategy.id' || + path.endsWith('.version') + ) { + groups.versions = true; + } +} + +function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) return true; + if (typeof a !== typeof b) return false; + if (a === null || b === null || typeof a !== 'object') return false; + if (Array.isArray(a) || Array.isArray(b)) { + if (!Array.isArray(a) || !Array.isArray(b)) return false; + if (a.length !== b.length) return false; + return a.every((entry, index) => deepEqual(entry, b[index])); + } + const aKeys = Object.keys(a as object); + const bKeys = Object.keys(b as object); + if (aKeys.length !== bKeys.length) return false; + return aKeys.every((key) => deepEqual((a as Record)[key], (b as Record)[key])); +} + +// Re-exported types for callers that build contexts (kept here to avoid a core import in tests). +export type { + RunManifestBudget, + RunManifestFeatureFlags, + RunManifestModelParams, + RunManifestSearch, + RunManifestStrategy, + RunManifestTool, +}; diff --git a/packages/shared/src/research/recovery.test.ts b/packages/shared/src/research/recovery.test.ts index 70d8749c..d34a1431 100644 --- a/packages/shared/src/research/recovery.test.ts +++ b/packages/shared/src/research/recovery.test.ts @@ -24,12 +24,25 @@ function service(overrides: Partial = {}) { synthesizer: local, repository: repository(), ...overrides, }); } +/** + * Poll until the run reaches a terminal status. + * + * Wall-clock budget, not iteration count — same reasoning as + * `service.test.ts`'s `waitForTerminal`. A fixed `400 × 5ms` loop buys ~2s + * only when every poll is fast; once the poll round-trips through the on-disk + * run store on a saturated CI runner, the loop can exhaust its iterations + * before an otherwise healthy run settles, surfacing as a bogus + * "Run did not settle". + */ +const TERMINAL_WAIT_MS = 5_000; + async function settled(svc: ResearchService, id: string) { - for (let i = 0; i < 400; i++) { + const deadline = Date.now() + TERMINAL_WAIT_MS; + do { const run = await svc.getRun(id); if (run && ['interrupted', 'completed', 'partial', 'failed', 'cancelled'].includes(run.status)) return run; await Bun.sleep(5); - } + } while (Date.now() < deadline); throw new Error('Run did not settle'); } async function interrupted() { diff --git a/packages/shared/src/research/service.test.ts b/packages/shared/src/research/service.test.ts index 0ef6e1bd..5b13737e 100644 --- a/packages/shared/src/research/service.test.ts +++ b/packages/shared/src/research/service.test.ts @@ -89,6 +89,69 @@ describe('ResearchService', () => { expect(reports[0].symbol).toBe('NVDA.US'); }); + it('stamps the research strategy and resolved budget into the run manifest (#21 / #17)', async () => { + const registry = createCapabilityRegistry( + RESEARCH_CAPABILITY_PLAN.map((id) => fakeCap(id, 'success')) + ); + const service = new ResearchService({ + registry, + synthesizer: new LocalResearchSynthesizer(), + repository: new ResearchReportRepository(new JsonFileStore(dir)), + now: () => 1_700_000_000_000, + budgets: { defaults: { modelCalls: 8 }, ceiling: { modelCalls: 4 } }, + getRunManifestContext: () => ({ + runtimeMode: 'pi', + model: 'gpt-4o', + strategy: { id: 'copilot-agent', version: '0.5.0' }, + }), + }); + + const queued = await service.start('NVDA.US', 'value'); + await waitForTerminal(service, queued.id); + const manifest = (await service.getRun(queued.id))?.manifest; + + expect(manifest?.model).toBe('gpt-4o'); + // The research strategy wins over the host's default agent workflow: the id + // is the research strategy that ran, and the version is a content hash of + // its resolved capability plan — not the host's app version, which is + // already recorded separately as `appVersion`/`buildVersion`. + expect(manifest?.strategy?.id).toBe('value'); + expect(manifest?.strategy?.version).toMatch(/^[0-9a-f]{64}$/); + expect(manifest?.strategy?.version).not.toBe('0.5.0'); + expect(manifest?.budget).toEqual({ + defaults: { modelCalls: 8 }, + ceiling: { modelCalls: 4 }, + effective: { modelCalls: 4 }, + clamped: ['modelCalls'], + }); + }); + + it('content-addresses the strategy version so it tracks the resolved plan (#21)', async () => { + const registry = createCapabilityRegistry( + RESEARCH_CAPABILITY_PLAN.map((id) => fakeCap(id, 'success')) + ); + const service = new ResearchService({ + registry, + synthesizer: new LocalResearchSynthesizer(), + repository: new ResearchReportRepository(new JsonFileStore(dir)), + now: () => 1_700_000_000_000, + getRunManifestContext: () => ({ runtimeMode: 'pi', model: 'gpt-4o' }), + }); + + const value = await service.start('NVDA.US', 'value'); + await waitForTerminal(service, value.id); + const growth = await service.start('MSFT.US', 'growth'); + await waitForTerminal(service, growth.id); + + const valueManifest = (await service.getRun(value.id))?.manifest; + const growthManifest = (await service.getRun(growth.id))?.manifest; + + expect(valueManifest?.strategy?.version).toMatch(/^[0-9a-f]{64}$/); + // Different plans must never share a version, or a manifest diff would + // silently treat two behaviourally different strategies as identical. + expect(valueManifest?.strategy?.version).not.toBe(growthManifest?.strategy?.version); + }); + it('does not expose a terminal status until the report is persisted', async () => { const repository = new ResearchReportRepository(new JsonFileStore(dir)); const persist = repository.saveReport.bind(repository); diff --git a/packages/shared/src/research/service.ts b/packages/shared/src/research/service.ts index ec5d7fd2..fbbe0de3 100644 --- a/packages/shared/src/research/service.ts +++ b/packages/shared/src/research/service.ts @@ -1,11 +1,12 @@ import { randomUUID } from 'node:crypto'; import type { CapabilityRegistry, ResearchExecutionIdentity, ResearchReport, ResearchRunSummary, - ResearchSynthesizer, StrategyId, SupportedLocale, + ResearchSynthesizer, Run, RunManifestContext, StrategyId, SupportedLocale, } from '@finagent/core'; import { i18nCurrentLocale } from '@finagent/i18n'; import { redact } from '../diagnostics/redact.ts'; import { createCodeError } from '../agent/errors.ts'; +import { captureRunManifest, hashManifestInput } from '../kernel/run-manifest.ts'; import { createUsage, resolveBudget, type ResolveBudgetInput } from '../kernel/run-budget.ts'; import { isStrategyId } from '../strategies/presets.ts'; import { buildCapabilityInput, planForStrategy } from './planner.ts'; @@ -24,6 +25,12 @@ export interface ResearchServiceOptions { onRunComplete?: (result: ResearchRunResult) => Promise | void; getIdentity?: () => Promise; budgets?: ResolveBudgetInput; + /** + * Assembled run-config snapshot for the immutable manifest (#21). Called once + * at `start`; the service stamps it onto the `ResearchRunSummary`. Absent on + * runs started before the manifest feature, or when the host omits it. + */ + getRunManifestContext?: () => RunManifestContext | Promise; } interface ActiveRun { @@ -80,10 +87,37 @@ export class ResearchService { completedCapabilities: [], failedCapabilities: [], strategyId, locale: locale ?? i18nCurrentLocale(), recoveryCount: 0, }; + // #17: resolve the research budget once, before the manifest snapshot, so + // both the checkpoint and the immutable manifest record the same effective + // limits (defaults → overrides → ceiling). + const budget = resolveBudget(this.options.budgets ?? {}); + const manifestContext = await this.options.getRunManifestContext?.(); + if (manifestContext) { + // The service knows the run's strategy; fold it into the assembled context + // so the persisted manifest records which workflow drove the research. The + // research strategy id always wins so the manifest names the workflow that + // actually ran, not the host's default agent strategy. `version` is a + // content hash of the strategy's resolved capability plan: it is stable for + // a given strategy and changes the moment the plan (its capability set or + // order) changes, so two manifests whose strategy behaviour differs never + // share a version. + const strategyVersion = hashManifestInput(plan.map((p) => p.capabilityId).join('\n')); + const merged: RunManifestContext = { + ...manifestContext, + strategy: { ...manifestContext.strategy, id: strategyId, version: strategyVersion }, + budget: { + defaults: this.options.budgets?.defaults, + ceiling: this.options.budgets?.ceiling, + effective: budget.limits, + clamped: budget.clamped, + }, + }; + summary.manifest = captureRunManifest({ id: summary.id, startedAt: summary.startedAt } as Run, merged); + } const cp: ResearchCheckpoint = { version: CHECKPOINT_VERSION, summary, plan, outcomes: [], phase: 'fetching', identity: await this.identity(), - budget: { limits: resolveBudget(this.options.budgets ?? {}).limits, usage: createUsage() }, + budget: { limits: budget.limits, usage: createUsage() }, retry: { attempts: {}, synthesisAttempts: 0 }, inFlight: [], events: [{ runId: summary.id, parentRunId: summary.id, spanId: randomUUID(), type: 'started', at: this.now() }], }; diff --git a/packages/ui/src/atoms/sessionAtoms.test.ts b/packages/ui/src/atoms/sessionAtoms.test.ts index ff25dcdf..8eb46132 100644 --- a/packages/ui/src/atoms/sessionAtoms.test.ts +++ b/packages/ui/src/atoms/sessionAtoms.test.ts @@ -49,6 +49,9 @@ function makeClient(): FinagentClient { startRun: async () => ({ ok: false as const, error: { code: 'TEST', message: 'no-op' } }), cancelRun: async () => ({ ok: true as const, data: undefined }), streamReplay: async () => ({ ok: true as const, data: { recoverable: false, events: [], atEnd: true } }), + getManifest: async () => ({ ok: true as const, data: undefined }), + exportManifest: async () => ({ ok: true as const, data: undefined }), + compareManifests: async () => ({ ok: false as const, error: { code: 'TEST', message: 'no-op' } }), onAgentEvent: () => () => undefined, onStreamEvent: () => () => undefined, }, diff --git a/packages/ui/src/client.tsx b/packages/ui/src/client.tsx index 18763070..cc5f692e 100644 --- a/packages/ui/src/client.tsx +++ b/packages/ui/src/client.tsx @@ -43,6 +43,8 @@ import type { ResearchOpinion, ResearchOutcome, Run, + RunManifest, + RunManifestDiff, SessionMeta, Skill, SkillReadiness, @@ -189,6 +191,16 @@ export interface FinagentClient { ) => Promise>; cancelRun: (sessionId: string, runId: string) => Promise>; streamReplay: (input: { runId: string; lastSequence: number }) => Promise>; + /** #21: immutable run-config manifest for a copilot run. */ + getManifest: (input: { sessionId: string; runId: string }) => Promise>; + /** #21: pretty-printed, secret-redacted JSON export of a copilot run manifest. */ + exportManifest: (input: { sessionId: string; runId: string }) => Promise>; + /** #21: structured diff between two copilot run manifests. */ + compareManifests: (input: { + sessionId: string; + runIdA: string; + runIdB: string; + }) => Promise>; onAgentEvent: (callback: (event: AgentEvent) => void) => () => void; onStreamEvent: (callback: (payload: { sessionId: string; event: StreamEvent }) => void) => () => void; }; @@ -239,6 +251,10 @@ export interface FinagentClient { listReports: (input: { symbol?: string }) => Promise>; getReport: (input: { reportId: string }) => Promise>; getDiff: (input: { symbol: string }) => Promise>; + /** #21: immutable run-config manifest for a Deep Research run. */ + getManifest: (input: { runId: string }) => Promise>; + /** #21: structured diff between two Deep Research run manifests. */ + compareManifests: (input: { runIdA: string; runIdB: string }) => Promise>; }; screening?: ScreeningChannel; outcome?: { @@ -320,6 +336,9 @@ export const fallbackClient: FinagentClient = { startRun: missingClient('kernel.startRun'), cancelRun: missingClient('kernel.cancelRun'), streamReplay: missingClient('kernel.streamReplay'), + getManifest: missingClient('kernel.getManifest'), + exportManifest: missingClient('kernel.exportManifest'), + compareManifests: missingClient('kernel.compareManifests'), onAgentEvent: () => () => undefined, onStreamEvent: () => () => undefined, }, @@ -361,6 +380,8 @@ export const fallbackClient: FinagentClient = { listReports: missingClient('research.listReports'), getReport: missingClient('research.getReport'), getDiff: missingClient('research.getDiff'), + getManifest: missingClient('research.getManifest'), + compareManifests: missingClient('research.compareManifests'), }, screening: { run: missingClient('screening.run'), diff --git a/packages/ui/src/components/kernel/RunInfoPanel.test.tsx b/packages/ui/src/components/kernel/RunInfoPanel.test.tsx new file mode 100644 index 00000000..53012b53 --- /dev/null +++ b/packages/ui/src/components/kernel/RunInfoPanel.test.tsx @@ -0,0 +1,88 @@ +import { afterAll, beforeAll, expect, it } from 'bun:test'; +import React, { act } from 'react'; +import { createRoot } from 'react-dom/client'; +import type { RunManifest, RunManifestDiff } from '@finagent/core'; +import { installHappyDom } from '../../test/setupHappyDom'; + +let restore: () => void; +let ManifestView: typeof import('./RunInfoPanel')['ManifestView']; +let DiffView: typeof import('./RunInfoPanel')['DiffView']; + +beforeAll(async () => { + restore = installHappyDom().restore; + ({ ManifestView, DiffView } = await import('./RunInfoPanel')); +}); +afterAll(() => restore()); + +const manifest: RunManifest = { + schemaVersion: 1, + runId: 'run-1', + createdAt: 1000, + runtimeMode: 'pi', + provider: 'openai', + model: 'gpt-4o', + prompt: { hash: 'abc123', version: 'v1', source: 'skill-index' }, + tools: [{ name: 'get_portfolio', enabled: true }], + search: { providerId: 'longbridge', configured: true }, + appVersion: '0.4.0', + locale: 'zh-CN', +}; + +async function render(node: React.ReactNode): Promise<{ container: HTMLElement; root: ReturnType }> { + const container = document.createElement('div'); + document.body.appendChild(container); + const root = createRoot(container); + await act(async () => { + root.render(node); + }); + return { container, root }; +} + +it('renders the manifest fields and a secret-free JSON export', async () => { + const { container, root } = await render(); + try { + const text = container.textContent ?? ''; + expect(text).toContain('gpt-4o'); + expect(text).toContain('abc123'); + expect(text).toContain('get_portfolio'); + expect(text).toContain('longbridge'); + expect(container.querySelector('pre')?.textContent).toContain('"runId": "run-1"'); + } finally { + await act(async () => root.unmount()); + container.remove(); + } +}); + +it('renders a structured diff with before -> after values and group flags', async () => { + const diff: RunManifestDiff = { + changed: true, + fields: [{ path: 'model', before: 'gpt-4o', after: 'gpt-4o-mini' }], + groups: { model: true, prompt: false, tools: false, config: false, versions: false }, + }; + const { container, root } = await render(); + try { + const text = container.textContent ?? ''; + expect(text).toContain('两次运行配置存在差异'); + expect(text).toContain('model'); + expect(text).toContain('gpt-4o'); + expect(text).toContain('gpt-4o-mini'); + } finally { + await act(async () => root.unmount()); + container.remove(); + } +}); + +it('renders an unchanged diff as identical', async () => { + const diff: RunManifestDiff = { + changed: false, + fields: [], + groups: { model: false, prompt: false, tools: false, config: false, versions: false }, + }; + const { container, root } = await render(); + try { + expect(container.textContent ?? '').toContain('两次运行配置一致'); + } finally { + await act(async () => root.unmount()); + container.remove(); + } +}); diff --git a/packages/ui/src/components/kernel/RunInfoPanel.tsx b/packages/ui/src/components/kernel/RunInfoPanel.tsx new file mode 100644 index 00000000..eaec9c6d --- /dev/null +++ b/packages/ui/src/components/kernel/RunInfoPanel.tsx @@ -0,0 +1,300 @@ +/** + * Run Info view (#21). + * + * Renders an immutable run-config manifest (RunManifest) in a human-readable + * form, offers a secret-free JSON export, and can show a structured two-run + * diff. The manifest never contains credentials — redaction happens at capture + * time in `@finagent/shared`, so the JSON shown here is safe to copy/share. + */ +import React, { useEffect, useState } from 'react'; +import type { RunManifest, RunManifestDiff } from '@finagent/core'; +import { Dialog } from '../primitives/Dialog'; +import { useFinagentClient } from '../../client'; + +const RUNTIME_MODE_LABEL: Record = { + local: '本地运行时', + pi: 'Pi 运行时', + demo: '演示数据', +}; + +function Field({ label, value }: { label: string; value: React.ReactNode }) { + if (value === undefined || value === null || value === '') return null; + return ( +
+ {label} + {value} +
+ ); +} + +function modelParamsText(params?: RunManifest['modelParams']): string | undefined { + if (!params) return undefined; + const parts: string[] = []; + if (params.api) parts.push(`api=${params.api}`); + if (params.baseUrl) parts.push(`baseUrl=${params.baseUrl}`); + if (params.thinkingLevel) parts.push(`thinking=${params.thinkingLevel}`); + if (params.contextWindow) parts.push(`ctx=${params.contextWindow}`); + if (params.maxTokens) parts.push(`max=${params.maxTokens}`); + if (params.reasoning !== undefined) parts.push(`reasoning=${params.reasoning}`); + return parts.length ? parts.join(' · ') : undefined; +} + +function featureFlagsText(flags?: RunManifest['featureFlags']): string | undefined { + if (!flags || Object.keys(flags).length === 0) return undefined; + return Object.entries(flags) + .map(([key, value]) => `${key}=${value}`) + .join(' · '); +} + +function toolsText(tools?: RunManifest['tools']): string | undefined { + if (!tools || tools.length === 0) return undefined; + return tools.map((tool) => tool.name).join(', '); +} + +const JsonExport: React.FC<{ manifest: RunManifest }> = ({ manifest }) => { + const [copied, setCopied] = useState(false); + const text = JSON.stringify(manifest, null, 2); + const copy = async () => { + try { + await navigator.clipboard.writeText(text); + setCopied(true); + setTimeout(() => setCopied(false), 1500); + } catch { + setCopied(false); + } + }; + return ( +
+
+ JSON 导出(已脱敏) + +
+
+        {text}
+      
+
+ ); +}; + +const ManifestView: React.FC<{ manifest: RunManifest }> = ({ manifest: m }) => ( +
+
+ + {RUNTIME_MODE_LABEL[m.runtimeMode]} + + + {m.provider ?? '—'} · {m.model ?? '—'} + +
+ + + + + + + + + + + + + + + + +
+); + +const DiffView: React.FC<{ diff: RunManifestDiff }> = ({ diff }) => ( +
+
+ {diff.changed ? '两次运行配置存在差异' : '两次运行配置一致'} +
+ {diff.changed && ( +
+ {diff.groups.model && } + {diff.groups.prompt && } + {diff.groups.tools && } + {diff.groups.config && } + {diff.groups.versions && } +
+ )} + {diff.changed ? ( +
+ {diff.fields.map((field) => ( +
+
{field.path}
+
+ {formatValue(field.before)} + → + {formatValue(field.after)} +
+
+ ))} +
+ ) : ( +

没有检测到模型、提示词、工具或配置层面的差异。

+ )} +
+); + +const Badge: React.FC<{ label: string }> = ({ label }) => ( + {label} +); + +function formatValue(value: unknown): string { + if (value === undefined) return '(无)'; + if (value === null) return 'null'; + if (typeof value === 'object') return JSON.stringify(value); + return String(value); +} + +async function fetchManifest( + client: ReturnType, + kind: 'research' | 'agent', + runId: string, + sessionId: string | undefined +): Promise { + const res = + kind === 'research' + ? await client.research!.getManifest({ runId }) + : await client.kernel.getManifest({ sessionId: sessionId ?? '', runId }); + return res.ok ? res.data : undefined; +} + +/** + * The diff is computed in the main process (`diffRunManifests` lives in + * `@finagent/shared`, which must never be imported by the renderer). The UI + * only consumes the structured result over IPC. + */ +async function fetchDiff( + client: ReturnType, + kind: 'research' | 'agent', + runIdA: string, + runIdB: string, + sessionId: string | undefined +): Promise { + const res = + kind === 'research' + ? await client.research!.compareManifests({ runIdA, runIdB }) + : await client.kernel.compareManifests({ sessionId: sessionId ?? '', runIdA, runIdB }); + if (!res.ok) throw new Error(res.error.message); + return res.data; +} + +/** Single-run Run Info dialog (#21). */ +export const RunInfoDialog: React.FC<{ + open: boolean; + onClose: () => void; + kind: 'research' | 'agent'; + runId: string; + sessionId?: string; +}> = ({ open, onClose, kind, runId, sessionId }) => { + const client = useFinagentClient(); + const [manifest, setManifest] = useState(); + const [missing, setMissing] = useState(false); + + useEffect(() => { + if (!open) return; + let cancelled = false; + setManifest(undefined); + setMissing(false); + (async () => { + const data = await fetchManifest(client, kind, runId, sessionId); + if (cancelled) return; + if (data) setManifest(data); + else setMissing(true); + })(); + return () => { + cancelled = true; + }; + }, [open, kind, runId, sessionId, client]); + + return ( + + {missing ? ( +

该运行没有可显示的清单(manifest)。

+ ) : manifest ? ( + + ) : ( +

加载中…

+ )} +
+ ); +}; + +/** Two-run diff dialog (#21). */ +export const RunInfoCompareDialog: React.FC<{ + open: boolean; + onClose: () => void; + kind: 'research' | 'agent'; + runIdA: string; + runIdB: string; + sessionId?: string; +}> = ({ open, onClose, kind, runIdA, runIdB, sessionId }) => { + const client = useFinagentClient(); + const [diff, setDiff] = useState(); + const [error, setError] = useState(); + + useEffect(() => { + if (!open) return; + let cancelled = false; + setDiff(undefined); + setError(undefined); + (async () => { + try { + const result = await fetchDiff(client, kind, runIdA, runIdB, sessionId); + if (cancelled) return; + setDiff(result); + } catch (e) { + if (!cancelled) setError(e instanceof Error ? e.message : String(e)); + } + })(); + return () => { + cancelled = true; + }; + }, [open, kind, runIdA, runIdB, sessionId, client]); + + return ( + + {error ? ( +

{error}

+ ) : diff ? ( + + ) : ( +

加载中…

+ )} +
+ ); +}; + +export { ManifestView, DiffView }; diff --git a/packages/ui/src/components/research/ResearchPanel.tsx b/packages/ui/src/components/research/ResearchPanel.tsx index 4670c361..c42d4ff7 100644 --- a/packages/ui/src/components/research/ResearchPanel.tsx +++ b/packages/ui/src/components/research/ResearchPanel.tsx @@ -1,7 +1,7 @@ import React, { useEffect, useState } from 'react'; import { useAtom, useAtomValue, useSetAtom } from 'jotai'; import { useTranslation } from 'react-i18next'; -import { Check, Search } from 'lucide-react'; +import { Check, Search, Info } from 'lucide-react'; import type { ResearchRunSummary, ResearchReport, StrategyId } from '@finagent/core'; import { activeSymbolAtom, navSectionAtom } from '../../atoms'; import { pendingResearchStrategyAtom, researchOriginAtom } from '../../atoms/discoverAtoms'; @@ -18,6 +18,7 @@ import { TERMINAL_RUN_STATUSES, } from '../../atoms/researchAtoms'; import { saveThesisFromReport } from '../../client/thesis'; +import { RunInfoDialog, RunInfoCompareDialog } from '../kernel/RunInfoPanel'; import { ResearchReportView } from './ResearchReportView'; import { ResearchMarketWorkspace } from './ResearchMarketWorkspace'; import { DEFAULT_STRATEGY_ID, StrategyPicker } from './StrategyPicker'; @@ -535,24 +536,74 @@ const RunHistory: React.FC<{ onSelect: (reportId: string) => void; }> = ({ runs, onSelect }) => { const { t } = useTranslation(); + const [selected, setSelected] = useState([]); + const [infoRunId, setInfoRunId] = useState(); + const [comparePair, setComparePair] = useState<[string, string] | undefined>(); + + const toggle = (runId: string) => + setSelected((prev) => (prev.includes(runId) ? prev.filter((id) => id !== runId) : [...prev, runId])); + if (runs.length === 0) return ; return (
- {runs.map((run) => ( + {selected.length >= 2 && ( + )} + {runs.map((run) => ( +
+ toggle(run.id)} + aria-label="选择用于对比" + className="h-3.5 w-3.5 shrink-0" + /> + + +
))} + setInfoRunId(undefined)} + kind="research" + runId={infoRunId ?? ''} + /> + setComparePair(undefined)} + kind="research" + runIdA={comparePair?.[0] ?? ''} + runIdB={comparePair?.[1] ?? ''} + />
); };