diff --git a/server/src/addie/jobs/compliance-heartbeat.ts b/server/src/addie/jobs/compliance-heartbeat.ts index b352dcb3e26..3dcb6415fe4 100644 --- a/server/src/addie/jobs/compliance-heartbeat.ts +++ b/server/src/addie/jobs/compliance-heartbeat.ts @@ -485,6 +485,7 @@ export async function runComplianceHeartbeatJob( agent_url: agent.agent_url, requested_compliance_target: runTarget.requested, adcp_version: runTarget.version, + runner_capability_version: LIBRARY_VERSION, lifecycle_stage: agent.lifecycle_stage as LifecycleStage, overall_status: 'failing', headline, diff --git a/server/src/addie/services/compliance-testing.ts b/server/src/addie/services/compliance-testing.ts index 6392dea59f1..69a5c935f81 100644 --- a/server/src/addie/services/compliance-testing.ts +++ b/server/src/addie/services/compliance-testing.ts @@ -1460,6 +1460,7 @@ export function complianceResultToDbInput( requested_compliance_target: (result as ComplianceResult & { requested_compliance_target?: string }) .requested_compliance_target ?? null, adcp_version: result.adcp_version ?? null, + runner_capability_version: result.summary.runner_capability_version ?? null, lifecycle_stage: lifecycleStage, overall_status, headline: result.summary.headline, diff --git a/server/src/db/compliance-db.ts b/server/src/db/compliance-db.ts index 32067838912..97b6e9710a5 100644 --- a/server/src/db/compliance-db.ts +++ b/server/src/db/compliance-db.ts @@ -117,6 +117,7 @@ export interface ComplianceRun { agent_url: string; requested_compliance_target: string | null; adcp_version: string | null; + runner_capability_version: string | null; lifecycle_stage: LifecycleStage; overall_status: OverallRunStatus; headline: string | null; @@ -285,6 +286,7 @@ export interface RecordComplianceRunInput { agent_url: string; requested_compliance_target?: string | null; adcp_version?: string | null; + runner_capability_version?: string | null; lifecycle_stage: LifecycleStage; overall_status: OverallRunStatus; headline?: string; @@ -696,8 +698,9 @@ export class ComplianceDatabase { tracks_skipped, tracks_partial, agent_profile_json, observations_json, triggered_by, triggered_org_id, dry_run, notices_json, refresh_operation_id, agent_quality_evaluation_id, - completeness, is_authoritative, storyboard_statuses_json, provenance_json - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24) + completeness, is_authoritative, storyboard_statuses_json, provenance_json, + runner_capability_version + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25) ON CONFLICT DO NOTHING RETURNING *`, [ @@ -725,6 +728,7 @@ export class ComplianceDatabase { authoritative, JSON.stringify(input.storyboard_statuses ?? []), input.provenance_json ? JSON.stringify(input.provenance_json) : null, + input.runner_capability_version ?? null, ], ); let run = runResult.rows[0] as ComplianceRun | undefined; diff --git a/server/src/db/migrations/608_agent_compliance_runner_version.sql b/server/src/db/migrations/608_agent_compliance_runner_version.sql new file mode 100644 index 00000000000..07ade74fba4 --- /dev/null +++ b/server/src/db/migrations/608_agent_compliance_runner_version.sql @@ -0,0 +1,5 @@ +ALTER TABLE agent_compliance_runs + ADD COLUMN IF NOT EXISTS runner_capability_version TEXT; + +COMMENT ON COLUMN agent_compliance_runs.runner_capability_version IS + 'Runner capability version recorded at execution time; NULL when not recorded, including historical runs.'; diff --git a/server/src/routes/registry-api.ts b/server/src/routes/registry-api.ts index 3a61b9fba1b..726fd55413b 100644 --- a/server/src/routes/registry-api.ts +++ b/server/src/routes/registry-api.ts @@ -7555,6 +7555,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { provenance: run.provenance_json ?? null, requested_compliance_target: run.requested_compliance_target ?? null, adcp_version: run.adcp_version ?? null, + runner_capability_version: run.runner_capability_version ?? null, overall_status: run.overall_status, headline: publicComplianceHeadline(run.overall_status, run.headline), tracks_passed: run.tracks_passed, diff --git a/server/src/schemas/registry.ts b/server/src/schemas/registry.ts index d2b2e69c6af..ff05d16d501 100644 --- a/server/src/schemas/registry.ts +++ b/server/src/schemas/registry.ts @@ -1497,6 +1497,9 @@ export const ComplianceRunSchema = z is_authoritative: z.boolean().optional(), requested_compliance_target: z.string().nullable().optional(), adcp_version: z.string().nullable().optional(), + runner_capability_version: z.string().nullable().optional().openapi({ + description: "Runner capability version recorded at execution time, separate from the AdCP version under test. Null when not recorded, including historical runs.", + }), overall_status: z.string(), headline: z.string().nullable(), tracks_passed: z.number().int(), diff --git a/server/tests/integration/registry-api-compliance-verdict-source.test.ts b/server/tests/integration/registry-api-compliance-verdict-source.test.ts index 4c20c2b9d31..5135f219f00 100644 --- a/server/tests/integration/registry-api-compliance-verdict-source.test.ts +++ b/server/tests/integration/registry-api-compliance-verdict-source.test.ts @@ -288,6 +288,61 @@ describe('GET /api/registry/agents/:encodedUrl/compliance — owner-scope gate ( const endpoint = `/api/registry/agents/${encodeURIComponent(AGENT_URL)}/compliance`; + it('returns null for a historical run without a recorded runner version', async () => { + const res = await request(app).get(`${endpoint}/history`); + + expect(res.status).toBe(200); + expect(res.body.runs).toEqual([ + expect.objectContaining({ id: complianceRunId, runner_capability_version: null }), + ]); + }); + + it('persists each runner version through real writes and history reads, including partial runs', async () => { + const agentUrl = `https://runner-version-${RUN_SUFFIX}.example.com/mcp`; + const db = new ComplianceDatabase(); + const expectedRuns = []; + + try { + for (const [overallStatus, runnerVersion] of [ + ['passing', '14.0.0-rc.12'], + ['partial', '14.0.0-rc.13'], + ['failing', null], + ] as const) { + const { run } = await db.recordComplianceRun({ + agent_url: agentUrl, + requested_compliance_target: '3.1', + adcp_version: '3.1.0', + runner_capability_version: runnerVersion, + lifecycle_stage: 'testing', + overall_status: overallStatus, + tracks_json: [], + tracks_passed: 0, + tracks_failed: 0, + tracks_skipped: 0, + tracks_partial: 0, + triggered_by: 'manual', + dry_run: false, + }); + expect(run.runner_capability_version).toBe(runnerVersion); + expectedRuns.push(expect.objectContaining({ + id: run.id, + overall_status: overallStatus, + adcp_version: '3.1.0', + runner_capability_version: runnerVersion, + })); + } + + const res = await request(app) + .get(`/api/registry/agents/${encodeURIComponent(agentUrl)}/compliance/history`); + expect(res.status).toBe(200); + expect(res.body.count).toBe(3); + expect(res.body.runs).toEqual(expect.arrayContaining(expectedRuns)); + } finally { + await pool.query('DELETE FROM agent_compliance_status WHERE agent_url = $1', [agentUrl]); + await pool.query('DELETE FROM agent_compliance_runs WHERE agent_url = $1', [agentUrl]); + } + }); + const OWNER_ONLY_KEYS = [ 'verdict_source', 'membership_tier', diff --git a/server/tests/unit/compliance-heartbeat.test.ts b/server/tests/unit/compliance-heartbeat.test.ts index 21671a831c7..deb295b2e27 100644 --- a/server/tests/unit/compliance-heartbeat.test.ts +++ b/server/tests/unit/compliance-heartbeat.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { LIBRARY_VERSION } from '@adcp/sdk'; const mocks = vi.hoisted(() => ({ getAgentsDueForCheck: vi.fn(), @@ -565,6 +566,26 @@ describe('runComplianceHeartbeatJob', () => { }); }); + it('records the invoking runner version when comply throws without a result', async () => { + mocks.comply.mockRejectedValueOnce(new Error('Timed out')); + + const { runComplianceHeartbeatJob } = await import('../../src/addie/jobs/compliance-heartbeat.js'); + const result = await runComplianceHeartbeatJob({ limit: 1 }); + + expect(result).toEqual({ checked: 0, passed: 0, failed: 0, skipped: 1 }); + expect(mocks.complianceResultToDbInput).not.toHaveBeenCalled(); + expect(mocks.recordComplianceRun).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + agent_url: 'https://agent.example.com/mcp', + adcp_version: target.version, + runner_capability_version: LIBRARY_VERSION, + overall_status: 'failing', + triggered_by: 'heartbeat', + dry_run: false, + }), + ); + }); + it('records malformed saved Basic auth as audit-only setup evidence', async () => { mocks.comply.mockRejectedValueOnce(new Error('step.auth.basic.username must be a non-empty string')); diff --git a/server/tests/unit/compliance-testing-effective-run-status.test.ts b/server/tests/unit/compliance-testing-effective-run-status.test.ts index 44a89af9c0e..095e2c4c38b 100644 --- a/server/tests/unit/compliance-testing-effective-run-status.test.ts +++ b/server/tests/unit/compliance-testing-effective-run-status.test.ts @@ -55,6 +55,34 @@ function makeResult(tracks: ReturnType[], overallStatus = 'par }; } +describe('complianceResultToDbInput runner version', () => { + it.each([ + ['pass', 'passing'], + ['partial', 'partial'], + ['fail', 'failing'], + ])('preserves the emitted version for %s results', (trackStatus, overallStatus) => { + const baseResult = makeResult([makeTrack(trackStatus)], overallStatus); + const result = { + ...baseResult, + adcp_version: '3.1.0', + summary: { ...baseResult.summary, runner_capability_version: '14.0.0-rc.12' }, + }; + const out = complianceResultToDbInput(result as any, 'https://agent.example.com/mcp', 'production'); + + expect(out.runner_capability_version).toBe('14.0.0-rc.12'); + expect(out.adcp_version).toBe('3.1.0'); + expect(out.overall_status).toBe(overallStatus); + }); + + it.each([{}, { runner_capability_version: null }])('does not invent a missing runner version: %j', fields => { + const baseResult = makeResult([makeTrack('pass')], 'passing'); + const result = { ...baseResult, summary: { ...baseResult.summary, ...fields } }; + const out = complianceResultToDbInput(result as any, 'https://agent.example.com/mcp', 'production'); + + expect(out.runner_capability_version).toBeNull(); + }); +}); + describe('complianceResultToDbInput — effectiveRunStatus', () => { it('promotes all-silent to passing with zero partial/failed counters', () => { const result = makeResult([makeTrack('silent'), makeTrack('silent')], 'partial'); diff --git a/static/openapi/registry.yaml b/static/openapi/registry.yaml index b971caa78a2..d3450219154 100644 --- a/static/openapi/registry.yaml +++ b/static/openapi/registry.yaml @@ -5503,6 +5503,11 @@ components: type: - string - "null" + runner_capability_version: + type: + - string + - "null" + description: Runner capability version recorded at execution time, separate from the AdCP version under test. Null when not recorded, including historical runs. overall_status: type: string headline: