diff --git a/examples/clients/typescript/everything-client.ts b/examples/clients/typescript/everything-client.ts index b3ccbed3..eed81093 100644 --- a/examples/clients/typescript/everything-client.ts +++ b/examples/clients/typescript/everything-client.ts @@ -196,6 +196,44 @@ async function runBasicClient(serverUrl: string): Promise { registerScenarios(['initialize', 'tools_call', 'tools-call'], runBasicClient); +// One complete enumeration: cursors are opaque strings, including ""; the +// presence of nextCursor, not page size or cursor truthiness, controls it. +// https://modelcontextprotocol.io/specification/2026-07-28/server/utilities/pagination +async function runPaginationClient(serverUrl: string): Promise { + const protocolVersion = process.env.MCP_CONFORMANCE_PROTOCOL_VERSION; + const stateless = protocolVersion + ? (STATELESS_SPEC_VERSIONS as readonly string[]).includes(protocolVersion) + : false; + const client = new Client( + { name: 'pagination-client', version: '1.0.0' }, + { capabilities: {} } + ); + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + try { + if (!stateless) await client.connect(transport); + const tools: { name: string }[] = []; + let cursor: string | undefined; + do { + const params = cursor === undefined ? {} : { cursor }; + const page = stateless + ? await statelessRequest(serverUrl, 'tools/list', params) + : await client.listTools(params); + tools.push(...page.tools); + cursor = page.nextCursor; + } while (cursor !== undefined); + if (!tools.some((tool) => tool.name === 'add_numbers')) { + throw new Error('Paginated discovery did not return add_numbers'); + } + const params = { name: 'add_numbers', arguments: { a: 2, b: 3 } }; + if (stateless) await statelessRequest(serverUrl, 'tools/call', params); + else await client.callTool(params); + } finally { + await client.close(); + } +} + +registerScenario('client-pagination', runPaginationClient); + // SEP-2106: json-schema-ref-no-deref advertises a tool whose inputSchema // contains a network-URI $ref. A conformant client lists tools normally and // simply never fetches that URI. The scenario's mock only serves tools/list, diff --git a/examples/clients/typescript/pagination-broken-client.ts b/examples/clients/typescript/pagination-broken-client.ts new file mode 100644 index 00000000..d0099136 --- /dev/null +++ b/examples/clients/typescript/pagination-broken-client.ts @@ -0,0 +1,91 @@ +/** + * Deliberately broken pagination strategies for the client-pagination scenario. + * MCP_PAGINATION_DEFECT selects a control; the default reproduces the truthiness + * condition reported in conformance#577. No language-model service is used. + */ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { sendStatelessRequest } from '../../../src/connection/stateless.js'; +import { STATELESS_SPEC_VERSIONS } from '../../../src/connection/select.js'; +import { LATEST_SPEC_VERSION, type SpecVersion } from '../../../src/types.js'; +import { runAsCli } from './helpers/cliRunner.js'; + +export type PaginationDefect = + | 'truthy-cursor' + | 'trim-cursor' + | 'recover-trim' + | 'empty-page' + | 'past-end' + | 'inert' + | 'rediscover'; + +export async function runClient( + serverUrl: string, + defect: PaginationDefect = (process.env.MCP_PAGINATION_DEFECT ?? + 'truthy-cursor') as PaginationDefect +): Promise { + if (defect === 'inert') return; + const specVersion = (process.env.MCP_CONFORMANCE_PROTOCOL_VERSION ?? + LATEST_SPEC_VERSION) as SpecVersion; + const stateless = (STATELESS_SPEC_VERSIONS as readonly string[]).includes( + specVersion + ); + const client = new Client( + { name: 'pagination-control-client', version: '1.0.0' }, + { capabilities: {} } + ); + const transport = new StreamableHTTPClientTransport(new URL(serverUrl)); + type ToolPage = { tools: { name: string }[]; nextCursor?: string }; + async function request(method: string, params: Record) { + const response = await sendStatelessRequest(serverUrl, method, params, { + specVersion + }); + if (response.body?.error || !response.body?.result) { + throw new Error(`${method}: ${JSON.stringify(response.body)}`); + } + return response.body.result; + } + async function list(cursor?: string): Promise { + const params = cursor === undefined ? {} : { cursor }; + return stateless + ? ((await request('tools/list', params)) as ToolPage) + : client.listTools(params); + } + try { + if (!stateless) await client.connect(transport); + const tools: { name: string }[] = []; + let cursor: string | undefined; + for (let page = 0; page < 8; page++) { + const result = await list(cursor); + tools.push(...result.tools); + // One wrong condition is enough to stop on a valid empty-string cursor. + if (defect === 'truthy-cursor' && !result.nextCursor) break; + if (defect === 'empty-page' && result.tools.length === 0) break; + if (result.nextCursor === undefined) { + // Continue the SAME traversal once after the server declared its end. + if (defect === 'past-end') await list(cursor); + break; + } + if (defect === 'recover-trim' && result.nextCursor !== '') { + try { + await list(result.nextCursor.trim()); + } catch { + // Recover with the advertised cursor; the earlier violation remains. + } + } + cursor = + defect === 'trim-cursor' ? result.nextCursor.trim() : result.nextCursor; + } + if (tools.some((tool) => tool.name === 'add_numbers')) { + const params = { name: 'add_numbers', arguments: { a: 2, b: 3 } }; + if (stateless) await request('tools/call', params); + else await client.callTool(params); + } + // A NEW discovery after the requested traversal and tool call is allowed. + if (defect === 'rediscover') await list(); + } finally { + await client.close(); + } +} + +runAsCli(runClient, import.meta.url); diff --git a/src/scenarios/client/pagination.test.ts b/src/scenarios/client/pagination.test.ts new file mode 100644 index 00000000..689b6efc --- /dev/null +++ b/src/scenarios/client/pagination.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, test } from 'vitest'; +import { getHandler } from '../../../examples/clients/typescript/everything-client'; +import { + runClient, + type PaginationDefect +} from '../../../examples/clients/typescript/pagination-broken-client'; +import { getScenario, listCoreScenarios } from '../index'; +import { testScenarioContext } from '../../mock-server/testing'; +import { DRAFT_PROTOCOL_VERSION, LATEST_SPEC_VERSION } from '../../types'; +import { PaginationScenario } from './pagination'; +import { + InlineClientRunner, + runClientAgainstScenario +} from './auth/test_helpers/testClient'; + +const EMPTY = 'pagination-empty-cursor'; +const OPAQUE = 'pagination-opaque-cursor'; +const TOOL = 'pagination-tool-discovered'; +const END = 'pagination-stops-at-end'; +const CHECK_IDS = [EMPTY, OPAQUE, TOOL, END]; + +test('client-pagination is registered in the core suite', () => { + expect(getScenario('client-pagination')).toBeInstanceOf(PaginationScenario); + expect(listCoreScenarios()).toContain('client-pagination'); +}); + +describe.each([LATEST_SPEC_VERSION, DRAFT_PROTOCOL_VERSION] as const)( + 'client-pagination at %s', + (specVersion) => { + test('everything-client completes the traversal with no skipped checks', async () => { + const handler = getHandler('client-pagination'); + if (!handler) throw new Error('Missing client-pagination handler'); + const checks = await runClientAgainstScenario( + new InlineClientRunner(handler), + 'client-pagination', + { specVersion } + ); + expect(checks.map((check) => check.id)).toEqual(CHECK_IDS); + expect(checks.every((check) => check.status === 'SUCCESS')).toBe(true); + }); + + const faults: Array<{ + defect: PaginationDefect; + failures: string[]; + successes: string[]; + }> = [ + { + defect: 'truthy-cursor', + failures: CHECK_IDS, + successes: [] + }, + { + defect: 'trim-cursor', + failures: [OPAQUE, TOOL, END], + successes: [EMPTY] + }, + { + defect: 'recover-trim', + failures: [OPAQUE], + successes: [EMPTY, TOOL, END] + }, + { + defect: 'empty-page', + failures: [OPAQUE, TOOL, END], + successes: [EMPTY] + }, + { + defect: 'past-end', + failures: [END], + successes: [EMPTY, OPAQUE, TOOL] + } + ]; + for (const { defect, failures, successes } of faults) { + test(`detects the ${defect} control`, async () => { + const checks = await runClientAgainstScenario( + new InlineClientRunner((url) => runClient(url, defect)), + 'client-pagination', + { + specVersion, + expectedFailureSlugs: failures, + expectedSuccessSlugs: successes + } + ); + expect( + checks + .filter((check) => check.status !== 'SUCCESS') + .map((check) => check.id) + ).toEqual(failures); + if (defect === 'past-end') { + expect(checks.find((check) => check.id === END)?.status).toBe( + 'WARNING' + ); + } + }); + } + + test('an inert client cannot pass any unexercised check', async () => { + const checks = await runClientAgainstScenario( + new InlineClientRunner((url) => runClient(url, 'inert')), + 'client-pagination', + { specVersion, expectedFailureSlugs: CHECK_IDS } + ); + expect(checks.map((check) => check.id)).toEqual(CHECK_IDS); + expect(checks.every((check) => check.details?.untestable === true)).toBe( + true + ); + expect(checks.map((check) => check.status)).toEqual([ + 'FAILURE', + 'FAILURE', + 'FAILURE', + 'WARNING' + ]); + }); + + test('allows a new discovery after the requested traversal and tool call', async () => { + await runClientAgainstScenario( + new InlineClientRunner((url) => runClient(url, 'rediscover')), + 'client-pagination', + { specVersion } + ); + }); + } +); + +test('getChecks is idempotent and the same scenario starts without old evidence', async () => { + const scenario = new PaginationScenario(); + const urls = await scenario.start(testScenarioContext()); + try { + const handler = getHandler('client-pagination'); + if (!handler) throw new Error('Missing client-pagination handler'); + await handler(urls.serverUrl); + const first = scenario.getChecks(); + const second = scenario.getChecks(); + expect(first.map((check) => [check.id, check.status])).toEqual( + second.map((check) => [check.id, check.status]) + ); + expect(first).toHaveLength(4); + expect(first.every((check) => check.status === 'SUCCESS')).toBe(true); + } finally { + await scenario.stop(); + } + await scenario.start(testScenarioContext()); + try { + expect( + scenario.getChecks().every((check) => check.details?.untestable) + ).toBe(true); + } finally { + await scenario.stop(); + } +}); diff --git a/src/scenarios/client/pagination.ts b/src/scenarios/client/pagination.ts new file mode 100644 index 00000000..79a691b8 --- /dev/null +++ b/src/scenarios/client/pagination.ts @@ -0,0 +1,228 @@ +import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js'; +import type { MockServer, ScenarioContext } from '../../mock-server'; +import type { ConformanceCheck, Scenario, ScenarioUrls } from '../../types'; +import { untestableCheck } from '../untestable'; + +const SPEC_REFERENCES = [ + { + id: 'MCP-Pagination', + url: 'https://modelcontextprotocol.io/specification/2025-06-18/server/utilities/pagination#implementation-guidelines' + }, + { + id: 'MCP-Pagination-Empty-Cursor', + url: 'https://modelcontextprotocol.io/specification/2026-07-28/server/utilities/pagination#implementation-guidelines' + } +]; + +// No cursor format is defined by the protocol. In particular, clients must not +// trim whitespace, decode punctuation, or stop on the empty-string cursor. +const OPAQUE_CURSOR = ' 0+/%雪?= '; +const ADD_NUMBERS = { + name: 'add_numbers', + description: 'Add two numbers', + inputSchema: { + type: 'object', + properties: { a: { type: 'number' }, b: { type: 'number' } }, + required: ['a', 'b'] + } +}; + +export class PaginationScenario implements Scenario { + name = 'client-pagination'; + // Opaque string cursors and variable page sizes are already specified here; + // the 2026 revision additionally spells out that an empty string is valid. + readonly source = { introducedIn: '2025-06-18' } as const; + description = + 'Enumerate tools across three pages, preserving empty and opaque cursors, ' + + 'then call the discovered add_numbers tool. Complete this one traversal ' + + 'when nextCursor is absent; page sizes are 1, 0, and 2.'; + private srv: MockServer | null = null; + + async start(ctx: ScenarioContext): Promise { + this.srv = await ctx.createServer({ + 'tools/list': (params) => { + if (params.cursor === undefined) { + return { + tools: [ + { name: 'pagination_decoy', inputSchema: { type: 'object' } } + ], + nextCursor: '' + }; + } + if (params.cursor === '') { + return { tools: [], nextCursor: OPAQUE_CURSOR }; + } + if (params.cursor === OPAQUE_CURSOR) { + return { + tools: [ + ADD_NUMBERS, + { name: 'pagination_tail', inputSchema: { type: 'object' } } + ] + }; + } + throw new McpError( + ErrorCode.InvalidParams, + 'Unknown pagination cursor' + ); + }, + 'tools/call': (params) => { + const args = params.arguments as + | { a?: unknown; b?: unknown } + | undefined; + if ( + params.name !== ADD_NUMBERS.name || + typeof args?.a !== 'number' || + typeof args.b !== 'number' + ) { + throw new McpError( + ErrorCode.InvalidParams, + 'Expected add_numbers(a, b)' + ); + } + return { content: [{ type: 'text', text: String(args.a + args.b) }] }; + } + }); + return { serverUrl: this.srv.url }; + } + + async stop(): Promise { + await this.srv?.close(); + this.srv = null; + } + + getChecks(): ConformanceCheck[] { + const requests = this.srv?.recorded ?? []; + const first = requests.findIndex( + (r) => r.method === 'tools/list' && r.params?.cursor === undefined + ); + const empty = requests.findIndex( + (r, i) => + i > first && r.method === 'tools/list' && r.params?.cursor === '' + ); + const terminal = requests.findIndex( + (r, i) => + i > empty && + r.method === 'tools/list' && + r.params?.cursor === OPAQUE_CURSOR + ); + const called = requests.findIndex((r, i) => { + const args = r.params?.arguments as + | { a?: unknown; b?: unknown } + | undefined; + return ( + i > terminal && + r.method === 'tools/call' && + r.params?.name === ADD_NUMBERS.name && + typeof args?.a === 'number' && + typeof args.b === 'number' + ); + }); + const timestamp = new Date().toISOString(); + const emptyContinued = first >= 0 && empty > first; + const terminalReached = emptyContinued && terminal > empty; + // Eventual success must not hide a modified cursor followed by a retry. + const unknownCursors = requests + .filter( + (r, i) => + i > empty && + (called < 0 || i < called) && + r.method === 'tools/list' && + r.params?.cursor !== undefined && + r.params.cursor !== '' && + r.params.cursor !== OPAQUE_CURSOR + ) + .map((r) => r.params?.cursor); + const opaquePreserved = terminalReached && unknownCursors.length === 0; + const taskCompleted = terminalReached && called > terminal; + // A new discovery AFTER the requested tool call is legitimate. Only grade + // requests within this single enumeration, not all future tools/list calls. + const ranPastEnd = requests + .slice(terminal + 1, called >= 0 ? called : undefined) + .some((r) => r.method === 'tools/list'); + return [ + first >= 0 + ? { + id: 'pagination-empty-cursor', + name: 'PaginationEmptyCursor', + description: 'Client continues with the empty-string nextCursor', + status: emptyContinued ? 'SUCCESS' : 'FAILURE', + timestamp, + specReferences: SPEC_REFERENCES, + errorMessage: emptyContinued + ? undefined + : 'Client did not continue the listing with cursor: ""', + details: { firstRequest: first, emptyCursorRequest: empty } + } + : untestableCheck( + 'pagination-empty-cursor', + 'PaginationEmptyCursor', + 'Client continues with the empty-string nextCursor', + 'Client never requested the initial tools/list page', + SPEC_REFERENCES + ), + emptyContinued + ? { + id: 'pagination-opaque-cursor', + name: 'PaginationOpaqueCursor', + description: + 'Client preserves the opaque cursor and continues past an empty page', + status: opaquePreserved ? 'SUCCESS' : 'FAILURE', + timestamp, + specReferences: SPEC_REFERENCES, + errorMessage: opaquePreserved + ? undefined + : 'Client did not preserve the opaque cursor from the empty second page', + details: { + emptyCursorRequest: empty, + terminalPageRequest: terminal, + unknownCursors + } + } + : untestableCheck( + 'pagination-opaque-cursor', + 'PaginationOpaqueCursor', + 'Client preserves the opaque cursor from the second page', + 'Client never requested the page that advertises the opaque cursor', + SPEC_REFERENCES + ), + taskCompleted + ? { + id: 'pagination-tool-discovered', + name: 'PaginationToolDiscovered', + description: + 'Client calls add_numbers after discovering the last page', + status: 'SUCCESS', + timestamp, + specReferences: SPEC_REFERENCES + } + : untestableCheck( + 'pagination-tool-discovered', + 'PaginationToolDiscovered', + 'Client completes the scenario by calling the last-page tool', + 'No valid add_numbers call followed the complete pagination traversal', + SPEC_REFERENCES + ), + taskCompleted + ? { + id: 'pagination-stops-at-end', + name: 'PaginationStopsAtEnd', + description: + 'Client treats an absent nextCursor as the end of this traversal (SHOULD)', + status: ranPastEnd ? 'WARNING' : 'SUCCESS', + timestamp, + specReferences: SPEC_REFERENCES, + errorMessage: ranPastEnd + ? 'Client requested another page after nextCursor was absent, before completing the traversal task' + : undefined + } + : untestableCheck( + 'pagination-stops-at-end', + 'PaginationStopsAtEnd', + 'Client treats an absent nextCursor as the end of this traversal (SHOULD)', + 'Client did not complete the requested traversal and tool call', + SPEC_REFERENCES, + 'WARNING' + ) + ]; + } +} diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index ff377f54..e8e964e5 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -13,6 +13,7 @@ import { InitializeScenario } from './client/initialize'; import { SkillsNoPrefetchScenario } from './client/skills/no-prefetch'; import { SkillsVerificationScenario } from './client/skills/verification'; import { ToolsCallScenario } from './client/tools_call'; +import { PaginationScenario } from './client/pagination'; import { ElicitationClientDefaultsScenario } from './client/elicitation-defaults'; import { SSERetryScenario } from './client/sse-retry'; import { RequestMetadataScenario } from './client/request-metadata'; @@ -319,6 +320,7 @@ export const clientScenariosForAuthorizationServer = new Map< const scenariosList: Scenario[] = [ new InitializeScenario(), new ToolsCallScenario(), + new PaginationScenario(), new ElicitationClientDefaultsScenario(), new SSERetryScenario(), new RequestMetadataScenario(), @@ -354,6 +356,7 @@ const scenariosList: Scenario[] = [ const coreScenariosList: Scenario[] = [ new InitializeScenario(), new ToolsCallScenario(), + new PaginationScenario(), new ElicitationClientDefaultsScenario(), new SSERetryScenario(), ...authScenariosList