diff --git a/README.md b/README.md index dc84f102..5ba49836 100644 --- a/README.md +++ b/README.md @@ -64,11 +64,12 @@ npx @modelcontextprotocol/conformance client --command "" --scen - `--command` - The command to run your MCP client (can include flags) - `--scenario` - The test scenario to run (e.g., "initialize") - `--suite` - Run a suite of tests in parallel (e.g., "auth") +- `--spec-version ` - Filter scenarios by spec version (e.g., `2025-11-25`, `draft`). `draft` runs the latest dated release plus any draft-only scenarios - `--expected-failures ` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures)) - `--timeout` - Timeout in milliseconds (default: 30000) - `--verbose` - Show verbose output -The framework appends `` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. +The framework appends `` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, the corresponding wire protocol version is forwarded as `MCP_CONFORMANCE_PROTOCOL_VERSION` (e.g., `--spec-version draft` sets it to the current draft identifier such as `DRAFT-2026-v1`); example clients can use this value directly as their `protocolVersion`. SDKs that hard-code their protocol version can ignore it. ### Server Testing diff --git a/src/checks/checks.test.ts b/src/checks/checks.test.ts index a08ba08a..b82f17c9 100644 --- a/src/checks/checks.test.ts +++ b/src/checks/checks.test.ts @@ -1,4 +1,5 @@ import { createClientInitializationCheck } from './client'; +import { DRAFT_PROTOCOL_VERSION } from '../types'; describe('createClientInitializationCheck', () => { it('should return SUCCESS for a valid initialize request', () => { @@ -68,6 +69,31 @@ describe('createClientInitializationCheck', () => { expect(check.errorMessage).toContain('Client version missing'); }); + it('should accept the current draft protocol version', () => { + const request = { + protocolVersion: DRAFT_PROTOCOL_VERSION, + clientInfo: { name: 'TestClient', version: '1.0.0' } + }; + + const check = createClientInitializationCheck(request); + expect(check.status).toBe('SUCCESS'); + expect(check.errorMessage).toBeUndefined(); + }); + + it.each(['DRAFT-2025-v1', 'draft'])( + 'should reject stale or non-canonical draft version %s', + (protocolVersion) => { + const request = { + protocolVersion, + clientInfo: { name: 'TestClient', version: '1.0.0' } + }; + + const check = createClientInitializationCheck(request); + expect(check.status).toBe('FAILURE'); + expect(check.errorMessage).toContain('Version mismatch'); + } + ); + it('should support custom expected spec version', () => { const request = { protocolVersion: '2024-11-05', diff --git a/src/checks/client.ts b/src/checks/client.ts index 99834f92..340f603f 100644 --- a/src/checks/client.ts +++ b/src/checks/client.ts @@ -1,4 +1,9 @@ -import { ConformanceCheck, CheckStatus } from '../types'; +import { + ConformanceCheck, + CheckStatus, + LATEST_SPEC_VERSION, + DRAFT_PROTOCOL_VERSION +} from '../types'; export function createServerInfoCheck(serverInfo: { name: string; @@ -23,12 +28,16 @@ export function createServerInfoCheck(serverInfo: { }; } -// Valid MCP protocol versions -const VALID_PROTOCOL_VERSIONS = ['2025-06-18', '2025-11-25']; +// Protocol versions the mock server will accept on initialize. +const VALID_PROTOCOL_VERSIONS = [ + '2025-06-18', + LATEST_SPEC_VERSION, + DRAFT_PROTOCOL_VERSION +]; export function createClientInitializationCheck( initializeRequest: any, - expectedSpecVersion: string = '2025-11-25' + expectedSpecVersion: string = LATEST_SPEC_VERSION ): ConformanceCheck { const protocolVersionSent = initializeRequest?.protocolVersion; diff --git a/src/index.ts b/src/index.ts index 38f7701a..64d1eaaa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -152,7 +152,8 @@ program options.command, scenarioName, timeout, - outputDir + outputDir, + specVersionFilter ); return { scenario: scenarioName, @@ -259,7 +260,8 @@ program validated.command, validated.scenario, timeout, - outputDir + outputDir, + specVersionFilter ); const { overallFailure } = printClientResults( diff --git a/src/runner/client.ts b/src/runner/client.ts index 4525416c..571d8f9d 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -1,7 +1,11 @@ import { spawn } from 'child_process'; import { promises as fs } from 'fs'; import path from 'path'; -import { ConformanceCheck } from '../types'; +import { + ConformanceCheck, + SpecVersion, + specVersionToProtocolVersion +} from '../types'; import { getScenario } from '../scenarios'; import { createResultDir, formatPrettyChecks } from './utils'; @@ -17,7 +21,8 @@ async function executeClient( scenarioName: string, serverUrl: string, timeout: number = 30000, - context?: Record + context?: Record, + specVersion?: SpecVersion ): Promise { const commandParts = command.split(' '); const executable = commandParts[0]; @@ -34,6 +39,12 @@ async function executeClient( // 3. Semantic separation: scenario identifies "which test", context provides "test data" const env = { ...process.env }; env.MCP_CONFORMANCE_SCENARIO = scenarioName; + const protocolVersion = specVersion + ? specVersionToProtocolVersion(specVersion) + : undefined; + if (protocolVersion) { + env.MCP_CONFORMANCE_PROTOCOL_VERSION = protocolVersion; + } if (context) { // Include scenario name in context for discriminated union parsing env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({ @@ -92,7 +103,8 @@ export async function runConformanceTest( clientCommand: string, scenarioName: string, timeout: number = 30000, - outputDir?: string + outputDir?: string, + specVersion?: SpecVersion ): Promise<{ checks: ConformanceCheck[]; clientOutput: ClientExecutionResult; @@ -123,7 +135,8 @@ export async function runConformanceTest( scenarioName, urls.serverUrl, timeout, - urls.context + urls.context, + specVersion ); // Print stdout/stderr if client exited with nonzero code diff --git a/src/scenarios/client/initialize.ts b/src/scenarios/client/initialize.ts index 70fb0d1a..ccf0d127 100644 --- a/src/scenarios/client/initialize.ts +++ b/src/scenarios/client/initialize.ts @@ -3,7 +3,9 @@ import { Scenario, ScenarioUrls, ConformanceCheck, - SpecVersion + SpecVersion, + LATEST_SPEC_VERSION, + DRAFT_PROTOCOL_VERSION } from '../../types'; import { clientChecks } from '../../checks/index'; @@ -117,11 +119,15 @@ export class InitializeScenario implements Scenario { this.checks.push(clientChecks.createServerInfoCheck(serverInfo)); // Echo back client's version if valid, otherwise use latest - const VALID_VERSIONS = ['2025-06-18', '2025-11-25']; + const VALID_VERSIONS = [ + '2025-06-18', + LATEST_SPEC_VERSION, + DRAFT_PROTOCOL_VERSION + ]; const clientVersion = initializeRequest?.protocolVersion; const responseVersion = VALID_VERSIONS.includes(clientVersion) ? clientVersion - : '2025-11-25'; + : LATEST_SPEC_VERSION; const response = { jsonrpc: '2.0', diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 70808490..f364073e 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -2,7 +2,9 @@ import { Scenario, ClientScenario, ClientScenarioForAuthorizationServer, - SpecVersion + SpecVersion, + DATED_SPEC_VERSIONS, + LATEST_SPEC_VERSION } from '../types'; import { InitializeScenario } from './client/initialize'; import { ToolsCallScenario } from './client/tools_call'; @@ -257,9 +259,7 @@ export { listMetadataScenarios }; // All valid spec versions, used by the CLI to validate --spec-version input. export const ALL_SPEC_VERSIONS: SpecVersion[] = [ - '2025-03-26', - '2025-06-18', - '2025-11-25', + ...DATED_SPEC_VERSIONS, 'draft', 'extension' ]; @@ -273,15 +273,31 @@ export function resolveSpecVersion(value: string): SpecVersion { process.exit(1); } +// `draft` selects everything in the latest dated release plus scenarios tagged +// draft-only, so SEP authors can run the full suite against an SDK tracking the +// in-progress spec without retagging core scenarios. +function matchesSpecVersion( + scenario: { specVersions: SpecVersion[] }, + version: SpecVersion +): boolean { + if (version === 'draft') { + return ( + scenario.specVersions.includes('draft') || + scenario.specVersions.includes(LATEST_SPEC_VERSION) + ); + } + return scenario.specVersions.includes(version); +} + export function listScenariosForSpec(version: SpecVersion): string[] { return scenariosList - .filter((s) => s.specVersions.includes(version)) + .filter((s) => matchesSpecVersion(s, version)) .map((s) => s.name); } export function listClientScenariosForSpec(version: SpecVersion): string[] { return allClientScenariosList - .filter((s) => s.specVersions.includes(version)) + .filter((s) => matchesSpecVersion(s, version)) .map((s) => s.name); } @@ -289,7 +305,7 @@ export function listClientScenariosForAuthorizationServerForSpec( version: SpecVersion ): string[] { return allClientScenariosListForAuthorizationServer - .filter((s) => s.specVersions.includes(version)) + .filter((s) => matchesSpecVersion(s, version)) .map((s) => s.name); } diff --git a/src/scenarios/spec-version.test.ts b/src/scenarios/spec-version.test.ts index 0b8e652f..d875df85 100644 --- a/src/scenarios/spec-version.test.ts +++ b/src/scenarios/spec-version.test.ts @@ -3,9 +3,16 @@ import { listScenarios, listClientScenarios, listScenariosForSpec, + listDraftScenarios, getScenarioSpecVersions, ALL_SPEC_VERSIONS } from './index'; +import { + DATED_SPEC_VERSIONS, + DRAFT_PROTOCOL_VERSION, + LATEST_SPEC_VERSION, + specVersionToProtocolVersion +} from '../types'; describe('specVersions helpers', () => { it('every Scenario has specVersions', () => { @@ -69,26 +76,41 @@ describe('specVersions helpers', () => { } }); - it('draft and extension scenarios are isolated', () => { - const draft = listScenariosForSpec('draft'); - for (const name of draft) { - expect(getScenarioSpecVersions(name)).toContain('draft'); + it('--spec-version draft is a superset of the latest dated release', () => { + const latest = new Set(listScenariosForSpec(LATEST_SPEC_VERSION)); + const draft = new Set(listScenariosForSpec('draft')); + for (const name of latest) { + expect(draft.has(name)).toBe(true); } - const ext = listScenariosForSpec('extension'); - for (const name of ext) { - expect(getScenarioSpecVersions(name)).toContain('extension'); + for (const name of listDraftScenarios()) { + expect(draft.has(name)).toBe(true); + } + }); + + it('draft-tagged scenarios are not also tagged with a dated version', () => { + for (const name of listDraftScenarios()) { + const versions = getScenarioSpecVersions(name)!; + for (const dated of DATED_SPEC_VERSIONS) { + expect( + versions, + `scenario "${name}" is tagged with both 'draft' and '${dated}'` + ).not.toContain(dated); + } } }); - it('draft scenarios are not in dated versions', () => { - const draft = listScenariosForSpec('draft'); - const dated = new Set([ - ...listScenariosForSpec('2025-03-26'), - ...listScenariosForSpec('2025-06-18'), - ...listScenariosForSpec('2025-11-25') - ]); - for (const name of draft) { - expect(dated.has(name)).toBe(false); + it('specVersionToProtocolVersion maps tags to wire versions', () => { + expect(specVersionToProtocolVersion('draft')).toBe(DRAFT_PROTOCOL_VERSION); + expect(specVersionToProtocolVersion(LATEST_SPEC_VERSION)).toBe( + LATEST_SPEC_VERSION + ); + expect(specVersionToProtocolVersion('extension')).toBeUndefined(); + }); + + it('extension scenarios are isolated', () => { + const ext = listScenariosForSpec('extension'); + for (const name of ext) { + expect(getScenarioSpecVersions(name)).toContain('extension'); } }); }); diff --git a/src/types.ts b/src/types.ts index 193686f6..33ab7b37 100644 --- a/src/types.ts +++ b/src/types.ts @@ -23,12 +23,34 @@ export interface ConformanceCheck { logs?: string[]; } -export type SpecVersion = - | '2025-03-26' - | '2025-06-18' - | '2025-11-25' - | 'draft' - | 'extension'; +export const DATED_SPEC_VERSIONS = [ + '2025-03-26', + '2025-06-18', + '2025-11-25' +] as const; + +export type DatedSpecVersion = (typeof DATED_SPEC_VERSIONS)[number]; + +export const LATEST_SPEC_VERSION: DatedSpecVersion = '2025-11-25'; + +// Mirrors LATEST_PROTOCOL_VERSION in the spec repo's schema/draft/schema.ts. +// Bump when that constant changes. +export const DRAFT_PROTOCOL_VERSION = 'DRAFT-2026-v1'; + +export type SpecVersion = DatedSpecVersion | 'draft' | 'extension'; + +export function specVersionToProtocolVersion( + version: SpecVersion +): string | undefined { + if (version === 'draft') return DRAFT_PROTOCOL_VERSION; + // TODO(#253 follow-up): 'extension' isn't a spec version — it's a scenario + // category that got lumped into SpecVersion so `--spec-version extension` + // could reuse the filter plumbing. It has no corresponding wire + // protocolVersion. Split it out of this type when moving to + // introducedIn/removedIn tagging. + if (version === 'extension') return undefined; + return version; +} export interface ScenarioUrls { serverUrl: string;