import { test } from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync, writeFileSync, existsSync } from 'node:fs'; import { join } from 'node:path'; import { FIXTURES, ROOT, run, tempDir } from './helpers.mjs'; import { start } from './fixtures/http-server.mjs'; import { start as startRaw } from './fixtures/raw-http-server.mjs'; const sdkServer = ['--', process.execPath, join(FIXTURES, 'sdk-server.mjs')]; const raw = (fixture) => ['--env', `FIXTURE=${fixture}`, '--', process.execPath, join(FIXTURES, 'raw-server.mjs')]; test('snapshot of a 2026-07-28 stdio server writes menu.json and exits 0', async () => { const r = await run(['snapshot', ...sdkServer]); assert.equal(r.code, 0, r.stderr); assert.match(r.stdout, /protocol 2026-07-28 · 3 tools/); const menu = JSON.parse(readFileSync(join(r.cwd, 'menu.json'), 'utf8')); assert.equal(menu.toolmenu, 1); assert.deepEqual(menu.tools.map((t) => t.name), ['search_forms', 'get_form', 'delete_form']); assert.equal(menu.server.era, 'modern'); assert.equal(menu.listMeta.cacheScope, 'private'); assert.ok(menu.tools.every((t) => t.tokens > 0)); assert.equal(menu.totalTokens, menu.tools.reduce((s, t) => s + t.tokens, 0)); }); test('a 2025-era server is reported, and 2026 rules are skipped', async () => { const r = await run(['snapshot', '--no-write', '--json', '--env', 'LEGACY=1', ...sdkServer]); assert.equal(r.code, 0, r.stderr); const out = JSON.parse(r.stdout); assert.equal(out.server.protocolVersion, '2025-11-25'); assert.deepEqual(out.findings.map((f) => f.rule), ['spec/discover']); assert.ok(!existsSync(join(r.cwd, 'menu.json'))); }); test('snapshot over Streamable HTTP, with a header', async () => { const server = await start(); try { const r = await run(['snapshot', '--no-write', '--json', '--header', 'Authorization: Bearer t0ken', server.url]); assert.equal(r.code, 0, r.stderr); const out = JSON.parse(r.stdout); assert.equal(out.server.protocolVersion, '2026-07-28'); assert.equal(out.tools, 3); assert.ok(server.seen.authorization.includes('Bearer t0ken')); } finally { await server.close(); } }); test('cli: snapshot, session --init and --auto wait out a rate limit, and say so when it outlasts the waits', async () => { const env = { TOOLMENU_RATE_LIMIT_WAITS_MS: '150,300' }; const commands = [['snapshot', '--no-write', '--json'], ['session', '--init'], ['session', '--auto']]; // Refused from the first request for 100 ms: each is waited out. for (const args of commands) { const server = await start({ limit: 0, resetAfterMs: 100 }); try { const r = await run([...args, server.url], { env }); assert.equal(r.code, 0, `${args.join(' ')}: ${r.stderr}`); assert.ok(server.seen.refused > 0, args.join(' ')); } finally { await server.close(); } } // Refused throughout: the explanation, not the transport's error. for (const args of commands) { // The refusal as a JSON-RPC body: its message is quoted, not the JSON. const server = await start({ limit: 0, json: true }); try { const r = await run([...args, server.url], { env }); assert.equal(r.code, 2, `${args.join(' ')}: ${r.stderr}`); assert.match(r.stderr, /^toolmenu: Rate-limited while connecting, and still after waiting 0\.5 s: “Too many requests — retry later”\. The limit counts every request from this address/, args.join(' ')); assert.doesNotMatch(r.stderr, /Error POSTing|jsonrpc/, args.join(' ')); } finally { await server.close(); } } }); test('--json stays valid JSON when a library logs, and a missing tools capability is an error', async () => { const r = await run(['snapshot', '--json', '--no-write', '--processes', '1', '--env', 'NO_TOOLS_CAPABILITY=1', ...raw('clean')]); assert.equal(r.code, 1, r.stderr); const report = JSON.parse(r.stdout); assert.equal(report.tools, 0); assert.ok(report.findings.some((f) => f.rule === 'spec/tools-capability' && f.severity === 'error')); assert.match(r.stderr, /does not advertise tools capability/, 'the SDK line went to stderr'); }); test('an unstable order fails the run', async () => { const r = await run(['snapshot', '--no-write', ...raw('shuffle')]); assert.equal(r.code, 1); assert.match(r.stdout, /ERROR menu\/nondeterministic/); assert.match(r.stdout, /different order/); }); test('a description that changes between calls fails the run', async () => { const r = await run(['snapshot', '--no-write', ...raw('drift')]); assert.equal(r.code, 1); assert.match(r.stdout, /get_time: description: "Get the time. Generated at call 1." vs "Get the time. Generated at call 2."/); }); test('a result the SDK rejects is still read and reported against the schema', async () => { const r = await run(['snapshot', '--no-write', ...raw('badschema')]); assert.equal(r.code, 1); assert.match(r.stdout, /ERROR spec\/schema/); assert.match(r.stdout, /required property 'inputSchema'/); assert.match(r.stdout, /official MCP SDK client rejects/); }); test('warnings pass by default and fail with --fail-on warn', async () => { const r = await run(['snapshot', '--no-write', ...raw('smells')]); assert.equal(r.code, 0, r.stderr); for (const rule of ['naming/vague-id', 'ids/authored', 'write/unannotated', 'write/no-dry-run', 'naming/shared-word']) { assert.match(r.stdout, new RegExp(rule)); } const strict = await run(['snapshot', '--no-write', '--fail-on', 'warn', ...raw('smells')]); assert.equal(strict.code, 1); }); test('routes.yml pins the audit collision', async () => { const cwd = tempDir(); writeFileSync(join(cwd, 'routes.yml'), 'audit:\n must_match: [list_team_audits]\n must_not_match: [get_audit_trail]\n'); const r = await run(['snapshot', '--no-write', '--routes', 'routes.yml', ...raw('smells')], { cwd }); assert.equal(r.code, 1); assert.match(r.stdout, /ERROR naming\/route/); assert.match(r.stdout, /"audit" matches get_audit_trail/); }); test('config file changes severities', async () => { const cwd = tempDir(); writeFileSync(join(cwd, 'toolmenu.config.json'), JSON.stringify({ rules: { 'naming/vague-id': 'error' } })); const r = await run(['snapshot', '--no-write', ...raw('smells')], { cwd }); assert.equal(r.code, 1); assert.match(r.stdout, /ERROR naming\/vague-id/); }); test('a config that names a retired rule loads, and says the setting changes nothing', async () => { const cwd = tempDir(); writeFileSync(join(cwd, 'toolmenu.config.json'), JSON.stringify({ rules: { 'description/late-instruction': 'error' } })); const r = await run(['snapshot', '--no-write', ...raw('smells')], { cwd }); assert.notEqual(r.code, 2, r.stderr); assert.match(r.stderr, /rules\["description\/late-instruction"\] changes nothing: retired after 0\.28\.1/); assert.doesNotMatch(r.stdout, /late-instruction/); }); test('a config that names a rule that does not exist loads, says so and names the closest', async () => { const cwd = tempDir(); writeFileSync(join(cwd, 'toolmenu.config.json'), JSON.stringify({ rules: { 'naming/vauge-id': 'error', 'naming/vague-id': 'error' } })); const r = await run(['snapshot', '--no-write', ...raw('smells')], { cwd }); assert.equal(r.code, 1, r.stderr); assert.match(r.stderr, /rules\["naming\/vauge-id"\] isn't a rule, so it changes nothing\. Did you mean naming\/vague-id\?/); assert.doesNotMatch(r.stderr, /rules\["naming\/vague-id"\]/); assert.match(r.stdout, /ERROR naming\/vague-id/); }); test('github format prints workflow annotations', async () => { const r = await run(['snapshot', '--no-write', '--format', 'github', ...raw('shuffle')]); assert.match(r.stdout, /^::error title=toolmenu menu\/nondeterministic::/m); }); test('usage errors exit 2', async () => { assert.equal((await run(['snapshot'])).code, 2); assert.equal((await run(['snapshot', 'not-a-url'])).code, 2); assert.equal((await run(['session', 'x'])).code, 2, 'session without --scenario'); assert.equal((await run(['snapshot', '--format', 'xml', 'http://x'])).code, 2); assert.equal((await run([])).code, 2); assert.equal((await run(['--help'])).code, 0); }); test('a server that fails to start exits 2 with a message', async () => { const r = await run(['snapshot', '--no-write', '--timeout', '5000', '--', process.execPath, '-e', 'process.exit(3)']); assert.equal(r.code, 2); assert.match(r.stderr, /^toolmenu: /); }); test('cli: a broken config file is named in the error', async () => { const cwd = tempDir(); writeFileSync(join(cwd, 'toolmenu.config.json'), '{"rules": {'); const r = await run(['diff', 'a.json', 'b.json'], { cwd }); assert.equal(r.code, 2); assert.match(r.stderr, /toolmenu\.config\.json isn't valid JSON/); }); test('cli: --version is package.json\'s version', async () => { const r = await run(['--version']); assert.equal(r.stdout.trim(), JSON.parse(readFileSync(join(ROOT, 'package.json'), 'utf8')).version); }); test('snapshot: a menu that differs per process is caught, and says what differs', async () => { const r = await run(['snapshot', '--no-write', ...raw('seedorder')]); assert.equal(r.code, 1); assert.match(r.stdout, /ERROR menu\/process-variance/); assert.match(r.stdout, /get_issue: inputSchema\.properties\.fields\.default: same 5 items, different order/); assert.match(r.stdout, /PYTHONHASHSEED=0/); // The main process is pinned: two snapshots save the same menu. const [a, b] = await Promise.all([1, 2].map(async () => { const cwd = tempDir(); await run(['snapshot', '--processes', '1', ...raw('seedorder')], { cwd }); return JSON.parse(readFileSync(join(cwd, 'menu.json'), 'utf8')).tools; })); assert.deepEqual(a, b); }); test('snapshot: --processes 1 turns the check off; a stable server passes it', async () => { const off = await run(['snapshot', '--no-write', '--processes', '1', ...raw('seedorder')]); assert.doesNotMatch(off.stdout, /process-variance/); const clean = await run(['snapshot', '--no-write', ...raw('clean')]); assert.doesNotMatch(clean.stdout, /process-variance/); const bad = await run(['snapshot', '--no-write', '--processes', '0', ...raw('clean')]); assert.equal(bad.code, 2); }); test('snapshot over http: a menu that differs per connection is caught', async () => { const server = await startRaw({ vary: true }); try { const r = await run(['snapshot', '--no-write', server.url]); assert.match(r.stdout, /menu\/connection-variance/); } finally { await server.close(); } }); test('--protocol legacy checks a dual-era server as a 2025 server', async () => { const r = await run(['snapshot', '--no-write', '--json', '--protocol', 'legacy', ...sdkServer]); assert.equal(r.code, 0, r.stderr); const out = JSON.parse(r.stdout); assert.equal(out.server.protocolVersion, '2025-11-25'); assert.equal(out.server.era, 'legacy'); assert.deepEqual(out.findings.map((f) => f.rule), ['spec/discover']); }); test('--protocol modern pins 2026-07-28, and says what to do when the server lacks it', async () => { const ok = await run(['snapshot', '--no-write', '--json', '--protocol', 'modern', ...sdkServer]); assert.equal(ok.code, 0, ok.stderr); assert.equal(JSON.parse(ok.stdout).server.protocolVersion, '2026-07-28'); const bad = await run(['snapshot', '--no-write', '--protocol', 'modern', '--env', 'LEGACY=1', ...sdkServer]); assert.equal(bad.code, 2); assert.match(bad.stderr, /doesn't speak the 2026-07-28 protocol/); assert.match(bad.stderr, /→ Next: .*--protocol auto/); }); test('--protocol and the config file: the flag wins, and a bad value is a usage error', async () => { const dir = tempDir(); writeFileSync(join(dir, 'toolmenu.config.json'), JSON.stringify({ protocol: 'legacy' })); const fromConfig = await run(['snapshot', '--no-write', '--json', ...sdkServer], { cwd: dir }); assert.equal(JSON.parse(fromConfig.stdout).server.era, 'legacy'); const flag = await run(['snapshot', '--no-write', '--json', '--protocol', 'auto', ...sdkServer], { cwd: dir }); assert.equal(JSON.parse(flag.stdout).server.era, 'modern'); const bad = await run(['snapshot', '--protocol', 'old', ...sdkServer]); assert.equal(bad.code, 2); assert.match(bad.stderr, /--protocol must be auto, legacy or modern/); writeFileSync(join(dir, 'toolmenu.config.json'), JSON.stringify({ protocol: 'old' })); const badConfig = await run(['snapshot', '--no-write', ...sdkServer], { cwd: dir }); assert.equal(badConfig.code, 2); assert.match(badConfig.stderr, /protocol must be auto, legacy or modern/); }); // 0.21: behind a proxy, a stdio server that calls an API at startup failed with nothing // to say why (Stripe, Supabase): only PATH, HOME and the like reached it. test('stdio servers get this machine\'s proxy and CA settings, and nothing else from it', async () => { const dir = tempDir(); const out = join(dir, 'menu.json'); // This machine's own settings cleared, so only these reach the CLI. const env = { HTTPS_PROXY: 'http://proxy.test:3128', HTTP_PROXY: '', https_proxy: '', http_proxy: '', NO_PROXY: 'localhost', no_proxy: '', NODE_EXTRA_CA_CERTS: '/etc/ca.pem', SSL_CERT_FILE: '', SECRET_TOKEN: 'not-for-the-server' }; const r = await run(['snapshot', '--out', out, '--processes', '1', '--', process.execPath, join(FIXTURES, 'env-server.mjs')], { env: { ...env, NODE_USE_ENV_PROXY: '' } }); assert.equal(r.code, 0, r.stderr); const seen = JSON.parse(JSON.parse(readFileSync(out, 'utf8')).tools[0].description); assert.deepEqual(seen, { HTTPS_PROXY: 'http://proxy.test:3128', NO_PROXY: 'localhost', NODE_USE_ENV_PROXY: '1', NODE_EXTRA_CA_CERTS: '/etc/ca.pem' }); // --env still wins. const own = join(dir, 'own.json'); await run(['snapshot', '--out', own, '--processes', '1', '--env', 'HTTPS_PROXY=http://other.test:8080', '--', process.execPath, join(FIXTURES, 'env-server.mjs')], { env: { ...env, NODE_USE_ENV_PROXY: '' } }); assert.equal(JSON.parse(JSON.parse(readFileSync(own, 'utf8')).tools[0].description).HTTPS_PROXY, 'http://other.test:8080'); }); // 0.21: Atlassian Rovo answered any Authorization header with a menu, not 401. test('auth/invalid-accepted: a server that serves a menu to an invalid credential, not one that refuses it', async () => { const serving = await start({ auth: { token: 'good', onBad: 'serve' } }); const refusing = await start({ auth: { token: 'good', onBad: 'refuse' } }); try { const snap = async (url, extra = []) => JSON.parse((await run(['snapshot', '--no-write', '--json', '--no-auth', ...extra, url])).stdout); const bad = await snap(serving.url, ['--header', 'Authorization: Bearer good']); const [f] = bad.findings.filter((x) => x.rule === 'auth/invalid-accepted'); assert.equal(f.severity, 'warn'); assert.match(f.message, /^With an invalid credential in place of this run's, the server served the same \d+ tools instead of refusing \(401\)\./); // The probe sent a credential that can't be valid, in the same scheme. assert.ok(serving.seen.authorization.includes('Bearer toolmenu-invalid-credential')); const good = await snap(refusing.url, ['--header', 'Authorization: Bearer good']); assert.deepEqual(good.findings.filter((x) => x.rule === 'auth/invalid-accepted'), []); // --processes 1 opens no extra connection, this probe included. const before = refusing.seen.authorization.length; await snap(refusing.url, ['--header', 'Authorization: Bearer good', '--processes', '1']); assert.ok(!refusing.seen.authorization.slice(before).includes('Bearer toolmenu-invalid-credential')); } finally { await serving.close(); await refusing.close(); } }); test('withInvalidCredential: same scheme, every credential header, other headers kept', async () => { const { withInvalidCredential } = await import('../dist/snapshot.js'); const t = withInvalidCredential({ kind: 'http', url: 'https://x.test/mcp', headers: { Authorization: 'Basic dTpw', 'X-Api-Key': 'k', 'X-Team': 'a' } }); assert.equal(t.headers.Authorization, `Basic ${Buffer.from('toolmenu-invalid:toolmenu-invalid').toString('base64')}`); assert.equal(t.headers['X-Api-Key'], 'toolmenu-invalid-credential'); assert.equal(t.headers['X-Team'], 'a'); assert.equal(t.noAuth, true); // A stored login (no header of its own) gets an invalid Bearer token. assert.equal(withInvalidCredential({ kind: 'http', url: 'https://x.test/mcp' }).headers.Authorization, 'Bearer toolmenu-invalid-credential'); }); // 0.22, from the 0.21 corpus run: inite-visibility serves 3 tools offline, 10 online. test('snapshot --offline-check: a menu that depends on the network; a server that needs it to start; quiet when the same', async () => { const { createServer } = await import('node:http'); // "The network": a host only a proxy reaches. The main process is given the proxy // (--env); the offline process gets the dead one in its place, --env or not. const remote = createServer((req, res) => { res.statusCode = /^remote\.test(:\d+)?$/.test(req.headers.host ?? '') ? 200 : 404; res.end(); }); // Node's proxy agent tunnels plain HTTP too: answer CONNECT, then serve the tunnel here. remote.on('connect', (req, sock) => { sock.write('HTTP/1.1 200 Connection Established\r\n\r\n'); remote.emit('connection', sock); }); await new Promise((r) => remote.listen(0, '127.0.0.1', r)); const REMOTE_URL = 'http://remote.test/'; const proxy = `http://127.0.0.1:${remote.address().port}`; const fixture = join(FIXTURES, 'network-server.mjs'); const env = {}; // Both spellings: Node reads the lowercase one first, and the machine running the tests may set it. const via = ['--env', `REMOTE_URL=${REMOTE_URL}`, '--env', `HTTP_PROXY=${proxy}`, '--env', `http_proxy=${proxy}`, '--env', 'NODE_USE_ENV_PROXY=1']; const snap = async (more = []) => JSON.parse((await run(['snapshot', '--no-write', '--json', '--processes', '1', '--offline-check', ...via, ...more, '--', process.execPath, fixture], { env })).stdout).findings.filter((f) => f.rule === 'menu/network-dependent'); try { const [f] = await snap(); assert.equal(f.severity, 'warn'); assert.match(f.message, /^With the network cut off, the server serves a different menu: 1 tool, not 2 \(1 missing\)\./); assert.deepEqual(f.detail, ['missing offline: remote_rules']); const [g] = await snap(['--env', 'REQUIRE=1']); assert.equal(g.severity, 'info'); assert.match(g.message, /^The server didn't start or list its tools with the network cut off: the server exited with code 1/); // Not asked for: not run. const none = JSON.parse((await run(['snapshot', '--no-write', '--json', '--processes', '1', ...via, '--', process.execPath, fixture], { env })).stdout); assert.equal(none.findings.filter((x) => x.rule === 'menu/network-dependent').length, 0); // Loopback isn't the network: a server that reaches 127.0.0.1 directly keeps its menu. const local = createServer((req, res) => res.end('ok')); await new Promise((r) => local.listen(0, '127.0.0.1', r)); const direct = JSON.parse((await run(['snapshot', '--no-write', '--json', '--processes', '1', '--offline-check', '--env', `REMOTE_URL=http://127.0.0.1:${local.address().port}/`, '--', process.execPath, fixture], { env })).stdout); local.close(); assert.deepEqual(direct.findings.filter((x) => x.rule === 'menu/network-dependent'), [], 'a local helper on loopback'); // A server that fetches nothing: quiet. assert.deepEqual(JSON.parse((await run(['snapshot', '--no-write', '--json', '--processes', '1', '--offline-check', '--', process.execPath, join(FIXTURES, 'client-server.mjs')])).stdout).findings.filter((x) => x.rule === 'menu/network-dependent'), []); // The config key, for scan too. const dir = tempDir(); writeFileSync(join(dir, 'toolmenu.config.json'), JSON.stringify({ offlineCheck: true })); writeFileSync(join(dir, 'mcp.json'), JSON.stringify({ mcpServers: { net: { command: process.execPath, args: [fixture], env: { REMOTE_URL, HTTP_PROXY: proxy, http_proxy: proxy, NODE_USE_ENV_PROXY: '1' } } } })); const scanned = JSON.parse((await run(['scan', '--json', '--processes', '1', 'mcp.json'], { cwd: dir, env })).stdout); assert.ok(scanned.servers[0].findings.some((x) => x.rule === 'menu/network-dependent')); writeFileSync(join(dir, 'toolmenu.config.json'), JSON.stringify({ offlineCheck: 'yes' })); assert.match((await run(['snapshot', '--no-write', '--', process.execPath, fixture], { cwd: dir })).stderr, /offlineCheck must be true or false/); const http = await run(['snapshot', '--offline-check', 'https://example.com/mcp']); assert.equal(http.code, 2); assert.match(http.stderr, /--offline-check goes with a stdio server/); } finally { remote.close(); } }); // 0.22, from running github-mcp-server 1.14: delete_repository is listed only to clients with elicitation. test('menu/capability-dependent: a tool listed only to a client with elicitation is in the menu file, and said', async () => { const dir = tempDir(); const out = join(dir, 'menu.json'); const raw = join(FIXTURES, 'raw-server.mjs'); const r = JSON.parse((await run(['snapshot', '--json', '--out', out, '--env', 'FIXTURE=confirm', '--', process.execPath, raw])).stdout); assert.equal(r.tools, 2); assert.deepEqual(JSON.parse(readFileSync(out, 'utf8')).tools.map((t) => t.name), ['get_thing', 'delete_thing'], 'the baseline holds the tool'); const [f] = r.findings.filter((x) => x.rule === 'menu/capability-dependent'); assert.equal(f.severity, 'info'); assert.equal(f.tool, 'delete_thing'); assert.match(f.message, /^delete_thing is listed only to a client that supports elicitation \(can ask the user to confirm\)\. A client without elicitation gets 1 tool, not 2\./); // --processes 1 opens nothing more: not checked. const one = JSON.parse((await run(['snapshot', '--json', '--no-write', '--processes', '1', '--env', 'FIXTURE=confirm', '--', process.execPath, raw])).stdout); assert.equal(one.findings.filter((x) => x.rule === 'menu/capability-dependent').length, 0); // A server that lists the same either way: quiet. const same = JSON.parse((await run(['snapshot', '--json', '--no-write', '--env', 'FIXTURE=clean', '--', process.execPath, raw])).stdout); assert.equal(same.findings.filter((x) => x.rule === 'menu/capability-dependent').length, 0); }); // 0.22, from the corpus run: under load mastra's version probe fell back to 2025 in the // main process only, and the second process's 2026-07-28 menu read as process variance. test('snapshot: a later process that ends on another protocol than the main one is asked again on that one', async () => { const dir = tempDir(); const log = join(dir, 'methods.log'); const r = await run(['snapshot', '--no-write', '--json', '--env', `LOG=${log}`, '--', process.execPath, join(FIXTURES, 'first-legacy-server.mjs')]); const out = JSON.parse(r.stdout); assert.equal(out.server.protocolVersion, '2025-11-25'); assert.match(out.findings.find((f) => f.rule === 'spec/discover').message, /^This connection fell back to protocol 2025-11-25: .* A later process got 2026-07-28, so the server speaks it/); // The first method each process got: the main one, then each later one negotiated // as usual (server/discover) and, having got 2026-07-28, asked again on 2025 (initialize). const [main, ...later] = readFileSync(log, 'utf8').trim().split('\n'); assert.equal(main, 'server/discover'); // (0.22 before this fix sent no initialize after the main process: every later one stayed on 2026-07-28.) assert.ok(later.filter((m) => m === 'initialize').length >= 2, `the variance probe and the capability listing, each asked again on 2025: ${later.join(', ')}`); }); // MCP Apps (ext-apps): the UI a tool names, who sees the tool, and what a client without Apps gets. test('apps/*: UI resources read as a host would, visibility, the deprecated key, app-only tools kept from the agent', async () => { const raw = join(FIXTURES, 'raw-server.mjs'); const snap = async (...env) => JSON.parse((await run(['snapshot', '--json', '--no-write', ...env.flatMap((e) => ['--env', e]), '--', process.execPath, raw])).stdout); const r = await snap('FIXTURE=apps'); const by = (rule) => r.findings.filter((f) => f.rule === rule); const [res] = by('apps/resource'); assert.equal(res.severity, 'warn'); assert.match(res.message, /^3 tools name an MCP Apps UI that a host can't render: show_map, show_table and show_web\. An Apps host then shows no UI for the tool; the agent still gets its text result\.$/); assert.deepEqual(res.detail, [ 'show_map: resources/read of ui://fx/missing failed (Resource not found: ui://fx/missing): the spec says the resource MUST exist on the server', 'show_table: resources/read of ui://fx/table gave mimeType "text/html", not "text/html;profile=mcp-app"', "show_web: https://example.com/view isn't a ui:// URI, which MCP Apps require", ]); assert.match(res.fix, /mimeType "text\/html;profile=mcp-app"/); const [legacy] = by('apps/legacy-key'); assert.equal(legacy.tool, 'show_legacy'); assert.equal(legacy.severity, 'info'); assert.match(legacy.fix, /^Add _meta: \{ ui: \{ resourceUri/); const [vis] = by('apps/visibility'); assert.equal(vis.tool, 'hidden_tool'); assert.match(vis.message, /names neither "model" nor "app", so no one can call it/); // The agent never sees update_chart_data: rules about what the agent reads leave it out. // The host still confirms the app's calls by annotations: write/unannotated names it. assert.equal(by('write/unannotated')[0]?.tool, 'update_chart_data'); assert.ok(!by('write/no-dry-run').some((f) => f.tool === 'update_chart_data' || f.detail?.some((d) => d.includes('update_chart_data'))), 'no-dry-run is about the agent'); assert.deepEqual(r.appOnly, { tools: 2, tokens: r.appOnly.tokens }); assert.ok(!r.breakdown.top.some((t) => t.name === 'update_chart_data'), 'not in the token breakdown'); // Listed only to an Apps host: open_dashboard; a UI variant of show_chart. Not the app-only tool. const [cap] = by('menu/capability-dependent'); assert.match(cap.message, /^open_dashboard is listed only to a client that renders MCP Apps \(declares io\.modelcontextprotocol\/ui\); show_chart is listed with another definition to a client that renders MCP Apps/); assert.match(cap.message, /A client without MCP Apps gets 6 tools, one of them different, not 8\./); assert.equal(by('apps/app-only-listed').length, 0, 'update_chart_data goes only to Apps hosts'); // The same app-only tool listed to every client: said. const leak = await snap('FIXTURE=apps', 'APPS_LEAK=1'); const [l] = leak.findings.filter((f) => f.rule === 'apps/app-only-listed'); assert.equal(l.tool, 'update_chart_data'); assert.match(l.message, /yet listed to a client that doesn't declare MCP Apps too/); // Done right: no apps finding, no capability-dependent note for the app-only tool. const clean = await snap('FIXTURE=apps-clean'); assert.deepEqual(clean.findings.filter((f) => f.rule.startsWith('apps/') || f.rule === 'menu/capability-dependent'), []); }); // ext-server-card: the card SHOULD NOT contradict the live server's version and protocol versions. test("apps/resource: a UI read that failed for the run's reasons is info, not the server's", async () => { const raw = join(FIXTURES, 'raw-server.mjs'); const snap = async (args, ...env) => JSON.parse((await run(['snapshot', '--json', '--no-write', ...args, ...env.flatMap((e) => ['--env', e]), '--', process.execPath, raw])).stdout); const apps = (r) => r.findings.filter((f) => f.rule === 'apps/resource'); // A dummy key (teamwork's "not authenticated"): the chart's and table's reads are info; the missing map stays the server's. const auth = apps(await snap([], 'FIXTURE=apps', 'UI_READ=auth')); assert.equal(auth.length, 2); const [info] = auth.filter((f) => f.severity === 'info'); assert.equal(info.message, "6 tools name MCP Apps UIs that toolmenu couldn't read here (credentials: not authenticated), so they weren't checked: a setup problem, not the server's."); assert.equal(info.detail.length, 6); assert.ok(info.detail.includes('show_table: ui://fx/table: credentials (not authenticated)'), info.detail.join('\n')); assert.equal(info.fix, 'Rerun with the credentials the server needs (--header, --env) to check the UI.'); const [warn] = auth.filter((f) => f.severity === 'warn'); assert.ok(warn.detail.some((d) => d.startsWith('show_map: resources/read of ui://fx/missing failed (Resource not found')), 'a missing resource is still the server\'s'); assert.deepEqual(warn.detail, ['show_map: resources/read of ui://fx/missing failed (Resource not found: ui://fx/missing): the spec says the resource MUST exist on the server', "show_web: https://example.com/view isn't a ui:// URI, which MCP Apps require"]); // None sent (a hosted server that lists without one, then wants one to read). const header = apps(await snap([], 'FIXTURE=apps', 'UI_READ=header')).find((f) => f.severity === 'info'); assert.match(header.message, /\(credentials: (?:MCP error -32000: )?missing authorization header\)/); // No answer in time, and the reads' time running out. const slow = apps(await snap(['--timeout', '500'], 'FIXTURE=apps', 'UI_READ=slow')).find((f) => f.severity === 'info'); assert.match(slow.message, /couldn't read here \((?:no answer in time|the reads ran out of time): /); assert.equal(slow.fix, 'Rerun where the server can answer (the network, the backend it reads from, more --timeout) to check the UI.'); }); test('spec/server-card: a card at /server-card that contradicts the live server; a matching one and no card are quiet', async () => { const card = (over) => ({ $schema: 'https://static.modelcontextprotocol.io/schemas/v1/server-card.schema.json', name: 'io.example/raw', description: 'A test server.', version: '0.0.1', ...over }); const cases = [ { card: card({ version: '9.9.9', remotes: [{ type: 'streamable-http', url: 'PLACEHOLDER', supportedProtocolVersions: ['2026-07-28'] }] }) }, { card: card({}), cardType: 'application/json' }, { card: { version: 3 } }, { card: card({}) }, {}, // A catch-all JSON answer, not a card: no $schema, not served as one. { card: { jsonrpc: '2.0', error: { code: -32601, message: 'Not found' } }, cardType: 'application/json' }, ]; const got = []; for (const c of cases) { const server = await startRaw(c); try { for (const r of c.card?.remotes ?? []) r.url = server.url; const r = await run(['snapshot', '--no-write', '--json', server.url]); got.push(JSON.parse(r.stdout).findings.filter((f) => f.rule === 'spec/server-card')); } finally { await server.close(); } } const [contradicts, wrongType, invalid, matching, none, catchAll] = got; assert.deepEqual(catchAll, []); assert.equal(contradicts.length, 1); assert.equal(contradicts[0].severity, 'warn'); assert.match(contradicts[0].message, /^The Server Card at http:\/\/127\.0\.0\.1:\d+\/mcp\/server-card contradicts the live server \(version: the card says 9\.9\.9, the server 0\.0\.1; protocol: the card lists 2026-07-28 for http:\/\/127\.0\.0\.1:\d+\/mcp, the server negotiated 2025-11-25\)\./); assert.match(contradicts[0].fix, /^Build the card from the same values the server reports/); assert.deepEqual(wrongType.map((f) => [f.severity, f.message.replace(/\d+/g, 'N')]), [['info', 'The Server Card at http://N.N.N.N:N/mcp/server-card is served as application/json, not application/mcp-server-card+json, which ext-server-card says servers SHOULD use.']]); assert.equal(invalid.length, 1); assert.match(invalid[0].message, /doesn't match the Server Card schema/); assert.ok(invalid[0].detail.length > 1); assert.deepEqual(matching, []); assert.deepEqual(none, []); }); // From Supabase's hosted server: tools listed only to a client that can't be asked. test('menu/capability-dependent: a tool listed only without elicitation names the capability it goes away with', async () => { const r = JSON.parse((await run(['snapshot', '--json', '--no-write', '--env', 'FIXTURE=cost', '--', process.execPath, join(FIXTURES, 'raw-server.mjs')])).stdout); const [f] = r.findings.filter((x) => x.rule === 'menu/capability-dependent'); assert.equal(f.tool, 'confirm_cost'); assert.equal(f.message, "confirm_cost is listed only to a client that doesn't support elicitation. A client without elicitation gets 2 tools, not 1. The menu file holds the menu of a client with both, which doesn't have it, so diff doesn't see changes to it."); }); // 2026-11-05: an option another command takes was ignored without a word (scan --no-auth). test('cli: an option for another command is ignored with a warning that names it', async () => { const dir = tempDir(); const menu = join(dir, 'm.json'); writeFileSync(menu, JSON.stringify({ toolmenu: 1, server: { name: 's', version: '1' }, tools: [] })); const r = await run(['diff', '--header', 'A: b', '--auto', menu, menu]); assert.equal(r.code, 0, r.stderr); assert.match(r.stderr, /^toolmenu: diff ignores --timeout, --header, --auto: they are for other commands\.\n → Next: Leave them out to silence this; toolmenu diff --help lists the options diff takes\.\n/); // The helper passes --timeout to every command; snapshot takes it, so only --release is named. const one = await run(['snapshot', '--release', '1.0.0', '--no-write', '--', process.execPath, join(FIXTURES, 'client-server.mjs')]); assert.match(one.stderr, /^toolmenu: snapshot ignores --release: it is for other commands\./); }); test('a toolmenu.config.json that isn\'t JSON: where it broke, never what it holds', async () => { const dir = tempDir(); writeFileSync(join(dir, 'toolmenu.config.json'), '//registry.npmjs.org/:_authToken=npm_S3CRET\n'); const r = await run(['snapshot', '--no-write', '--', process.execPath, '-e', '0'], { cwd: dir }); assert.equal(r.code, 2); assert.match(r.stderr, /toolmenu\.config\.json isn't valid JSON/); assert.doesNotMatch(r.stderr, /S3CRET|registry/); }); test('cli: a server/discover refused with 401 falls back to the 2025 handshake, and a server that refuses both wants credentials', async () => { // Dreamwork 1.10.0: server/discover gets a 401, initialize serves a guest menu. const guest = await startRaw({ discoverStatus: 401 }); try { const r = await run(['snapshot', guest.url, '--processes', '1']); assert.equal(r.code, 0, r.stderr); assert.match(r.stdout, /protocol 2025-11-25 · 3 tools/); assert.doesNotMatch(r.stderr, /wants credentials/); assert.equal(guest.discovers(), 1); } finally { await guest.close(); } const forbidden = await startRaw({ discoverStatus: 403 }); try { const r = await run(['snapshot', forbidden.url, '--processes', '1']); assert.equal(r.code, 0, r.stderr); assert.match(r.stdout, /protocol 2025-11-25 · 3 tools/); } finally { await forbidden.close(); } // Asked for its own handshake, toolmenu doesn't swap it. const pinned = await startRaw({ discoverStatus: 401 }); try { const r = await run(['snapshot', pinned.url, '--processes', '1', '--protocol', 'modern']); assert.notEqual(r.code, 0); } finally { await pinned.close(); } // A 2026-only server behind a login: the 2025 retry fails for another reason, so the // 401 that explains it is what's reported. const modernOnly = await startRaw({ discoverStatus: 401, initializeStatus: 400 }); try { const r = await run(['snapshot', modernOnly.url, '--processes', '1']); assert.equal(r.code, 2, r.stdout); assert.match(r.stderr, /wants credentials: server\/discover got HTTP 401 Unauthorized\./); } finally { await modernOnly.close(); } const walled = await startRaw({ requireAuth: true }); try { const r = await run(['snapshot', walled.url, '--processes', '1']); assert.equal(r.code, 2, r.stdout); assert.match(r.stderr, /wants credentials: initialize got HTTP 401 Unauthorized\./); assert.match(r.stderr, /→ Next: If the server uses OAuth, log in once with: toolmenu auth login /); assert.equal(walled.discovers(), 1); } finally { await walled.close(); } }); test('instructions/unknown-name: the instructions name tools and a resource the server does not list', async () => { const missing = ['--env', 'FIXTURE=instructions', '--env', 'INSTRUCTIONS=missing', '--', process.execPath, join(FIXTURES, 'raw-server.mjs')]; const r = await run(['snapshot', '--no-write', '--json', '--lists', 'resources,prompts', ...missing]); const found = JSON.parse(r.stdout).findings.filter((f) => f.rule === 'instructions/unknown-name'); assert.equal(found.length, 1, r.stderr); assert.equal(found[0].severity, 'warn'); assert.equal(found[0].confidence, 'unsure'); // "token usage" in the sentence is no reason the tool is missing, so submit_feedback still counts assert.match(found[0].message, /^The server's instructions name 2 tools the menu doesn't list \(submit_feedback, send_digest\), and 1 resource no listed resource or template matches \(acme:\/\/changelog\)\./); assert.match(found[0].message, /listed without credentials/); assert.doesNotMatch(found[0].message, /weren't listed/, 'prompts were listed'); assert.deepEqual(found[0].detail.map((d) => d.split(':')[0]), ['submit_feedback', 'send_digest', 'acme']); // the quote is the mention that counted: not "mode=send_digest", but "Call send_digest weekly." assert.match(found[0].detail[1], /^send_digest: “Call send_digest weekly\.”$/); assert.match(found[0].fix, /^Name only what the server lists/); assert.doesNotMatch(found[0].message, /get_story|sort_order/); // without the resource lists, only the tools are checked; prompts declared and not read are said const toolsOnly = JSON.parse((await run(['snapshot', '--no-write', '--json', ...missing])).stdout).findings.find((f) => f.rule === 'instructions/unknown-name'); assert.match(toolsOnly.message, /name 2 tools the menu doesn't list \(submit_feedback, send_digest\)\. /); assert.match(toolsOnly.message, /The server declares prompts and they weren't listed, so one of these may be a prompt: list them with --lists prompts\./); const text = await run(['snapshot', '--no-write', '--lists', 'resources,prompts', ...missing]); assert.match(text.stdout, /WARN instructions\/unknown-name · unsure/); assert.match(text.stdout, /→ Next: Name only what the server lists/); }); test('instructions/unknown-name: quiet on what is listed, templates with operators, bold URIs, placeholders, fields, values, subcommands, explained names, listed prompts, or no instructions', async () => { const fine = ['--env', 'FIXTURE=instructions', '--', process.execPath, join(FIXTURES, 'raw-server.mjs')]; const r = await run(['snapshot', '--no-write', '--json', '--lists', 'resources,prompts', ...fine]); assert.deepEqual(JSON.parse(r.stdout).findings.filter((f) => f.rule === 'instructions/unknown-name'), [], r.stderr); const none = await run(['snapshot', '--no-write', '--json', ...raw('smells')]); assert.deepEqual(JSON.parse(none.stdout).findings.filter((f) => f.rule === 'instructions/unknown-name'), []); }); test('spec/discover-refused: server/discover refused while the 2025 handshake serves the menu, checked against the SDK client', async () => { const { Client, StreamableHTTPClientTransport } = await import('@modelcontextprotocol/client'); for (const status of [401, 403]) { const server = await startRaw({ discoverStatus: status }); try { // the official SDK client stops there when it tries 2026-07-28 first (versionNegotiation auto); // on its default (legacy) it never sends server/discover and connects const auto = new Client({ name: 'probe', version: '0' }, { versionNegotiation: { mode: 'auto' } }); await assert.rejects(auto.connect(new StreamableHTTPClientTransport(new URL(server.url))), new RegExp(`Version negotiation failed: the server (requires authorization|denied access) \\(HTTP ${status}\\)`)); const legacy = new Client({ name: 'probe', version: '0' }); await legacy.connect(new StreamableHTTPClientTransport(new URL(server.url))); await legacy.close(); const r = await run(['snapshot', '--no-write', '--json', server.url, '--processes', '1']); assert.equal(r.code, 0, r.stderr); const findings = JSON.parse(r.stdout).findings; const refused = findings.find((f) => f.rule === 'spec/discover-refused'); assert.equal(refused.severity, 'warn'); assert.match(refused.message, new RegExp(`^server/discover got HTTP ${status} while the 2025 handshake \\(initialize\\) served the menu without credentials\\.`)); assert.match(refused.message, /official TypeScript SDK client \(2\.4\) with versionNegotiation auto fails/); assert.match(refused.message, /Its default \(legacy\) never asks/); assert.match(refused.fix, /^Answer server\/discover as initialize is answered: without credentials/); // said once: not spec/discover's 2025 note as well assert.equal(findings.some((f) => f.rule === 'spec/discover'), false); } finally { await server.close(); } } // a 2025 server that answers server/discover as 2025 servers do, or a run on 2025 alone: quiet const plain = await startRaw(); try { for (const args of [[], ['--protocol', 'legacy']]) { const findings = JSON.parse((await run(['snapshot', '--no-write', '--json', plain.url, '--processes', '1', ...args])).stdout).findings; assert.equal(findings.some((f) => f.rule === 'spec/discover-refused'), false); } } finally { await plain.close(); } });