#!/usr/bin/env bun // A new app, as a coding agent arriving cold meets it: the command `.mcp.json` names answers an MCP // `initialize` and a non-empty `tools/list`, and the app's own `POST /mcp` is mounted under `x dev` // — 401 with no token, `health` listed and callable with the development token, and a boot log // that says `app mcp mounted` rather than X_MCP_APP_UNMOUNTED — and `x docs` answers a task-phrased // question from the wiki a publish packs (`scaffold-smoke-overrides.ts`). Run by `scaffold-smoke`. // // bun run scripts/scaffold-agent.ts [--json] import { DEV_BINDING, quoteArg } from '@ultimat3/cli'; import { renderThrowable } from '@ultimat3/core'; import { isPortClash } from '../packages/testing/src/free-port-bind'; import { parseScriptArgs } from './lib/args'; import type { Finding, ScriptResult } from './lib/log'; import { report } from './lib/log'; import { repoRoot } from './lib/run'; import { ScriptError } from './lib/script-error'; import { strangerEnvAt } from './lib/stranger-env'; import { appDirWord, type DevServer, freeDevPorts, type PortBinder, portTaken, } from './scaffold-admin'; import { appBin } from './scaffold-first-run'; const SCRIPT = 'scaffold-agent'; const MCP_JSON = '.mcp.json'; /** What the scaffold's `apps/web/app/auth/agent-token.ts` honours under `x dev` alone. */ export const DEV_AGENT_TOKEN = 'dev-agent.admin'; /** The action every scaffold exposes over MCP (`apps/web/api/health.ts`). */ export const PROBE_TOOL = 'health'; const BOOT_BUDGET_MS = 60_000; /** The dev server's whole life: boot, then a handful of requests. A check that outlives it hung. */ const SERVE_BUDGET_MS = 3 * BOOT_BUDGET_MS; /** A question as an agent phrases it, never as a page is titled (plan 101 19b, Y9). */ export const DOCS_QUERY = 'paginate a query with a cursor'; /** The one server `.mcp.json` names, as an argv — or `undefined` when the file says none. */ export function mcpCommand(config: unknown): readonly string[] | undefined { const servers = (config as { readonly mcpServers?: unknown } | null)?.mcpServers; if (typeof servers !== 'object' || servers === null) return undefined; const [first] = Object.values(servers as Record); const { command, args } = (first ?? {}) as { command?: unknown; args?: unknown }; if (typeof command !== 'string' || !Array.isArray(args)) return undefined; return args.every((arg) => typeof arg === 'string') ? [command, ...args] : undefined; } /** A JSON-RPC request, numbered by the caller so answers are matched by `id`. */ export const rpc = (id: number, method: string, params: unknown = {}) => ({ jsonrpc: '2.0', id, method, params, }); export const INITIALIZE = rpc(1, 'initialize', { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: SCRIPT, version: '0' }, }); /** The tool names a `tools/list` answer carries; `[]` for anything that is not one. */ export function toolNames(answer: unknown): readonly string[] { const tools = (answer as { readonly result?: { readonly tools?: unknown } } | null)?.result ?.tools; if (!Array.isArray(tools)) return []; return tools.flatMap((tool: unknown) => { const name = (tool as { readonly name?: unknown } | null)?.name; return typeof name === 'string' ? [name] : []; }); } export interface AgentCheckIo { /** Spawn `argv` in `dir`, write each message as one NDJSON line, answer the replies by `id`. */ readonly stdio: ( dir: string, argv: readonly string[], messages: readonly unknown[], ) => Promise; readonly boot: (dir: string, port: number) => DevServer; readonly fetcher: (url: string, init: RequestInit) => Promise; readonly ports: PortBinder; readonly now: () => number; readonly sleep: (ms: number) => Promise; /** `x docs --json` in `dir`: its parsed stdout, or `undefined` when it printed none. */ readonly docs: (dir: string, query: string) => Promise; } const finding = (dir: string, cause: string, fix: string): Finding => ({ code: 'X_SCAFFOLD_FIRST_RUN_FAILED', cause, fix, at: dir, }); /** The `.mcp.json` half: the file's own command, a handshake, a catalog. */ async function checkStdio(dir: string, io: AgentCheckIo): Promise { const file = Bun.file(`${dir}/${MCP_JSON}`); const argv = (await file.exists()) ? mcpCommand(await file.json()) : undefined; const where = quoteArg(appDirWord(dir)); if (argv === undefined) { return [ finding(dir, `${dir}/${MCP_JSON} names no server command`, `cd ${where} && cat .mcp.json`), ]; } const answers = await io.stdio(dir, argv, [INITIALIZE, rpc(2, 'tools/list')]); const initialized = answers.some((answer) => (answer as { id?: unknown } | null)?.id === 1); const tools = toolNames(answers.find((answer) => (answer as { id?: unknown } | null)?.id === 2)); if (initialized && tools.length > 0) return []; return [ finding( dir, `${argv.join(' ')} (from ${MCP_JSON}) answered ${initialized ? '' : 'no initialize and '}${String(tools.length)} tool(s)`, `bun run scripts/scaffold-agent.ts ${where} --json`, ), ]; } const post = (body: unknown, token?: string): RequestInit => ({ method: 'POST', headers: { 'content-type': 'application/json', ...(token === undefined ? {} : { authorization: `Bearer ${token}` }), }, body: JSON.stringify(body), }); /** The `POST /mcp` half, over a running `x dev`: refused, then listed, then called. */ async function askEndpoint(url: string, io: AgentCheckIo): Promise { const problems: string[] = []; const anonymous = await io.fetcher(url, post(rpc(1, 'tools/list'))); if (anonymous.status !== 401) problems.push(`POST /mcp with no token answered ${String(anonymous.status)}, not 401`); // A page that rebinds its name to 127.0.0.1 sends an Origin equal to the endpoint's own, and core's // proveSameOrigin admits it: the route's 403 is for an Origin that is neither this app nor listed // in http.cors.origins. What refuses the rebound page is the resolver — the development token is // honoured only with no Origin at all — so the probe sends exactly that pair and expects its 401. const fromPage = await io.fetcher(url, { ...post(rpc(4, 'tools/list'), DEV_AGENT_TOKEN), headers: { 'content-type': 'application/json', authorization: `Bearer ${DEV_AGENT_TOKEN}`, origin: new URL(url).origin, }, }); if (fromPage.status !== 401) problems.push( `the development token with an Origin answered ${String(fromPage.status)}, not 401`, ); const listed = toolNames( await (await io.fetcher(url, post(rpc(2, 'tools/list'), DEV_AGENT_TOKEN))).json(), ); if (!listed.includes(PROBE_TOOL)) problems.push( `tools/list with the development token lists [${listed.join(', ')}], no ${PROBE_TOOL}`, ); const called: unknown = await ( await io.fetcher( url, post(rpc(3, 'tools/call', { name: PROBE_TOOL, arguments: {} }), DEV_AGENT_TOKEN), ) ).json(); const result = (called as { readonly result?: { readonly isError?: unknown } } | null)?.result; if (result === undefined || result.isError === true) problems.push(`tools/call ${PROBE_TOOL} answered ${JSON.stringify(called).slice(0, 200)}`); return problems; } /** One boot on `port`: boot `x dev`, ask the endpoint, stop it — always. */ async function bootAndAsk(dir: string, io: AgentCheckIo, port: number) { const url = `http://${DEV_BINDING.hostname}:${String(port)}/mcp`; const fix = `cd ${quoteArg(appDirWord(dir))} && bun run dev --port ${quoteArg(String(port))}`; const started = io.now(); const server = io.boot(dir, port); let up = false; let lastDial = 'no dial was made'; let problems: readonly string[] = []; let output = ''; try { while (!up && !server.exited() && io.now() - started < BOOT_BUDGET_MS) { up = await io.fetcher(url, post(rpc(0, 'ping'))).then( () => true, (error: unknown) => { lastDial = renderThrowable(error); return false; }, ); if (!up) await io.sleep(100); } if (up) problems = await askEndpoint(url, io); } finally { output = await server.stop(); } return { url, fix, up, lastDial, problems, output }; } /** * Boot `x dev`, ask the endpoint, stop it — always — and judge what it printed. A boot that died * on a port another process held is booted again on a fresh pair (`io.ports`). */ async function checkEndpoint(dir: string, io: AgentCheckIo): Promise { // Read off the boot's WHOLE output: the refusal's cause keeps only its tail. let clashed = false; const run = await io .ports( async (port) => { const booted = await bootAndAsk(dir, io, port); clashed = !booted.up && isPortClash(booted.output); if (clashed) throw portTaken(port, booted.output, booted.fix); return booted; }, () => clashed, ) .catch((error: unknown) => { if (error instanceof ScriptError) return error; throw error; }); if (run instanceof ScriptError) return [finding(dir, run.cause, run.fix)]; const { url, fix, up, lastDial, problems, output } = run; if (!up) return [finding(dir, `x dev answered nothing at ${url}: ${lastDial}`, fix)]; const log: string[] = []; if (output.includes('X_MCP_APP_UNMOUNTED')) log.push('the boot log carries X_MCP_APP_UNMOUNTED'); if (!output.includes('app mcp mounted')) log.push('the boot log never says app mcp mounted'); return [...problems, ...log].map((problem) => finding(dir, problem, fix)); } /** `x docs` over an installed app: an answer with at least one page, not `ships no wiki/`. */ async function checkDocs(dir: string, io: AgentCheckIo): Promise { const answer = (await io.docs(dir, DOCS_QUERY)) as { readonly ok?: unknown; readonly data?: { readonly matches?: unknown }; } | null; const matches = answer?.data?.matches; if (answer?.ok === true && Array.isArray(matches) && matches.length > 0) return []; return [ finding( dir, `x docs ${quoteArg(DOCS_QUERY)} answered ${JSON.stringify(answer ?? null).slice(0, 200)}`, `cd ${quoteArg(appDirWord(dir))} && bunx x docs ${quoteArg(DOCS_QUERY)} --json`, ), ]; } export async function checkAgent(dir: string, io: AgentCheckIo): Promise { if (!(await Bun.file(appBin(dir)).exists())) { const where = quoteArg(appDirWord(dir)); const findings = [ finding( dir, `${appBin(dir)} does not exist, so the app cannot be booted`, `cd ${where} && bun run setup`, ), ]; return { ok: false, script: SCRIPT, summary: `no x binary in ${dir}`, findings }; } const findings = [ ...(await checkStdio(dir, io)), ...(await checkEndpoint(dir, io)), ...(await checkDocs(dir, io)), ]; return { ok: findings.length === 0, script: SCRIPT, summary: findings.length === 0 ? `${dir}: ${MCP_JSON} answers initialize and tools/list; POST /mcp refuses no token and serves ${PROBE_TOOL} to the development agent; x docs answers` : `${String(findings.length)} agent-facing finding(s) in ${dir}`, findings, }; } /** Every reply line that carries an `id`, until each asked `id` has one or the budget is spent. */ async function stdioAnswers(dir: string, argv: readonly string[], messages: readonly unknown[]) { const child = Bun.spawn([...argv], { cwd: dir, // A whole env: `Bun.spawn` replaces, and `.mcp.json`'s command is the app's, not this repo's. env: strangerEnvAt(repoRoot()), stdin: 'pipe', stdout: 'pipe', stderr: 'ignore', timeout: BOOT_BUDGET_MS, }); child.stdin.write(`${messages.map((message) => JSON.stringify(message)).join('\n')}\n`); await child.stdin.flush(); const answers: unknown[] = []; const reader = child.stdout.getReader(); const decoder = new TextDecoder(); let buffered = ''; const deadline = Bun.sleep(BOOT_BUDGET_MS).then(() => 'timeout' as const); try { while (answers.length < messages.length) { const next = await Promise.race([reader.read(), deadline]); if (next === 'timeout' || next.done) break; buffered += decoder.decode(next.value, { stream: true }); const lines = buffered.split('\n'); buffered = lines.pop() ?? ''; for (const line of lines.filter((one) => one.trim().startsWith('{'))) { const parsed: unknown = JSON.parse(line); if ((parsed as { id?: unknown } | null)?.id !== undefined) answers.push(parsed); } } } finally { child.kill(); await child.exited; } return answers; } /** The real thing: the file's own command, the app's own `x dev`, a free port, the wall clock. */ const realIo: AgentCheckIo = { stdio: stdioAnswers, boot: (dir, port) => { const server = Bun.spawn([appBin(dir), 'dev', '--port', String(port), '--json'], { cwd: dir, env: strangerEnvAt(repoRoot()), stdout: 'pipe', stderr: 'pipe', timeout: SERVE_BUDGET_MS, }); const printed = Promise.all([ new Response(server.stdout).text(), new Response(server.stderr).text(), ]); return { exited: () => server.exitCode !== null, stop: async () => { server.kill('SIGTERM'); await server.exited; return (await printed).join('\n'); }, }; }, fetcher: fetch, // A free port PAIR `x dev` binds itself — the rule `scaffold-admin.ts` keeps. ports: freeDevPorts, now: () => performance.now(), sleep: (ms) => Bun.sleep(ms), docs: async (dir, query) => { const child = Bun.spawn([appBin(dir), 'docs', query, '--json'], { cwd: dir, env: strangerEnvAt(repoRoot()), stdout: 'pipe', stderr: 'ignore', timeout: BOOT_BUDGET_MS, }); const printed = (await new Response(child.stdout).text()).trim(); await child.exited; try { return printed === '' ? undefined : (JSON.parse(printed) as unknown); } catch { return printed; } }, }; if (import.meta.main) { const args = parseScriptArgs(Bun.argv.slice(2)); const dir = args.positionals[0]; report( dir === undefined ? { ok: false, script: SCRIPT, summary: 'a scaffolded app directory is required', findings: [ finding( 'scripts/scaffold-agent.ts', 'no app directory given, so there is no scaffolded app to ask', 'bun run scripts/scaffold-agent.ts /tmp/demoapp --json', ), ], } : await checkAgent(dir, realIo), args.json, ); }