#!/usr/bin/env node /** * Generate and validate a disposable OpenAI-compatible capture provider. * * The generated Worker stores assertion definitions and pass/fail results in a * disposable D1 database. It never stores request bodies, credentials, or raw * prompts. Operators provide CAPTURE_API_KEY and CAPTURE_ADMIN_KEY as Wrangler * secrets after generation; neither secret is accepted on this CLI's argv. */ import { mkdirSync, writeFileSync } from 'node:fs'; import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; export const CAPTURE_PROVIDER_SCHEMA = 'chickpea-live-capture-provider/v1'; export const CAPTURE_MAX_BODY_BYTES = 262_144; export const CAPTURE_MAX_ASSERTIONS = 64; export function validateCaptureConfiguration(input) { if (!input || typeof input !== 'object' || Array.isArray(input)) { throw new Error('Capture configuration must be an object.'); } const runMarker = exactToken(input.runMarker, 'run marker', 128); const responseText = boundedText(input.responseText, 'response text', 2_000); if (!Array.isArray(input.assertions) || input.assertions.length < 1 || input.assertions.length > CAPTURE_MAX_ASSERTIONS) { throw new Error(`Capture configuration needs 1-${CAPTURE_MAX_ASSERTIONS} assertions.`); } const seen = new Set(); const assertions = input.assertions.map((item) => { if (!item || typeof item !== 'object' || Array.isArray(item)) throw new Error('Capture assertion must be an object.'); const id = exactToken(item.id, 'assertion ID', 100); if (seen.has(id)) throw new Error(`Duplicate assertion ID: ${id}`); seen.add(id); const model = exactToken(item.model, 'model ID', 200); if (!model.startsWith(`${runMarker}:`)) { throw new Error(`Assertion ${id} model must carry the exact run marker.`); } const needle = boundedText(item.needle, 'assertion needle', 8_000); if (item.expect !== 'present' && item.expect !== 'absent') { throw new Error(`Assertion ${id} expect must be present or absent.`); } return { id, model, needle, expect: item.expect }; }); return { schema: CAPTURE_PROVIDER_SCHEMA, runMarker, responseText, assertions }; } export function evaluateCaptureRequest(bodyText, configuration, requestedModel) { const config = validateCaptureConfiguration(configuration); if (typeof bodyText !== 'string' || Buffer.byteLength(bodyText, 'utf8') > CAPTURE_MAX_BODY_BYTES) { throw new Error(`Capture request body exceeds ${CAPTURE_MAX_BODY_BYTES} bytes.`); } const model = exactToken(requestedModel, 'requested model', 200); return config.assertions .filter((assertion) => assertion.model === model) .map((assertion) => { const found = bodyText.includes(assertion.needle); return { id: assertion.id, model, expect: assertion.expect, passed: assertion.expect === 'present' ? found : !found, }; }); } export function openAiCaptureStream(responseText, model = 'capture-model') { const text = boundedText(responseText, 'response text', 2_000); const modelId = exactToken(model, 'model ID', 200); const base = { id: 'chatcmpl-chickpea-capture', object: 'chat.completion.chunk', created: 0, model: modelId }; const chunks = [ { ...base, choices: [{ index: 0, delta: { role: 'assistant', content: text }, finish_reason: null }] }, { ...base, choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] }, { ...base, choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } }, ]; return `${chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join('')}data: [DONE]\n\n`; } export function renderCaptureProviderWorker() { return `// Generated by scripts/live-test-capture-provider.mjs. Disposable acceptance resource only. const SCHEMA = ${JSON.stringify(CAPTURE_PROVIDER_SCHEMA)}; const MAX_BODY_BYTES = ${CAPTURE_MAX_BODY_BYTES}; const MAX_ASSERTIONS = ${CAPTURE_MAX_ASSERTIONS}; export default { async fetch(request, env) { await initialize(env.CAPTURE_DB); const url = new URL(request.url); if (url.pathname === '/__capture/configure' && request.method === 'POST') { if (!authorized(request, env.CAPTURE_ADMIN_KEY)) return json({ ok: false }, 401); const input = await boundedJson(request); const config = validateConfiguration(input); await env.CAPTURE_DB.batch([ env.CAPTURE_DB.prepare('DELETE FROM capture_assertions'), env.CAPTURE_DB.prepare('DELETE FROM capture_results'), env.CAPTURE_DB.prepare('DELETE FROM capture_meta'), env.CAPTURE_DB.prepare('INSERT INTO capture_meta (key, value) VALUES (?, ?)').bind('run_marker', config.runMarker), env.CAPTURE_DB.prepare('INSERT INTO capture_meta (key, value) VALUES (?, ?)').bind('response_text', config.responseText), ...config.assertions.map((item) => env.CAPTURE_DB.prepare( 'INSERT INTO capture_assertions (id, model, needle, expectation) VALUES (?, ?, ?, ?)' ).bind(item.id, item.model, item.needle, item.expect)), ]); return json({ ok: true, schema: SCHEMA, assertionCount: config.assertions.length }); } if (url.pathname === '/__capture/results' && request.method === 'GET') { if (!authorized(request, env.CAPTURE_ADMIN_KEY)) return json({ ok: false }, 401); const meta = await env.CAPTURE_DB.prepare('SELECT key, value FROM capture_meta').all(); const results = await env.CAPTURE_DB.prepare( 'SELECT assertion_id AS id, model, expectation AS expect, passed FROM capture_results ORDER BY request_id, assertion_id' ).all(); const map = Object.fromEntries((meta.results || []).map((row) => [row.key, row.value])); return json({ ok: true, schema: SCHEMA, runMarker: map.run_marker || null, results: (results.results || []).map((row) => ({ ...row, passed: row.passed === 1 })), }); } if (url.pathname === '/v1/chat/completions' && request.method === 'POST') { if (!authorized(request, env.CAPTURE_API_KEY)) return json({ error: { type: 'invalid_request_error' } }, 401); const bodyText = await boundedText(request); let body; try { body = JSON.parse(bodyText); } catch { return json({ error: { type: 'invalid_request_error' } }, 400); } const model = exactToken(body && body.model, 'requested model', 200); const rows = await env.CAPTURE_DB.prepare( 'SELECT id, needle, expectation FROM capture_assertions WHERE model = ? ORDER BY id' ).bind(model).all(); if (!rows.results || rows.results.length === 0) return json({ error: { type: 'unconfigured_model' } }, 409); const requestId = crypto.randomUUID(); await env.CAPTURE_DB.batch(rows.results.map((row) => { const found = bodyText.includes(row.needle); const passed = row.expectation === 'present' ? found : !found; return env.CAPTURE_DB.prepare( 'INSERT INTO capture_results (request_id, assertion_id, model, expectation, passed, created_at) VALUES (?, ?, ?, ?, ?, ?)' ).bind(requestId, row.id, model, row.expectation, passed ? 1 : 0, Date.now()); })); const response = await env.CAPTURE_DB.prepare('SELECT value FROM capture_meta WHERE key = ?').bind('response_text').first(); return new Response(openAiStream(response && response.value || 'Capture accepted.', model), { headers: { 'content-type': 'text/event-stream', 'cache-control': 'no-store' }, }); } return json({ ok: false }, 404); }, }; async function initialize(db) { await db.exec(\` CREATE TABLE IF NOT EXISTS capture_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); CREATE TABLE IF NOT EXISTS capture_assertions ( id TEXT PRIMARY KEY, model TEXT NOT NULL, needle TEXT NOT NULL, expectation TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS capture_results ( request_id TEXT NOT NULL, assertion_id TEXT NOT NULL, model TEXT NOT NULL, expectation TEXT NOT NULL, passed INTEGER NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY (request_id, assertion_id) ); \`); } function authorized(request, expected) { return typeof expected === 'string' && expected.length >= 24 && request.headers.get('authorization') === 'Bearer ' + expected; } async function boundedJson(request) { const text = await boundedText(request); try { return JSON.parse(text); } catch { throw new Error('Invalid JSON.'); } } async function boundedText(request) { const declared = Number(request.headers.get('content-length') || 0); if (declared > MAX_BODY_BYTES) throw new Error('Request too large.'); const text = await request.text(); if (new TextEncoder().encode(text).byteLength > MAX_BODY_BYTES) throw new Error('Request too large.'); return text; } function validateConfiguration(input) { if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Invalid configuration.'); const runMarker = exactToken(input.runMarker, 'run marker', 128); const responseText = boundedValue(input.responseText, 'response text', 2000); if (!Array.isArray(input.assertions) || input.assertions.length < 1 || input.assertions.length > MAX_ASSERTIONS) { throw new Error('Invalid assertions.'); } const seen = new Set(); const assertions = input.assertions.map((item) => { const id = exactToken(item && item.id, 'assertion ID', 100); if (seen.has(id)) throw new Error('Duplicate assertion.'); seen.add(id); const model = exactToken(item.model, 'model ID', 200); if (!model.startsWith(runMarker + ':')) throw new Error('Model is outside this run.'); const needle = boundedValue(item.needle, 'needle', 8000); if (item.expect !== 'present' && item.expect !== 'absent') throw new Error('Invalid expectation.'); return { id, model, needle, expect: item.expect }; }); return { runMarker, responseText, assertions }; } function exactToken(value, label, maximum) { if (typeof value !== 'string' || value.length < 1 || value.length > maximum || !/^[A-Za-z0-9][A-Za-z0-9._:-]*$/.test(value)) { throw new Error('Invalid ' + label + '.'); } return value; } function boundedValue(value, label, maximum) { if (typeof value !== 'string' || value.length < 1 || value.length > maximum || /[\\u0000]/.test(value)) { throw new Error('Invalid ' + label + '.'); } return value; } function openAiStream(text, model) { const base = { id: 'chatcmpl-chickpea-capture', object: 'chat.completion.chunk', created: 0, model }; const chunks = [ { ...base, choices: [{ index: 0, delta: { role: 'assistant', content: text }, finish_reason: null }] }, { ...base, choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] }, { ...base, choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } }, ]; return chunks.map((chunk) => 'data: ' + JSON.stringify(chunk) + '\\n\\n').join('') + 'data: [DONE]\\n\\n'; } function json(value, status = 200) { return Response.json(value, { status, headers: { 'cache-control': 'no-store' } }); } `; } export function renderCaptureWranglerConfig({ workerName, databaseName, databaseId }) { const name = exactToken(workerName, 'Worker name', 63); const dbName = exactToken(databaseName, 'D1 name', 63); const dbId = exactToken(databaseId, 'D1 ID', 64); return `${JSON.stringify({ name, main: './capture-worker.mjs', compatibility_date: '2026-08-12', workers_dev: true, d1_databases: [{ binding: 'CAPTURE_DB', database_name: dbName, database_id: dbId }], }, null, 2)}\n`; } export function writeCaptureProviderBundle(directoryInput, options) { const directory = resolve(directoryInput); mkdirSync(directory, { recursive: true, mode: 0o700 }); writeFileSync(resolve(directory, 'capture-worker.mjs'), renderCaptureProviderWorker(), { encoding: 'utf8', mode: 0o600 }); writeFileSync(resolve(directory, 'wrangler.json'), renderCaptureWranglerConfig(options), { encoding: 'utf8', mode: 0o600 }); return { directory, workerSource: resolve(directory, 'capture-worker.mjs'), wranglerConfig: resolve(directory, 'wrangler.json') }; } function exactToken(value, label, maximum) { if (typeof value !== 'string' || value.length < 1 || value.length > maximum || !/^[A-Za-z0-9][A-Za-z0-9._:-]*$/.test(value)) { throw new Error(`Invalid ${label}.`); } return value; } function boundedText(value, label, maximum) { if (typeof value !== 'string' || value.length < 1 || value.length > maximum || value.includes('\0')) { throw new Error(`Invalid ${label}.`); } return value; } function arg(args, flag) { const index = args.indexOf(flag); return index >= 0 ? args[index + 1] : undefined; } async function cli() { const args = process.argv.slice(2); if (args[0] !== 'prepare') { throw new Error('Usage: live-test-capture-provider.mjs prepare --directory --worker-name --database-name --database-id '); } const directory = arg(args, '--directory'); const workerName = arg(args, '--worker-name'); const databaseName = arg(args, '--database-name'); const databaseId = arg(args, '--database-id'); if (!directory || !workerName || !databaseName || !databaseId) throw new Error('All prepare arguments are required.'); console.log(JSON.stringify(writeCaptureProviderBundle(directory, { workerName, databaseName, databaseId }))); } if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { cli().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; }); }