// @ts-check // One verification run of AskDB: a disposable project with its own config, the Quickstart's // shop.db, copies of the schema fixtures and a fake OpenAI endpoint, driven through the built // CLI, HTTP API and Studio. Every drive writes an evidence file that cleanup keeps. // // The replay tests (e2e/cli, e2e/api, e2e/studio) and the agent's CLI (control-askdb.mjs) both // drive AskDB through this module, so a scripted journey and an agent-driven one start from the // same state. The playbook that explains it is .agents/skills/verify-askdb/SKILL.md. import { spawn, spawnSync } from "node:child_process"; import { createHash, randomBytes } from "node:crypto"; import { appendFileSync, cpSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, realpathSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; import { createServer } from "node:net"; import { tmpdir } from "node:os"; import { dirname, join, relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; const require = createRequire(import.meta.url); const e2eDir = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const repo = resolve(e2eDir, ".."); /** Package folders, through e2e's own dependencies (pnpm links each into e2e/node_modules). */ const pkgDir = (/** @type {string} */ name) => realpathSync(join(e2eDir, "node_modules", name)); export const CLI = join(pkgDir("askdb"), "dist/cli.js"); export const HTTP_API = join(pkgDir("@askdb/http-api"), "dist/bin.js"); const STUDIO_CLIENT = join(pkgDir("@askdb/studio"), "dist/client/index.html"); /** Where runs live: ASKDB_VERIFY_ROOT, or /askdb-verify. */ export function verifyRoot() { return process.env.ASKDB_VERIFY_ROOT ?? join(tmpdir(), "askdb-verify"); } /** The run `setup` last made from this checkout, so agent commands in separate shells find it. */ export function currentRunFile() { return join(verifyRoot(), `current-${createHash("sha256").update(repo).digest("hex").slice(0, 12)}`); } /** @typedef {{ pid: number, marker: string, log: string }} Proc */ /** @typedef {{ id: string, live: boolean, ports: { model: number, api: number, studio: number }, project: string, evidence: string }} RunState */ /** @typedef {{ command: string[], cwd: string, exitCode: number | null, stdout: string, stderr: string, evidence: string }} CliResult */ /** @typedef {{ status: number, json: any, text: string, evidence: string }} HttpResult */ async function freePort() { return new Promise((resolvePort, reject) => { const srv = createServer(); srv.once("error", reject); srv.listen(0, "127.0.0.1", () => { const address = srv.address(); const port = typeof address === "object" && address ? address.port : 0; srv.close(() => resolvePort(port)); }); }); } /** Command line of `pid`, or undefined when it isn't running. */ /** @param {number} pid */ function commandOf(pid) { const ps = spawnSync("ps", ["-p", String(pid), "-o", "command="], { encoding: "utf8" }); return ps.status === 0 ? ps.stdout.trim() : undefined; } /** Whether `proc` is still the process the run started: alive, with its command line naming `marker`. */ /** @param {Proc | undefined} proc */ export function isOurs(proc) { if (!proc?.pid) return false; const cmd = commandOf(proc.pid); return cmd !== undefined && cmd.includes(proc.marker); } /** * The workspace packages a run executes: e2e's dependencies and, through workspace links, theirs * (dependencies, optional and peer; devDependencies only for e2e itself). Derived from the * manifests, so a package the CLI or Studio comes to depend on is covered without an edit here. * @returns {string[]} absolute package folders */ export function workspaceClosure() { /** @type {Set} */ const seen = new Set(); const queue = [e2eDir]; while (queue.length > 0) { const dir = /** @type {string} */ (queue.shift()); const manifest = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")); const names = Object.keys({ ...manifest.dependencies, ...manifest.optionalDependencies, ...manifest.peerDependencies, ...(dir === e2eDir ? manifest.devDependencies : {}), }); for (const name of names) { const link = join(dir, "node_modules", name); if (!existsSync(link)) continue; const target = realpathSync(link); if (!target.startsWith(repo + sep) || target.includes(`${sep}node_modules${sep}`) || seen.has(target)) continue; seen.add(target); queue.push(target); } } return [...seen].sort(); } const sleep = (/** @type {number} */ ms) => new Promise((r) => setTimeout(r, ms)); export class AskdbRun { /** @param {string} id @param {RunState} state */ constructor(id, state) { this.id = id; this.state = state; this.dir = join(verifyRoot(), id); } get project() { return this.state.project; } get evidence() { return this.state.evidence; } /** @param {"api" | "studio" | "model"} kind */ url(kind) { return `http://127.0.0.1:${this.state.ports[kind]}`; } /** * A new run. Throws when a build is missing or the id is taken. * @param {{ id?: string, live?: boolean }} [options] */ static async create({ id, live = false } = {}) { for (const built of [CLI, HTTP_API, STUDIO_CLIENT]) { if (!existsSync(built)) throw new Error(`${built} is missing. Build first: pnpm exec turbo run build --filter=@askdb/e2e^...`); } const runId = id ?? `v${new Date().toISOString().replace(/[-:T]/g, "").slice(0, 14)}-${randomBytes(2).toString("hex")}`; const dir = join(verifyRoot(), runId); const project = join(dir, "project"); const evidence = join(dir, "evidence"); if (existsSync(project)) throw new Error(`run ${runId} already has a project at ${project}: pick a new id, or clean it up first.`); mkdirSync(project, { recursive: true }); mkdirSync(evidence, { recursive: true }); // Verification scaffolding: what an installed project resolves from its own node_modules. mkdirSync(join(project, "node_modules/@askdb"), { recursive: true }); symlinkSync(pkgDir("@askdb/config"), join(project, "node_modules/@askdb/config")); symlinkSync(pkgDir("better-sqlite3"), join(project, "node_modules/better-sqlite3")); // The Quickstart's shop database, and copies of the schema fixtures (Studio writes into its schema folder). const Database = require("better-sqlite3"); const db = new Database(join(project, "shop.db")); db.exec(readFileSync(join(repo, "examples/quickstart-shop/shop.sql"), "utf8")); db.close(); for (const fixture of ["orders-users-sensitive.schema", "agency-multi-tenant.schema"]) { cpSync(join(repo, "fixtures/schemas", fixture), join(project, "schemas", fixture), { recursive: true }); } const ports = { model: await freePort(), api: await freePort(), studio: await freePort() }; const ai = live ? `{ provider: "openai", providerConfig: { openai: { apiKey: env("OPENAI_API_KEY") } }, language: { model: env("OPENAI_MODEL") } }` : `{ provider: "openai", providerConfig: { openai: { apiKey: "sk-verify", baseUrl: "http://127.0.0.1:${ports.model}/v1" } }, language: { model: "gpt-4o-mini" } }`; writeFileSync( join(project, "askdb.config.ts"), `import { defineConfig, env } from "@askdb/config"; // Written by the verify-askdb harness (e2e/harness/askdb-run.mjs) for run ${runId}. export default defineConfig({ ai: ${ai}, introspection: { provider: "sqlite", providerConfig: { sqlite: { file: "./shop.db" } }, outputDir: "./askdb" }, rag: { embedder: "mock", store: "file", storeConfig: { file: {} } }, studio: { listen: { host: "127.0.0.1", port: ${ports.studio} }, execute: { enabled: true, provider: "sqlite", file: "./shop.db" } }, httpApi: { listen: { host: "127.0.0.1", port: ${ports.api} } }, }); `, ); if (live && existsSync(join(repo, ".env"))) cpSync(join(repo, ".env"), join(project, ".env")); const run = new AskdbRun(runId, { id: runId, live, ports, project, evidence }); run.save(); if (!live) run.setProcess("model", await run.startProcess("model", [join(e2eDir, "harness/fake-openai.mjs"), dir, String(ports.model)], "fake-openai listening")); return run; } /** @param {string} id */ static load(id) { const file = join(verifyRoot(), id, "state.json"); if (!existsSync(file)) throw new Error(`run ${id} has no state at ${file}: create it first.`); return new AskdbRun(id, JSON.parse(readFileSync(file, "utf8"))); } save() { writeFileSync(join(this.dir, "state.json"), JSON.stringify(this.state, null, 2) + "\n"); } /** * The processes this run started, read fresh from one `proc-.json` file each, so two * commands starting or stopping different kinds at once never drop each other's record. * @returns {Record} */ processes() { /** @type {Record} */ const out = {}; for (const f of existsSync(this.dir) ? readdirSync(this.dir) : []) { const kind = /^proc-(.+)\.json$/.exec(f)?.[1]; if (kind) out[kind] = JSON.parse(readFileSync(join(this.dir, f), "utf8")); } return out; } /** @param {string} kind @param {Proc} proc */ setProcess(kind, proc) { writeFileSync(join(this.dir, `proc-${kind}.json`), JSON.stringify(proc) + "\n"); } /** * Reserve the next evidence file, `/-