#!/usr/bin/env node
// "With plugin vs without" end to end: install coding.sh from a tarball and the
// pinned Claude Code from npm under a throwaway HOME, point it at the local stub
// model, and run `coding.sh compare base --with plugin:
` on one task with a
// tiny local plugin: a SessionStart hook (an executable shell script) that logs
// the session's cwd and prints a marker into the context, plus a skill.
//
// The stub writes a correct sum.js only when the hook's marker reached the
// model, so B (with the plugin) passes and A fails. Each run's work dir names
// its run id, which the store maps to its bundle: the hook must have run in
// every B run and no A run, every B request must carry the marker and list the
// plugin's skill, and no A request may. Then `--without plugin:tiny` on the
// derived bundle reverses it, and the user's Claude config dir (a fake one with
// the plugin installed) must come out byte-for-byte the same.
//
// node scripts/e2e/plugin-capture.mjs [--tarball coding.sh-x.y.z.tgz] [--k 2] [--keep]
//
// Without --tarball it packs packages/coding.sh (build it first: npm run build).
// Needs network for npm. Never touches the real HOME. Exit 0 when every check passed.
import { createHash } from "node:crypto";
import { execFileSync } from "node:child_process";
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { delimiter, dirname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { spawn } from "../../packages/coding.sh/src/core/spawn.ts";
import { startStub, toolNames } from "./stub-model.mjs";
const REPO = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
const CLAUDE_PKG = "@anthropic-ai/claude-code@2.1.319";
const MARKER = "TINY-PLUGIN-HOOK-MARKER: sum.js must use the + operator.";
const SKILL = "sum-plus";
const INSTRUCTION = "Make sum.js add its two arguments.";
const GOOD = "module.exports = (a, b) => a + b;\n";
const BAD = "module.exports = (a, b) => a - b;\n";
function parseArgs(argv) {
const o = { keep: false, tarball: undefined, k: 2 };
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a === "--keep") o.keep = true;
else if (a === "--tarball") o.tarball = resolve(argv[++i]);
else if (a === "--k") o.k = Number(argv[++i]);
else throw new Error(`unknown argument ${a}`);
}
return o;
}
const log = (...a) => console.log("[plugin-capture]", ...a);
const failures = [];
function check(ok, what, detail) {
if (ok) log(`ok ${what}`);
else {
log(`FAIL ${what}${detail === undefined ? "" : `\n ${typeof detail === "string" ? detail : JSON.stringify(detail)}`}`);
failures.push(what);
}
return ok;
}
function run(file, args, { cwd, env, timeoutMs = 900_000, quiet = false } = {}) {
return new Promise((done) => {
const child = spawn(file, args, { cwd, env: cwd && env ? { ...env, PWD: cwd } : env, stdio: ["ignore", "pipe", "pipe"], windowsHide: true });
let stdout = "";
let stderr = "";
child.stdout.on("data", (d) => (stdout += d));
child.stderr.on("data", (d) => (stderr += d));
const timer = setTimeout(() => child.kill(), timeoutMs);
child.on("error", (e) => {
clearTimeout(timer);
done({ code: -1, stdout, stderr: stderr + String(e) });
});
child.on("close", (code) => {
clearTimeout(timer);
if (!quiet && code !== 0 && code !== 1) log(`${file} ${args.join(" ")} exited ${code}\n--- stdout\n${stdout.slice(-4000)}\n--- stderr\n${stderr.slice(-4000)}`);
done({ code, stdout, stderr });
});
});
}
function fingerprint(dir) {
const out = {};
const walk = (d) => {
for (const e of readdirSync(d, { withFileTypes: true })) {
const abs = join(d, e.name);
if (e.isDirectory()) walk(abs);
else out[relative(dir, abs)] = `${createHash("sha256").update(readFileSync(abs)).digest("hex")}:${statSync(abs).mode}`;
}
};
walk(dir);
return out;
}
/** The run id in a coding.sh run's work dir (//work). */
const runIdOf = (path) => /[\\/](run_[A-Za-z0-9_-]+)[\\/]work\b/.exec(path ?? "")?.[1];
/** What each model request carried, by the run it came from. */
const seen = [];
function scenario(body) {
const names = toolNames(body);
if (!Array.isArray(body.messages) || !names.includes("Read") || !names.includes("Write")) return { text: "ok" };
const all = JSON.stringify(body);
if (!all.includes(INSTRUCTION)) return { text: "ok" };
const cwd = /[Ww]orking directory: ([^\s"\\]+)/.exec(all)?.[1];
const marker = all.includes(MARKER);
seen.push({ run: runIdOf(cwd), marker, skill: all.includes(SKILL) });
if (!cwd) return { text: "ok" };
const file = `${cwd}/sum.js`;
const step = body.messages.reduce((n, m) => n + (Array.isArray(m.content) ? m.content.filter((c) => c?.type === "tool_result").length : 0), 0);
const plan = [
{ tool: { name: "Read", args: { file_path: file } } },
{ tool: { name: "Write", args: { file_path: file, content: marker ? GOOD : BAD } } },
];
return plan[step] ?? { text: "Done." };
}
// ------------------------------------------------------------------ main
const opts = parseArgs(process.argv.slice(2));
const root = mkdtempSync(join(process.env.RUNNER_TEMP || tmpdir(), "csh-plugin-"));
const home = join(root, "home");
const prefix = join(root, "npm-global");
const csHome = join(home, ".coding.sh");
const claudeDir = join(home, ".claude");
const hookLog = join(root, "hook.log");
mkdirSync(home, { recursive: true });
mkdirSync(prefix, { recursive: true });
writeFileSync(join(home, ".gitconfig"), "[user]\n\tname = coding.sh e2e\n\temail = e2e@coding.sh.invalid\n[init]\n\tdefaultBranch = main\n");
const binDir = process.platform === "win32" ? prefix : join(prefix, "bin");
const stub = await startStub({ decide: scenario });
const env = {};
for (const [k, v] of Object.entries(process.env)) {
if (/^(XDG_|CLAUDE|ANTHROPIC|OPENAI|CODEX|CODING_SH|npm_config_|NPM_CONFIG_)/i.test(k)) continue;
env[k] = v;
}
const pathKey = Object.keys(env).find((k) => k.toUpperCase() === "PATH") ?? "PATH";
Object.assign(env, {
HOME: home,
USERPROFILE: home,
CODING_SH_HOME: csHome,
[pathKey]: [binDir, env[pathKey]].filter(Boolean).join(delimiter),
npm_config_prefix: prefix,
npm_config_fund: "false",
npm_config_audit: "false",
npm_config_update_notifier: "false",
CODING_SH_NO_UPDATE_CHECK: "1",
CODING_SH_NO_UI: "1",
ANTHROPIC_BASE_URL: stub.url,
ANTHROPIC_API_KEY: "sk-stub",
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
DISABLE_AUTOUPDATER: "1",
DISABLE_TELEMETRY: "1",
...(process.getuid?.() === 0 ? { IS_SANDBOX: "1" } : {}),
});
if (process.env.npm_config_cache) env.npm_config_cache = process.env.npm_config_cache;
log(`temp root ${root}`);
log(`stub model on ${stub.url}`);
/** The tiny plugin: an executable SessionStart hook that logs its cwd and prints the marker, plus a skill. */
function writePlugin(dir) {
const files = {
".claude-plugin/plugin.json": JSON.stringify({ name: "tiny", version: "0.1.0", description: "coding.sh e2e plugin" }),
"hooks/hooks.json": JSON.stringify({ hooks: { SessionStart: [{ hooks: [{ type: "command", command: '"${CLAUDE_PLUGIN_ROOT}/hooks/start.sh"' }] }] } }),
"hooks/start.sh": `#!/bin/sh\ninput=$(cat)\nprintf '%s\\t%s\\n' "$CLAUDE_PLUGIN_ROOT" "$input" | tr -d '\\r' >> ${JSON.stringify(hookLog)}\necho ${JSON.stringify(MARKER)}\n`,
[`skills/${SKILL}/SKILL.md`]: `---\nname: ${SKILL}\ndescription: Use when changing sum.js.\n---\nUse the + operator.\n`,
};
for (const [p, c] of Object.entries(files)) {
mkdirSync(dirname(join(dir, p)), { recursive: true });
writeFileSync(join(dir, p), c);
}
chmodSync(join(dir, "hooks/start.sh"), 0o755);
}
/** Hook log lines: the plugin root and the session's cwd. */
const hookRuns = () =>
existsSync(hookLog)
? readFileSync(hookLog, "utf8")
.trim()
.split("\n")
.filter(Boolean)
.map((l) => {
const [pluginRoot, input] = l.split("\t");
let cwd;
try {
cwd = JSON.parse(input).cwd;
} catch {}
return { pluginRoot, cwd, run: runIdOf(cwd) };
})
: [];
let exitCode = 0;
try {
let tarball = opts.tarball;
if (!tarball) {
const pkgDir = join(REPO, "packages", "coding.sh");
if (!existsSync(join(pkgDir, "dist", "cli.mjs"))) throw new Error("packages/coding.sh/dist is missing: run npm run build first, or pass --tarball");
const packed = await run("npm", ["pack", "--silent", "--pack-destination", root], { cwd: pkgDir });
if (packed.code !== 0) throw new Error("npm pack failed");
tarball = join(root, packed.stdout.trim().split(/\r?\n/).pop());
}
log(`installing ${tarball} and ${CLAUDE_PKG}`);
const install = await run("npm", ["install", "-g", "--no-fund", "--no-audit", tarball, CLAUDE_PKG], { env, cwd: root });
if (install.code !== 0) throw new Error("npm install -g failed");
const cs = (args, o = {}) => run("coding.sh", args, { env, cwd: root, ...o });
const json = (r, what) => {
try {
return JSON.parse(r.stdout);
} catch {
throw new Error(`${what}: not JSON (exit ${r.code}):\n${r.stdout.slice(0, 2000)}\n${r.stderr.slice(0, 2000)}`);
}
};
log(`coding.sh ${(await cs(["--version"])).stdout.trim()}`);
log(`claude ${(await run("claude", ["--version"], { env, cwd: root })).stdout.trim()}`);
const init = await cs(["init", "--agent", "claude-code", "--yes", "--no-mcp", "--json"]);
check(init.code === 0, "coding.sh init --agent claude-code", init.stderr);
// The user's Claude config: the plugin installed and enabled, as Claude Code would leave it.
const installed = join(claudeDir, "plugins", "cache", "e2e", "tiny", "0.1.0");
writePlugin(installed);
writeFileSync(join(claudeDir, "plugins", "installed_plugins.json"), JSON.stringify({ version: 2, plugins: { "tiny@e2e": [{ scope: "user", installPath: installed, version: "0.1.0" }] } }));
const settingsFile = join(claudeDir, "settings.json");
const settings = existsSync(settingsFile) ? JSON.parse(readFileSync(settingsFile, "utf8")) : {};
writeFileSync(settingsFile, JSON.stringify({ ...settings, enabledPlugins: { "tiny@e2e": true } }, null, 2));
const configBefore = fingerprint(claudeDir);
// The same plugin in a plain dir, for plugin:.
const pluginDir = join(root, "tiny-plugin");
writePlugin(pluginDir);
const repo = join(root, "repo");
mkdirSync(repo);
writeFileSync(join(repo, "sum.js"), BAD);
writeFileSync(join(repo, "test-sum.js"), 'if (require("./sum.js")(2, 3) !== 5) process.exit(1);\n');
const git = (...a) => execFileSync("git", a, { cwd: repo, env, stdio: "pipe" });
git("init", "-q");
git("add", "-A");
git("commit", "-qm", "init");
const fix = `require("fs").writeFileSync("sum.js", ${JSON.stringify(GOOD)})`;
check((await run("coding.sh", ["run", "--", process.execPath, "-e", fix], { env, cwd: repo })).code === 0, "reference session");
const created = json(await run("coding.sh", ["eval", "new", "latest", "--name", "sum", "--verifier", "node test-sum.js", "--instruction", INSTRUCTION, "--json"], { env, cwd: repo }), "eval new");
const taskId = created.task?.id ?? created.id;
git("checkout", "-q", "sum.js");
const h = join(root, "harness-base");
mkdirSync(h);
writeFileSync(join(h, "CLAUDE.md"), "Keep changes small.\n");
writeFileSync(join(h, "harness.toml"), `name = "base"\nagent = "claude-code"\nmodel = "claude-sonnet-4-5"\ninclude = ["CLAUDE.md"]\n`);
check((await cs(["harness", "snapshot", "--file", join(h, "harness.toml"), "--json"])).code === 0, "harness snapshot base");
/** Run id → bundle name, from the store. */
const armsOf = async (experimentId) => {
const q = await cs(["sql", `select r.id as id, b.name as bundle from runs r join bundles b on b.id = r.bundle_id where r.experiment_id = '${experimentId}'`, "--json"]);
const rows = json(q, "sql runs").rows ?? json(q, "sql runs");
return new Map(rows.map((r) => [r.id, r.bundle]));
};
/** One comparison: B's runs ran the plugin, A's did not, judged from the hook log, the model requests and the store. */
const verify = async (label, args, withArm, withoutArm) => {
const hooksBefore = hookRuns().length;
const seenBefore = seen.length;
const r = await cs(["compare", ...args, "-t", taskId, "--k", String(opts.k), "-c", "2", "--yes", "--json"]);
const out = json(r, `compare ${args.join(" ")}`);
const text = await cs(["compare", ...args, "--experiment", out.experimentId], { quiet: true });
log(`${label} (exit ${r.code}):\n${text.stdout}`);
const { baseline, candidate } = out.report;
const arm = (name) => (baseline.name === name ? baseline : candidate);
check(arm(withArm).passes === opts.k && arm(withoutArm).passes === 0, `${label}: ${withArm} passes ${opts.k}/${opts.k}, ${withoutArm} 0/${opts.k}`, { baseline, candidate });
const arms = await armsOf(out.experimentId);
check(arms.size === 2 * opts.k, `${label}: ${2 * opts.k} runs stored`, [...arms]);
const hooks = hookRuns().slice(hooksBefore);
log(`hook runs ${JSON.stringify(hooks.map((x) => ({ run: x.run, arm: arms.get(x.run), pluginRoot: x.pluginRoot })))}`);
check(hooks.length === opts.k && hooks.every((x) => arms.get(x.run) === withArm), `${label}: the plugin's SessionStart hook ran in every ${withArm} run and no ${withoutArm} run`, hooks);
check(
hooks.every((x) => x.pluginRoot && /[\\/]home[\\/]\.coding\.sh-plugins[\\/]tiny$/.test(x.pluginRoot) && !x.pluginRoot.startsWith(claudeDir) && !x.pluginRoot.includes(`${x.run}/work`)),
`${label}: the plugin loaded from the run's own home, not the work dir or the user's config`,
hooks.map((x) => x.pluginRoot),
);
const reqs = seen.slice(seenBefore).filter((s) => s.run && arms.has(s.run));
const inArm = (name) => reqs.filter((s) => arms.get(s.run) === name);
log(`requests ${withArm}: ${inArm(withArm).length} (marker ${inArm(withArm).filter((s) => s.marker).length}, skill ${inArm(withArm).filter((s) => s.skill).length}); ${withoutArm}: ${inArm(withoutArm).length} (marker ${inArm(withoutArm).filter((s) => s.marker).length}, skill ${inArm(withoutArm).filter((s) => s.skill).length})`);
check(inArm(withArm).length > 0 && inArm(withArm).every((s) => s.marker && s.skill), `${label}: every ${withArm} model request carries the hook's context and lists the plugin's skill`);
check(inArm(withoutArm).length > 0 && inArm(withoutArm).every((s) => !s.marker && !s.skill), `${label}: no ${withoutArm} model request has either`);
return { out, text: text.stdout };
};
const w = await verify("base vs --with plugin:", ["base", "--with", `plugin:${pluginDir}`], "base-with-plugin-tiny", "base");
check(/^ {2}\+ plugin tiny$/m.test(w.text), "the bundle diff reads + plugin tiny");
const wo = await verify("--without plugin:tiny", ["base-with-plugin-tiny", "--without", "plugin:tiny"], "base-with-plugin-tiny", "base-with-plugin-tiny-without-plugin-tiny");
check(/^ {2}- plugin tiny$/m.test(wo.text), "the bundle diff reads - plugin tiny");
// By name, from the (fake) user's installed plugins: the same derived bundle content, read-only.
const byName = json(await cs(["compare", "base", "--with", "plugin:tiny", "-t", taskId, "--dry-run", "--json"]), "compare --with plugin:tiny --dry-run");
check(byName.candidate?.name === "base-with-plugin-tiny", "--with plugin:tiny resolves the installed plugin", byName);
const list = json(await cs(["harness", "list", "--json"]), "harness list");
const derived = list.bundles.find((b) => b.name === "base-with-plugin-tiny");
check(derived?.plugins?.[0]?.name === "tiny", "harness list shows the derived bundle's plugin", derived);
check(JSON.stringify(fingerprint(claudeDir)) === JSON.stringify(configBefore), "the user's Claude config dir is byte-for-byte untouched");
} catch (e) {
log(`error: ${e instanceof Error ? e.stack : e}`);
failures.push(String(e));
} finally {
await stub.close();
if (failures.length) {
exitCode = 1;
log(`requests seen: ${JSON.stringify(seen.slice(-20))}`);
const errLog = join(csHome, "spool", "errors.log");
if (existsSync(errLog)) log(`hook errors:\n${readFileSync(errLog, "utf8").slice(-4000)}`);
log(`${failures.length} check(s) failed: ${failures.join("; ")}`);
} else log("all checks passed");
if (!opts.keep) rmSync(root, { recursive: true, force: true, maxRetries: 5 });
else log(`kept ${root}`);
}
process.exit(exitCode);