/** * A full sandboxed lifecycle for each app.env that ships outside this repo: * install → health gate → update → idempotence → rollback. * * WHY A SEPARATE FILE FROM app-env-conformance.test.ts. That file asks whether * the framework's new keys WORK; this one asks whether a real app can be * DEPLOYED with them, which is a different question with a different cost. Here * every run is a complete install or update — clone or tarball, install, build, * unit write, restart, health poll — so the suite is minutes rather than * seconds, and a failure inside it is about a lifecycle, not about one key. * * WHAT IS SANDBOXED, AND WHY IT IS ENOUGH. Nothing privileged runs and nothing * outside the fixture tree is written: * * INSTALL_DIR / DATA_DIR / RUN_DIR / HOOKS_DIR / EXEC_START / UNIT_FILE are * redirected into a temp tree — in the app.env COPY, because load_app_env() * SOURCES app.env and a plain assignment there beats the process environment. * Both new app.envs carry `RUN_DIR=""`, which resolve_target_user() then * derives from `getent passwd` into the operator's REAL ~/.ziptask or * ~/.subagentix. That is the #1101 incident exactly, and it happened while * every assertion was green. * sudo and systemctl are PATH stubs. sudo has its OWN secure_path, so a stub * does not cover it in general — but here the stub is what the framework * resolves `sudo` to (it is first on PATH) and nothing else can reach the * real one, because the fixture never invokes an absolute path. AGENTS.md §8 * records a "sandboxed" demo that restarted the live service five times; the * difference is that this suite has no real systemctl to fall through to. * curl is a stub serving the release asset from a local directory and a chosen * /health body. No network, and no service is ever actually started — which * is the point of the health assertions: they exercise the GATE, not a * process. * guardRealStateDir() fingerprints the operator's real state dir for the whole * run and fails the run if it moved. That is the check which would have * caught #1101, and it verifies the containment from the OUTSIDE — it holds * even if a fixture bypasses every sandbox above. * * The app.env under test is the app's OWN file, unmodified except for the paths * above. Every other key — DIST, the health contract, INSTALL_FLAGS, BUILD_CMD, * BINARY_* — is what the repo ships. */ import { describe, expect, test, afterEach } from 'bun:test' import { spawnSync } from 'node:child_process' import { chmodSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { join } from 'node:path' import { readAppEnvValues, sandboxAppEnvText } from './app-env-targets' import { guardProductionPaths, resolveTargets } from './app-targets' import { guardRealStateDir, installCleanup, isolatedEnv, mkTempTree } from './tmp-fixtures' installCleanup() guardRealStateDir() const DEPLOY_DIR = import.meta.dir const TARGETS = resolveTargets() // Containment for the TARGETS, not just for this repo's own app: guardRealStateDir() // watches ~/.synaptomind, which says nothing about ~/.ziptask or ~/.subagentix. // See guardProductionPaths for why the check lives at process exit and why the // live tracker's own data directory is excluded from the watch list. guardProductionPaths(TARGETS.resolved) /** The /health body both new apps answer with (ADR §3d). */ const HEALTHY_BODY = '{"ok":true}' /** The same contract with the ok-value flipped — must fail, or the gate is vacuous. */ const UNHEALTHY_BODY = '{"ok":false}' let ROOT = '' afterEach(() => { if (ROOT) rmSync(ROOT, { recursive: true, force: true }) ROOT = '' }) function q(value: string): string { return `'${value.replace(/'/g, `'\\''`)}'` } function ensureRoot(prefix = 'sandbox-'): string { if (!ROOT) ROOT = mkTempTree(prefix) return ROOT } // ── Fixture ────────────────────────────────────────────────────────────────── interface Sandbox { target: { label: string; deployDir: string; appEnvPath: string } root: string installDir: string dataDir: string runDir: string unitFile: string stubsDir: string privLog: string /** The file the curl stub reads the /health body from. */ healthFile: string /** The file the bun stub records its argv in. */ bunLog: string /** Where the systemctl stub models the running unit. */ stateDir: string binary: boolean /** The version currently published on the origin / in the release dir. */ version: string /** Publish a new version on the origin (source) or in the release dir (binary). */ publishVersion(version: string): void /** The version the installed payload reports right now, or '' when it cannot. */ installedVersion(): string } const V1 = '1.0.0' const V2 = '2.0.0' function seedSandbox( target: { label: string; deployDir: string; appEnvPath: string }, opts: { initialVersion?: string } = {} ): Sandbox { const root = ensureRoot(`sandbox-${target.label}-`) const label = target.label const values = readAppEnvValues(target.appEnvPath) const binary = values.DIST === 'binary' const version = opts.initialVersion ?? V1 // ── the origin the framework will reach ── // A real bare git repo for DIST=source, a real `file://` release directory for // DIST=binary. Both are ordinary remotes to `git clone` / `curl`, because the // lifecycle under test includes the fetch: a fixture that handed install.sh a // pre-made INSTALL_DIR would skip the step that most often breaks on cutover. const originWork = join(root, 'origin-work') const originGit = join(root, 'origin.git') const releases = join(root, 'releases') const gitEnv = { ...process.env, GIT_AUTHOR_NAME: 'fixture', GIT_AUTHOR_EMAIL: 'fixture@test', GIT_COMMITTER_NAME: 'fixture', GIT_COMMITTER_EMAIL: 'fixture@test' } const git = (...args: string[]) => { const res = spawnSync('git', args, { encoding: 'utf8', cwd: originWork, env: gitEnv }) expect(res.status, `git ${args.join(' ')}: ${res.stderr}`).toBe(0) } const writeTree = (v: string) => { mkdirSync(originWork, { recursive: true }) writeFileSync( join(originWork, 'package.json'), `${JSON.stringify({ name: label, version: v }, null, 2)}\n` ) writeFileSync(join(originWork, '.env.example'), `${label.toUpperCase()}_SETTING=\n`) } const publishSource = (v: string) => { writeTree(v) git('init', '--quiet', '--initial-branch=main') git('add', '-A') git('commit', '--quiet', '-m', `release ${v}`) git('tag', `v${v}`) rmSync(originGit, { recursive: true, force: true }) git('clone', '--quiet', '--bare', originWork, originGit) } const publishBinary = (v: string) => { const tag = `v${v}` const topDir = `${label}-${v}-linux-x86_64` const stage = join(root, '.payload', topDir) mkdirSync(stage, { recursive: true }) writeFileSync(join(stage, label), `#!/bin/sh\necho "${label} ${v}"\n`) chmodSync(join(stage, label), 0o755) mkdirSync(join(releases, tag), { recursive: true }) const tar = spawnSync( 'tar', [ '-czf', join(releases, tag, `${label}-${tag}-linux-x86_64.tar.gz`), '-C', join(root, '.payload'), topDir ], { encoding: 'utf8' } ) expect(tar.status, tar.stderr).toBe(0) } binary ? publishBinary(version) : publishSource(version) // ── the sandboxed deploy tree ── const installDir = join(root, 'opt') const dataDir = join(root, 'data') const runDir = join(root, 'run') const unitFile = join(root, 'unit', `${label}.service`) mkdirSync(join(root, 'unit'), { recursive: true }) const deployDir = join(root, 'deploy') cpSync(DEPLOY_DIR, deployDir, { recursive: true, filter: src => !src.endsWith('.test.ts') && !src.endsWith('app-targets.ts') && !src.endsWith('app-env-targets.ts') }) const overrides: Record = { INSTALL_DIR: installDir, DATA_DIR: dataDir, RUN_DIR: runDir, HOOKS_DIR: join(runDir, 'hooks'), EXEC_START: join(installDir, label), ...(binary ? { RELEASES_BASE: `file://${releases}`, RELEASE_API: '' } : { REPO_URL: `file://${originGit}`, CHECKOUT_POLICY: 'stable' }) } for (const [key, value] of Object.entries(overrides)) { expect( value.startsWith(`${root}/`) || !['INSTALL_DIR', 'DATA_DIR', 'RUN_DIR', 'HOOKS_DIR', 'EXEC_START'].includes(key), `${key}=${value} escapes the fixture root` ).toBe(true) } writeFileSync(join(deployDir, 'app.env'), sandboxAppEnvText(target.appEnvPath, overrides)) // ── the stub layer ── const stubsDir = join(root, 'stubs') const privLog = join(root, 'privileged.log') mkdirSync(stubsDir, { recursive: true }) writeFileSync(privLog, '') // sudo: logs, allows only fixture-root paths, and execs a short allow-list. // `chown` is allowed so apply_ownership's real path is exercised; the refusal // path has its own dedicated test in app-env-conformance.test.ts. writeFileSync( join(stubsDir, 'sudo'), [ '#!/usr/bin/env bash', 'printf \'%s\' "$0" >> "$STUB_PRIV_LOG"', 'for a in "$@"; do printf \' %s\' "$a" >> "$STUB_PRIV_LOG"; done', 'printf \'\\n\' >> "$STUB_PRIV_LOG"', 'case "${1:-}" in', ' touch|chmod|cp|mv|rm|install|systemctl|mkdir|ln|chown) ;;', ' *) echo "STUB-SUDO: refused command ${1:-}" >&2; exit 99 ;;', 'esac', 'exec "$@"' ].join('\n') ) // systemctl: `is-active` must answer for real, because update.sh branches on // it; `restart` records the payload's version the way ExecStart would. const stateDir = join(root, 'unitstate') mkdirSync(stateDir, { recursive: true }) writeFileSync( join(stubsDir, 'systemctl'), [ '#!/usr/bin/env bash', 'printf \'%s\' "$0" >> "$STUB_PRIV_LOG"', 'for a in "$@"; do printf \' %s\' "$a" >> "$STUB_PRIV_LOG"; done', 'printf \'\\n\' >> "$STUB_PRIV_LOG"', `launch() { printf '%s' "$(${q(join(installDir, label))} --version 2>/dev/null | awk '{print $2}')" > ${q(join(stateDir, 'serving'))}; printf '%s launched %s\\n' "$1" "$(cat ${q(join(stateDir, 'serving'))})" >> ${q(join(stateDir, 'events'))}; }`, 'case "${1:-}" in', ' is-system-running) echo running ;;', ' is-active) [ -f ' + q(join(stateDir, 'serving')) + ' ] && exit 0 || exit 3 ;;', ' start|try-restart) [ -f ' + q(join(stateDir, 'serving')) + ' ] || launch "$1" ;;', ' restart) launch restart ;;', ' stop) rm -f ' + q(join(stateDir, 'serving')) + ' ;;', 'esac', 'exit 0' ].join('\n') ) // curl: serves the release asset from the local release dir, and the health // body the current test wants. `healthBody` is a mutable cell so a test can // flip the endpoint between runs without re-seeding the stub. const healthFile = join(root, 'health-body') writeFileSync(healthFile, HEALTHY_BODY) writeFileSync( join(stubsDir, 'curl'), [ '#!/usr/bin/env bash', 'url=""; out=""; prev=""', 'for a in "$@"; do', ' case "$a" in *://*) url="$a" ;; esac', ' if [ "$prev" = "-o" ]; then out="$a"; fi', ' prev="$a"', 'done', `case "$url" in`, // The URL carries a file:// scheme; `cp` does not want it. Stripping it // here rather than at the call site keeps one stub for every fetch. ` file://*) cp "\${url#file://}" "$out"; exit \$? ;;`, ` *) cat ${q(healthFile)}; printf '\\n'; exit 0 ;;`, 'esac' ].join('\n') ) // bun: records argv and creates the artefact a build would produce, so a build // step that "succeeded" is observable as a file rather than as an exit code. const bunLog = join(root, 'bun.log') writeFileSync(bunLog, '') writeFileSync( join(stubsDir, 'bun'), [ '#!/usr/bin/env bash', `printf 'bun %s\\n' "$*" >> ${q(bunLog)}`, 'if [ "${1:-}" = "run" ] && [ "${2:-}" = "build" ]; then', ` mkdir -p ${q(join(installDir, 'build'))}`, ` printf 'built\\n' > ${q(join(installDir, 'build', 'index.js'))}`, 'fi', 'exit 0' ].join('\n') ) // stat: report a foreign owner so apply_ownership takes the branch it added // (it skips a chown that would change nothing). Only %U:%G is intercepted; // %a drives the unit's preserved mode and must stay real. writeFileSync( join(stubsDir, 'stat'), [ '#!/usr/bin/env bash', 'for a in "$@"; do', ` if [ "$a" = "%U:%G" ]; then printf 'root:root\\n'; exit 0; fi`, 'done', 'exec /usr/bin/stat "$@"' ].join('\n') ) for (const name of ['sudo', 'systemctl', 'curl', 'bun', 'stat']) chmodSync(join(stubsDir, name), 0o755) const sb: Sandbox = { target, root, installDir, dataDir, runDir, unitFile, stubsDir, privLog, healthFile, bunLog, stateDir, binary, version, publishVersion(v: string) { binary ? publishBinary(v) : publishSource(v) sb.version = v }, installedVersion() { const pkg = join(installDir, 'package.json') const bin = join(installDir, label) if (binary) { if (!existsSync(bin)) return '' const res = spawnSync(bin, ['--version'], { encoding: 'utf8' }) return (res.stdout ?? '').trim().split(/\s+/)[1] ?? '' } if (!existsSync(pkg)) return '' return /"version": *"([^"]*)"/.exec(readFileSync(pkg, 'utf8'))?.[1] ?? '' } } return sb } /** The environment every spawn in this file gets. */ function sandboxEnv(sb: Sandbox, extra: Record = {}): Record { return isolatedEnv(sb.root, { PATH: `${sb.stubsDir}:${process.env.PATH}`, STUB_PRIV_LOG: sb.privLog, APP_ENV_URL: '', LIB_RAW_URL: '', UNIT_FILE: sb.unitFile, HEALTH_TIMEOUT: '2', HEALTH_CONFIRM_TIMEOUT: '1', ...extra }) } /** * The `--version` a run needs, or nothing. * * For DIST=binary the tag comes from `release_resolve_tag`, which queries * RELEASE_API — and the sandbox sets that empty (there is no API to query, and * pointing it at github.com would put the test on the network). So every binary * run must name its tag explicitly, exactly as the framework's own error says: * "RELEASE_API is empty; set it in app.env or pass --version ". A source app * resolves its ref from the origin's tags and needs no flag. */ function tagArgs(sb: Sandbox): string[] { return sb.binary ? ['--version', `v${sb.version}`] : [] } /** Run install.sh out of the sandboxed deploy tree. */ function install(sb: Sandbox, args: string[] = [], extra: Record = {}) { return spawnSync('bash', [join(sb.root, 'deploy', 'install.sh'), ...tagArgs(sb), ...args], { encoding: 'utf8', env: sandboxEnv(sb, extra), cwd: sb.root, timeout: 120_000 }) } /** * Run the INSTALLED update.sh — the copy install.sh put in ${RUN_DIR}/scripts. * * Not the deploy/ copy: an operator updates through the installed one, and that * copy carries the INSTALLED app.env, which is the file a real cutover reads. * Running the repo's copy would test a configuration no operator ever has. */ function update(sb: Sandbox, args: string[] = [], extra: Record = {}) { const script = join(sb.runDir, 'scripts', 'update.sh') if (!existsSync(script)) throw new Error(`no installed update.sh at ${script} — install.sh never ran to completion`) return spawnSync('bash', [script, '--yes', ...tagArgs(sb), ...args], { encoding: 'utf8', env: sandboxEnv(sb, extra), cwd: sb.runDir, timeout: 120_000 }) } function privCalls(log: string): string[] { const raw = readFileSync(log, 'utf8').trim() return raw ? raw.split('\n') : [] } /** Set the body the health endpoint answers with for the next run. */ function serveHealth(sb: Sandbox, body: string): void { writeFileSync(sb.healthFile, body) } // ══════════════════════════════════════════════════════════════════════════════ /** * Per-test budget. Every run here is a full install or update, and a run that * exercises the FAILING gate deliberately spends HEALTH_TIMEOUT + * HEALTH_CONFIRM_TIMEOUT before it can report a verdict — so bun's 5 s default * expired mid-install and the failures read as timeouts rather than as the * health verdict under test. 120 s is generous for a temp-tree install; the * tests that finish fast still finish fast. */ const BUDGET_MS = 120_000 describe('deploy/sandbox — install, health gate, update, rollback', () => { for (const target of TARGETS.resolved) { const values = readAppEnvValues(target.appEnvPath) test( `${target.label} (DIST=${values.DIST}): a fresh install completes and the health gate passes`, () => { const sb = seedSandbox(target) const res = install(sb) expect(res.status, `${res.stdout}\n${res.stderr}`).toBe(0) expect(res.stdout).toContain('Done.') // The gate really was consulted, and it really accepted this app's own // `{"ok":true}` — which is what the three health keys in app.env buy. expect(`${res.stdout}${res.stderr}`).toMatch(/Service is healthy/) // The disabled version check is stated, once: a caveat reported by // silence would be indistinguishable from a gate that never ran. const warnings = res.stderr.split('\n').filter(l => l.includes('version check is DISABLED')) expect(warnings).toHaveLength(1) // The payload landed where the app.env said, and reports the version the // origin published. expect(sb.installedVersion()).toBe(V1) expect(existsSync(sb.dataDir), 'DATA_DIR was never created').toBe(true) // Canonical layout: the data dir is reachable from the install dir too. expect(existsSync(join(sb.installDir, 'data'))).toBe(true) // A unit was written — inside the fixture, named for THIS app. expect(existsSync(sb.unitFile), `no unit at ${sb.unitFile}`).toBe(true) const unit = readFileSync(sb.unitFile, 'utf8') expect(unit).toContain(`ExecStart=${join(sb.installDir, target.label)}`) expect(unit).toContain('Restart=always') expect(unit).toContain(`ReadWritePaths=${sb.installDir} ${sb.dataDir}`) // The sandboxed app.env copied into the state dir, not the shipped one. expect(readFileSync(join(sb.runDir, 'scripts', 'app.env'), 'utf8')).toContain( `RUN_DIR="${sb.runDir}"` ) // Helpers and hooks, so the update below runs the same files an operator has. for (const f of ['update.sh', 'updater.sh', 'uninstall.sh', 'common.sh', 'app.env']) { expect(existsSync(join(sb.runDir, 'scripts', f)), `helper ${f} was not installed`).toBe( true ) } for (const f of ['pre-update', 'post-update']) { expect(existsSync(join(sb.runDir, 'hooks', f)), `hook ${f} was not installed`).toBe(true) } // Nothing privileged escaped: every sudo/systemctl call the run made is in // the fixture's log, and that log contains only fixture paths. const calls = privCalls(sb.privLog) expect(calls.length).toBeGreaterThan(0) for (const call of calls) { expect(call, `privileged call outside the fixture: ${call}`).toContain(sb.root) } }, BUDGET_MS ) test( `${target.label}: an UNHEALTHY endpoint fails the install and never says Done`, () => { // The negative control for the test above. Without it, "the gate passed" // and "the gate was never reached" are the same observation. const sb = seedSandbox(target) serveHealth(sb, UNHEALTHY_BODY) const res = install(sb) expect(res.status, `${res.stdout}\n${res.stderr}`).not.toBe(0) expect(res.stdout).not.toContain('Done.') expect(`${res.stdout}${res.stderr}`).toMatch(/health check|did not pass the health check/i) }, BUDGET_MS ) test( `${target.label}: update is idempotent — a second run changes nothing`, () => { const sb = seedSandbox(target) expect(install(sb).status).toBe(0) const before = sb.installedVersion() expect(before).toBe(V1) const res = update(sb) expect(res.status, `${res.stdout}\n${res.stderr}`).toBe(0) // Idempotence is the OBSERVED state, not the message: same version still // installed, and the health gate passed again rather than being skipped. expect(sb.installedVersion()).toBe(before) expect(`${res.stdout}${res.stderr}`).toMatch(/Already up to date|Done\. Now at/) expect(`${res.stdout}${res.stderr}`).not.toMatch(/ERROR/) }, BUDGET_MS ) test( `${target.label}: update to a NEW version swaps the payload and re-passes the gate`, () => { const sb = seedSandbox(target) expect(install(sb).status).toBe(0) expect(sb.installedVersion()).toBe(V1) sb.publishVersion(V2) const res = update(sb) expect(res.status, `${res.stdout}\n${res.stderr}`).toBe(0) expect(sb.installedVersion(), `${res.stdout}\n${res.stderr}`).toBe(V2) expect(`${res.stdout}${res.stderr}`).toMatch(/Done\. Now at/) // A source app re-ran the same install+build pair the update path shares // with install.sh; the trace proves the pair ran on the UPDATE, which is // the invariant gap (a) closed by moving it into lib/common.sh. if (values.DIST === 'source') { const bunCalls = readFileSync(sb.bunLog, 'utf8').split('\n').filter(Boolean) expect(bunCalls.length, 'the update ran no build').toBeGreaterThanOrEqual(4) expect( bunCalls.some(c => c.includes('install')), 'the update installed no dependencies' ).toBe(true) expect( bunCalls.some(c => c.includes('run build')), 'the update ran no build' ).toBe(true) } }, BUDGET_MS ) test( `${target.label}: a failed health gate blocks the update and leaves a rollback point`, () => { const sb = seedSandbox(target) expect(install(sb).status).toBe(0) expect(sb.installedVersion()).toBe(V1) sb.publishVersion(V2) serveHealth(sb, UNHEALTHY_BODY) const res = update(sb) expect(res.status, `${res.stdout}\n${res.stderr}`).not.toBe(0) expect(res.stdout).not.toMatch(/Done\. Now at/) // Rollback is NEVER automatic (ADR §2.6): the payload is still the new one, // and the operator is told what to do instead. expect(res.stderr).toMatch(/rollback/i) expect(res.stderr).toMatch(/did not finish cleanly|not verified|unverified|health/i) }, BUDGET_MS ) test( `${target.label}: rollback works — the printed remedy restores the previous version`, () => { const sb = seedSandbox(target) expect(install(sb).status).toBe(0) expect(sb.installedVersion()).toBe(V1) sb.publishVersion(V2) serveHealth(sb, UNHEALTHY_BODY) const failed = update(sb) expect(failed.status, `${failed.stdout}\n${failed.stderr}`).not.toBe(0) expect( sb.installedVersion(), 'the failed update did not land its payload, so there is nothing to roll back' ).toBe(V2) // ── the rollback point ── // For DIST=binary the framework keeps a `.prev` beside every file in // BINARY_ROLLBACK_FILES. Asserting the FILE exists is the rollback point: // without it the remedy has nothing to move back. const prevPath = join(sb.installDir, `${target.label}.prev`) if (values.DIST === 'binary') { expect(existsSync(prevPath), `no rollback point at ${prevPath}`).toBe(true) const prev = spawnSync(prevPath, ['--version'], { encoding: 'utf8' }) expect( (prev.stdout ?? '').trim().split(/\s+/)[1], 'the .prev is not the previous version' ).toBe(V1) } // ── run the remedy the framework printed, and check the state afterwards ── // The remedy is executed, not just pattern-matched: a rollback instruction // that is printed but does not restore anything is the failure mode. serveHealth(sb, HEALTHY_BODY) if (values.DIST === 'binary') { for (const f of values.BINARY_ROLLBACK_FILES.split(/\s+/).filter(Boolean)) { const prev = join(sb.installDir, `${f}.prev`) if (existsSync(prev)) spawnSync('mv', ['-f', prev, join(sb.installDir, f)]) } } else { // The source remedy update.sh prints is a single command line built from // PREV_REF + INSTALL_FLAGS + BUILD_CMD. Take it verbatim off the output // rather than reconstructing it, so this proves the PRINTED remedy works. const line = failed.stderr.split('\n').find(l => l.includes('rollback:')) expect(line, `no rollback remedy in:\n${failed.stderr}`).toBeTruthy() const prevRef = /previous commit: (\S+)/.exec(failed.stderr)?.[1] expect(prevRef, 'the remedy does not name the previous commit').toBeTruthy() const remedy = line!.replace(/^.*rollback:\s*/, '').trim() expect(remedy).toContain(prevRef!) const run = spawnSync('bash', ['-c', remedy], { encoding: 'utf8', env: sandboxEnv(sb), cwd: sb.runDir, timeout: 120_000 }) expect(run.status, `the printed remedy failed: ${run.stderr}`).toBe(0) } expect(sb.installedVersion(), 'the rollback did not restore the previous version').toBe(V1) // And the rolled-back tree comes back up: a rollback that leaves a payload // that cannot serve is not a rollback. const res = install(sb, ['--force']) expect(res.status, `${res.stdout}\n${res.stderr}`).toBe(0) expect(`${res.stdout}${res.stderr}`).toMatch(/Service is healthy/) }, BUDGET_MS ) } })