name: CI on: push: branches: [main] pull_request: # `labeled` starts the full suite when a pull request gets the `ready` label. types: [opened, synchronize, reopened, labeled] workflow_dispatch: permissions: contents: read concurrency: group: ci-${{ github.ref }} cancel-in-progress: true # The full suite is every job below. It runs on a pull request that carries the `ready` label. # Only then does a check named "full suite" exist, and main requires it (CONTRIBUTING.md, "Pull # requests"), so nothing merges before every job has passed on the pull request merged with an # up-to-date main. A manual run runs every job but Site perf, and its closing check has another # name ("full suite without site perf"). Without the label a pull request gets the quick jobs # only: the Linux tests on Node 24, the build, the gate, the stores, the Python SDK, the Node # floor, the release smoke test and the GitHub Action, minus the ones scripts/ci-changes.mjs can # prove it does not need. Its Linux tests run the test files its diff can reach, or all of them # when a changed path has no rule (packages/coding.sh/scripts/test-select.ts). Windows, Node 22, # the container image and cli-suite run in the full suite only, and the full suite always runs # every test file. # A push to main is a merge the full suite has already passed on, tree for tree: main requires # the check on a branch that is up to date. So the push runs the quick jobs, with every test # file and no job skipped, and the real agent captures. The captures install agents from npm, # GitHub and PyPI, the part of CI that can change with no commit here, and their Windows leg # saves the Windows npm and uv caches: a pull request can restore main's caches and its own, # never another pull request's. A push has no closing check: nothing reads one. env: FULL: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'ready') }} jobs: # A pull request that only touches docs or the site (scripts/ci-changes.mjs) skips the jobs # that never read them. The full suite never skips, and neither does a pull request whose diff # can't be read: the gated jobs run unless this says code=false. changes: name: what changed runs-on: ubuntu-latest timeout-minutes: 5 outputs: code: ${{ steps.diff.outputs.code }} site_perf: ${{ steps.diff.outputs.site_perf }} steps: # The pull request's merge commit and its first parent, the base branch. - uses: actions/checkout@v5 with: fetch-depth: 2 - name: Docs or code id: diff env: EVENT: ${{ github.event_name }} run: | if [ "$FULL" != true ] && [ "$EVENT" = pull_request ] && files=$(git diff --name-only --no-renames HEAD^1 HEAD); then printf '%s\n' "$files" | node scripts/ci-changes.mjs | tee -a "$GITHUB_OUTPUT" else printf 'code=true\nsite_perf=true\n' | tee -a "$GITHUB_OUTPUT" fi # The test suite runs on Linux and Windows in shards of about equal time # (scripts/test-shards.ts), so the checks that need one run only, not one per OS and Node, live # in the build job below. A pull request and a push to main run Linux on Node 24. The full # suite (the `ready` label, or a manual run) runs both Nodes on both. No job runs on macOS. # The quick run of a pull request is the one a person waits on, so it splits the same files # into six legs; the full suite and a push keep three per Node. # Everything here is GitHub-hosted: free for this public repo, and it leaves the desk runners # to deploys and Site perf. test: name: node ${{ matrix.node }} / ubuntu-latest (${{ matrix.shard }}/${{ matrix.of }}) runs-on: ubuntu-latest timeout-minutes: 20 strategy: fail-fast: false matrix: # The suite runs .ts from source, which needs Node 22.18 or newer (type stripping on by # default). The runtime-floor job runs the packed CLI on the engines floor. # A pull request runs Node 24; the full suite adds Node 22. The Node 24 legs launch the # built CLI and the Node 22 legs launch it from source (the build step below). node: ${{ fromJSON((github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'ready')) && '["22.x", "24.x"]' || '["24.x"]') }} # `of` has one value, the number of legs, so the name and SHARD below can read it. shard: ${{ fromJSON((github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ready')) && '[1, 2, 3, 4, 5, 6]' || '[1, 2, 3]') }} of: ${{ fromJSON((github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ready')) && '[6]' || '[3]') }} env: # The viewer UI tests drive headless Chrome and skip without one. GitHub's Linux image ships # Chrome; naming it makes a missing one fail instead of skip. CODING_SH_TEST_CHROME: /usr/bin/google-chrome steps: # Two commits: the quick run of a pull request reads its diff (the Test step below). - uses: actions/checkout@v5 with: fetch-depth: 2 - uses: actions/setup-node@v5 with: node-version: ${{ matrix.node }} cache: npm # Some tests make throwaway commits in temp repos. - name: Git identity run: | git config --global user.name "coding.sh CI" git config --global user.email "ci@coding.sh.invalid" # The tests need only the CLI package's own dependencies (esbuild, the two TypeScripts and # Node's types). The site's and the SDK's are most of a whole-workspace install, so they are # left out. The one leg that also checks the SDK (below) installs the SDK's too. - name: Install run: npm ci --workspace packages/coding.sh ${{ (matrix.node == '22.x' && matrix.shard == 1) && '--workspace packages/sdk' || '' }} # The Node 24 legs launch the bundle (test/_cli.ts reads CODING_SH_TEST_CLI): a launch from # source strips types on every start. The Node 22 legs keep launching src/cli/main.ts, so the # full suite covers both. Set here, not in an `env:` expression: that would give the Node 22 # legs an empty value, which test/_cli.ts refuses. - name: Build the CLI the tests launch if: matrix.node == '24.x' working-directory: packages/coding.sh run: | npm run build echo "CODING_SH_TEST_CLI=$PWD/dist/cli.mjs" >> "$GITHUB_ENV" # A fresh runner's first Chrome start waits on a cold disk and can take longer than the 20 s a # viewer test gives a start. This step takes that start under a 60 s limit of its own and # prints its time, then starts Chrome once more under the tests' own limits # (scripts/warm-chrome.ts). No Chrome fails here. The runner's first viewer page is cold as # well: on Windows it took a median of 1.6 s, and 5-37 s while Windows was reading the disk # for itself, against the 5 s a test gives a page. So the step also loads the viewer's list and # Explore once, at the tests' page size, under the same 60 s. - name: Start Chrome working-directory: packages/coding.sh timeout-minutes: 3 run: node scripts/warm-chrome.ts # The quick run of a pull request (no `ready` label) runs the test files its diff can reach # (scripts/test-select.ts): TEST_SELECT names the commit to diff against, the base the pull # request is merged with. A path without a rule runs every test file, and so does any error. # The full suite and a push leave TEST_SELECT empty and never select: FULL is 'false' on a # push too, so the event is read here as well. Each leg prints what it runs and why. A # narrowed run writes no run times, so the upload below finds no file. - name: Test working-directory: packages/coding.sh env: SHARD: ${{ matrix.shard }}/${{ matrix.of }} TEST_DURATIONS_OUT: ${{ runner.temp }}/test-durations.json TEST_SELECT: ${{ github.event_name == 'pull_request' && env.FULL == 'false' && 'HEAD^1' || '' }} # A file that fails is run again alone; one that then passes is a warning, not a failure. run: node scripts/test-ci.mjs # The SDK allows Node 18 and up, so it is checked on Node 22 here as well as on 24 in build. - name: SDK typecheck and test on Node 22 if: matrix.node == '22.x' && matrix.shard == 1 working-directory: packages/sdk run: npm run check # Each file's run time, for refreshing scripts/test-durations.json (see scripts/test-shards.ts). - uses: actions/upload-artifact@v6 if: matrix.node == '24.x' continue-on-error: true with: name: test-durations-linux-${{ matrix.shard }} path: ${{ runner.temp }}/test-durations.json retention-days: 14 if-no-files-found: ignore build: name: build and typecheck # The checks that need one run and no site, on Linux and Node 24: the typecheck, the SDK, the # bundle, the package contents, the secret scan and the hook budget. runs-on: ubuntu-latest timeout-minutes: 20 steps: - uses: actions/checkout@v5 - uses: actions/setup-node@v5 with: node-version: 24.x cache: npm - name: Git identity run: | git config --global user.name "coding.sh CI" git config --global user.email "ci@coding.sh.invalid" - name: Install run: npm ci - name: Typecheck working-directory: packages/coding.sh run: npx tsc -p tsconfig.json --noEmit # The SDK's golden test builds sessions with coding.sh's core, so a core change can break it. - name: SDK typecheck and test working-directory: packages/sdk run: npm run check - name: Build working-directory: packages/coding.sh run: npm run build - name: Smoke test the bundle working-directory: packages/coding.sh env: NO_UPDATE_NOTIFIER: "1" run: | node dist/cli.mjs --version node dist/cli.mjs --help > /dev/null - name: Package contents working-directory: packages/coding.sh run: npm pack --dry-run # The built packages hold nothing shaped like a secret. The site job scans the site's output. - name: Secret scan of the built output run: node scripts/gate.mjs --only secret-scan # The hook budget: a slow hook slows every agent. A shared runner measures main's own build # a few percent over the design budget (docs/design/dogfood.md), so the step fails on more # than 20% over it. It runs once: a miss fails the step. - name: Hook budget run: node scripts/dogfood/bench.mjs hook --check --tolerance 20 site: name: site build, check and perf budget # The site by itself, on Linux and Node 24: it was the longest part of the build job. The # site's build builds the CLI first (apps/site's prebuild), so this job has no build step of # its own. runs-on: ubuntu-latest timeout-minutes: 20 env: # The deploy builds the nav with the repository's star count (site.yml). A fixed count # here puts that nav through the build, the site tests and the perf budget. SITE_GITHUB_STARS: "1234" steps: - uses: actions/checkout@v5 - uses: actions/setup-node@v5 with: node-version: 24.x cache: npm - name: Git identity run: | git config --global user.name "coding.sh CI" git config --global user.email "ci@coding.sh.invalid" - name: Install run: npm ci - name: Site build working-directory: apps/site run: npm run build # A missing build fails the site tests here instead of skipping them. - name: Site check working-directory: apps/site env: CODING_SH_REQUIRE_DIST: "1" run: npm run check # The Site workflow runs this on main and stops the deploy when a page is over budget. # Running it here, on the same OS and Node as that workflow, fails the pull request instead. - name: Site perf budget working-directory: apps/site run: npm run perf:budget # The built site holds nothing shaped like a secret. - name: Secret scan of the built output run: node scripts/gate.mjs --only secret-scan gate-static: name: gate (static checks) # The checks that need no build: repo hygiene, lint, the lint and Site perf gate tests, actionlint and # generated files. runs-on: ubuntu-latest timeout-minutes: 15 steps: - uses: actions/checkout@v5 with: fetch-depth: 0 - uses: actions/setup-node@v5 with: node-version: 24.x cache: npm - name: Install run: npm ci - name: actionlint run: | bash <(curl -fsSL https://raw.githubusercontent.com/rhysd/actionlint/v1.7.7/scripts/download-actionlint.bash) 1.7.7 "$RUNNER_TEMP" echo "$RUNNER_TEMP" >> "$GITHUB_PATH" # The commits whose authors the gate checks: the pushed ones, or the pull request's own # when the full suite runs, so a commit main would reject is caught before the merge. - name: Static gate env: GATE_COMMIT_RANGE: ${{ github.event_name == 'push' && format('{0}..{1}', github.event.before, github.sha) || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ready')) && format('{0}..{1}', github.event.pull_request.base.sha, github.event.pull_request.head.sha) || '' }} run: node scripts/gate.mjs --static --strict stores: name: store plugins / Postgres 17 and ClickHouse 26.3 needs: changes if: ${{ !cancelled() && needs.changes.outputs.code != 'false' }} # GitHub-hosted, for the service containers; each plugin's tests run against a real server. # One job for both stores: they share a checkout and an install, and each has its own step. runs-on: ubuntu-latest timeout-minutes: 20 services: postgres: image: postgres:17 env: POSTGRES_PASSWORD: ci POSTGRES_DB: coding ports: ["5432:5432"] options: >- --health-cmd "pg_isready -U postgres" --health-interval 2s --health-timeout 5s --health-retries 30 clickhouse: image: clickhouse/clickhouse-server:26.3.62.3 env: CLICKHOUSE_USER: coding CLICKHOUSE_PASSWORD: ci ports: ["8123:8123"] options: >- --health-cmd "wget -qO- http://127.0.0.1:8123/ping" --health-interval 2s --health-timeout 5s --health-retries 30 steps: - uses: actions/checkout@v5 - uses: actions/setup-node@v5 with: node-version: 24.x cache: npm - name: Install id: install run: npm ci - name: Typecheck and test (real Postgres) working-directory: packages/store-postgres env: CODING_SH_TEST_PG_URL: postgres://postgres:ci@127.0.0.1:5432/coding run: npm run check # The next two steps run when an earlier one fails too, so one run shows every result. - name: Typecheck and test blob-s3 if: ${{ !cancelled() && steps.install.outcome == 'success' }} working-directory: packages/blob-s3 run: npm run check - name: Typecheck and test (real ClickHouse) if: ${{ !cancelled() && steps.install.outcome == 'success' }} working-directory: packages/store-clickhouse env: CODING_SH_TEST_CH_URL: clickhouse://coding:ci@127.0.0.1:8123/coding run: npm run check python-sdk: name: python sdk / python 3.9 and 3.13 needs: changes if: ${{ !cancelled() && needs.changes.outputs.code != 'false' }} # One job for the SDK's floor (pyproject requires-python) and the newest Python: each has its # own install and its own test step. Python 3.9 has no build for ubuntu-latest (26.04), so # the job runs on 24.04. runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - uses: actions/checkout@v5 # Both end up on PATH as python3.9 and python3.13. - uses: actions/setup-python@v6 with: python-version: | 3.9 3.13 # The golden test runs the core package's .ts sources on Node. The typing, golden, otel # and real-SDK tests need mypy and the otel, openai and anthropic extras, so CI installs # them and CODING_SH_SDK_REQUIRE_ALL=1 makes a missing one fail instead of skip. - uses: actions/setup-node@v5 with: node-version: 24.x - name: Install the extras and mypy id: install working-directory: packages/sdk-python run: | python3.9 -m pip install --quiet ".[otel,openai,anthropic]" mypy python3.13 -m pip install --quiet ".[otel,openai,anthropic]" mypy - name: Unit tests on Python 3.9 working-directory: packages/sdk-python env: CODING_SH_SDK_REQUIRE_ALL: "1" run: | python3.9 --version python3.9 -m unittest discover -s tests # Runs when the 3.9 tests fail too, so one run shows both results. - name: Unit tests on Python 3.13 if: ${{ !cancelled() && steps.install.outcome == 'success' }} working-directory: packages/sdk-python env: CODING_SH_SDK_REQUIRE_ALL: "1" run: | python3.13 --version python3.13 -m unittest discover -s tests runtime-floor: name: release smoke test and the packed CLI on the Node floor needs: changes if: ${{ !cancelled() && needs.changes.outputs.code != 'false' }} # Two checks of what would be published, in one job with one install. The pre-publish smoke # test runs on a build of this checkout, in a throwaway home, on a current Node. Then the # job builds and packs on that Node, installs the tarball and runs it on the oldest Node # packages/coding.sh/package.json allows (engines). runs-on: ubuntu-latest timeout-minutes: 20 steps: - uses: actions/checkout@v5 - uses: actions/setup-node@v5 with: node-version: 24.x cache: npm - name: Install id: install run: npm ci - name: Smoke test run: sh scripts/release/smoke.sh # The floor steps run when the smoke test fails too, so one run shows both results. - name: Build and pack id: pack if: ${{ !cancelled() && steps.install.outcome == 'success' }} run: | cd packages/coding.sh npm run build mv "$(npm pack --silent | tail -n 1)" "$RUNNER_TEMP/coding.sh.tgz" - uses: actions/setup-node@v5 if: ${{ !cancelled() && steps.pack.outcome == 'success' }} with: node-version: 22.16.0 - name: Run it on the floor if: ${{ !cancelled() && steps.pack.outcome == 'success' }} run: sh scripts/release/runtime-floor.sh "$RUNNER_TEMP/coding.sh.tgz" action: name: GitHub Action needs: changes if: ${{ !cancelled() && needs.changes.outputs.code != 'false' }} # The action in action.yml, installing this checkout's package: a passing and a failing # regression suite, the exit code, and the report. The comment is tested in # test/scripts/github-action.test.ts against a stub API. runs-on: ubuntu-latest timeout-minutes: 15 steps: - uses: actions/checkout@v5 - uses: actions/setup-node@v5 with: node-version: 22.x cache: npm - name: Build and pack run: | npm ci cd packages/coding.sh npm run build mv "$(npm pack --silent | tail -n 1)" "$RUNNER_TEMP/coding.sh.tgz" - name: Install only uses: ./ with: path: ${{ runner.temp }}/coding.sh.tgz command: "" - name: Write a passing and a failing suite working-directory: ${{ runner.temp }} run: | mkdir proj && cd proj echo 'process.stdin.on("data", (d) => process.stdout.write(JSON.parse(d)));' > echo.mjs printf '{"input":"a","expected":"a"}\n{"input":"b","expected":"b"}\n' > good.jsonl printf '{"input":"a","expected":"a"}\n{"input":"b","expected":"c"}\n' > bad.jsonl coding.sh eval init good --cases good.jsonl --command 'node echo.mjs' -k 1 > /dev/null coding.sh eval init bad --cases bad.jsonl --command 'node echo.mjs' -k 1 > /dev/null - name: Passing suite id: good uses: ./ with: path: ${{ runner.temp }}/coding.sh.tgz command: eval run good --yes working-directory: ${{ runner.temp }}/proj # The failure is the test. fail: "false" records the exit code without failing the step, # so a green run carries no annotation (continue-on-error would still get GitHub's # "Process completed with exit code 1"). - name: Failing suite id: bad uses: ./ with: path: ${{ runner.temp }}/coding.sh.tgz command: eval run bad --yes working-directory: ${{ runner.temp }}/proj fail: "false" # Each step has its own $GITHUB_STEP_SUMMARY, so this reads the action's report output. - name: Check the exit codes and the reports env: GOOD: ${{ steps.good.outputs.exit-code }} GOOD_REPORT: ${{ steps.good.outputs.report }} BAD: ${{ steps.bad.outputs.exit-code }} BAD_REPORT: ${{ steps.bad.outputs.report }} BAD_OUTCOME: ${{ steps.bad.outcome }} run: | test "$GOOD" = 0 test "$BAD" = 1 test "$BAD_OUTCOME" = success grep -q '^### Evals: the suite passed$' "$GOOD_REPORT" grep -q '^| bad | regression | 2 cases | 1/2 | .* | \*\*fail\*\*: b fails most trials |$' "$BAD_REPORT" grep -q '`coding.sh eval run bad --yes` exited 1' "$BAD_REPORT" image: name: container image / self-host smoke if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'ready') }} # GitHub-hosted: it builds and runs pull-request code in Docker. Nothing is pushed; the # release workflow publishes the image on version tags. runs-on: ubuntu-latest timeout-minutes: 30 steps: - uses: actions/checkout@v5 with: persist-credentials: false - uses: actions/setup-node@v5 with: node-version: 24.x cache: npm # The smoke test drives the server with the real CLI and hook from this checkout. - name: Install and build the CLI run: | npm ci npm run build -w coding.sh - name: Static checks of the deploy recipes and fixtures run: node --no-warnings --test deploy/test/deploy.test.mjs deploy/test/fixtures.test.mjs - uses: docker/setup-buildx-action@v4 # The build stage runs on the runner's platform and the runtime stage only copies files, # so arm64 builds without emulation. This proves both platforms build, as release.yml does. - name: Build linux/amd64 and linux/arm64 run: docker buildx build --platform linux/amd64,linux/arm64 . - name: Build the image to test run: docker buildx build --load -t coding-sh:smoke . - name: Image is small and runs as nonroot run: | docker image inspect coding-sh:smoke --format '{{.Size}} {{.Config.User}} {{json .Config.Healthcheck.Test}}' size=$(docker image inspect coding-sh:smoke --format '{{.Size}}') test "$size" -lt 300000000 test "$(docker image inspect coding-sh:smoke --format '{{.Config.User}}')" = "65532" - name: docker run with no configuration run: node deploy/test/smoke.mjs docker coding-sh:smoke - name: Compose, SQLite run: node deploy/test/smoke.mjs compose coding-sh:smoke deploy/compose/compose.yml - name: Compose, Postgres run: node deploy/test/smoke.mjs compose coding-sh:smoke deploy/compose/compose.postgres.yml - name: Compose, ClickHouse run: node deploy/test/smoke.mjs compose coding-sh:smoke deploy/compose/compose.clickhouse.yml windows: name: node ${{ matrix.node }} / windows-latest (${{ matrix.shard }}/4) if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'ready') }} runs-on: windows-latest timeout-minutes: 20 strategy: fail-fast: false # The whole suite takes about 20 minutes on hosted Windows: 4 shards on each Node. # Shard 1 also runs the build and the installer. matrix: node: [22.x, 24.x] shard: [1, 2, 3, 4] defaults: run: # bash (Git for Windows) keeps the quoted test glob and /dev/null the same as on Linux. shell: bash steps: # About 11 minutes after a hosted Windows runner boots, its Storage Service reads the disk at # 25-47 MB/s for 30 s or more, and the Volume Shadow Copy service starts. A viewer page that # first loaded during that read took 5-37 s, against the 5 s a test gives a page. The tests # do not use the service, so it is stopped before anything else, and disabled so that nothing # starts it again. The step fails if the service cannot be stopped. - name: Stop the Storage Service shell: pwsh run: | Stop-Service -Name StorSvc -Force Set-Service -Name StorSvc -StartupType Disabled Get-Service -Name StorSvc | ForEach-Object { "Storage Service: $($_.Status), start type $($_.StartType)" } # .gitattributes keeps LF in the checkout, whatever core.autocrlf says. - uses: actions/checkout@v5 - uses: actions/setup-node@v5 with: node-version: ${{ matrix.node }} cache: npm - name: Git identity run: | git config --global user.name "coding.sh CI" git config --global user.email "ci@coding.sh.invalid" # As on Linux: the CLI package's own dependencies, not the whole workspace's. - name: Install run: npm ci --workspace packages/coding.sh # As on Linux: the runner's first Chrome start and first viewer pages, outside the tests. # GitHub's Windows image ships Chrome, and the step fails if it is gone. - name: Start Chrome working-directory: packages/coding.sh timeout-minutes: 3 run: node scripts/warm-chrome.ts - name: Test working-directory: packages/coding.sh env: SHARD: ${{ matrix.shard }}/4 TEST_DURATIONS_OUT: ${{ runner.temp }}/test-durations.json # A file that fails is run again alone; one that then passes is a warning, not a failure. run: node scripts/test-ci.mjs # Each file's run time on this OS, for its own table (see scripts/test-shards.ts). - uses: actions/upload-artifact@v6 if: matrix.node == '24.x' continue-on-error: true with: name: test-durations-win32-${{ matrix.shard }} path: ${{ runner.temp }}/test-durations.json retention-days: 14 if-no-files-found: ignore - name: Build if: matrix.shard == 1 working-directory: packages/coding.sh run: npm run build - name: Smoke test the bundle if: matrix.shard == 1 working-directory: packages/coding.sh env: NO_UPDATE_NOTIFIER: "1" run: | node dist/cli.mjs --version node dist/cli.mjs --help > /dev/null # install.sh in Git Bash, into a throwaway npm prefix and home, then init + doctor. - name: Installer (Git Bash) if: matrix.shard == 1 env: CODING_SH_YES: "1" CODING_SH_NO_UI: "1" CODING_SH_NO_UPDATE_CHECK: "1" run: | T=$(cygpath -u "$RUNNER_TEMP") mkdir -p "$T/home/.claude" "$T/npm-global" export CODING_SH_TARBALL="$PWD/packages/coding.sh/$(cd packages/coding.sh && npm pack --silent | tail -n 1)" export npm_config_prefix="$(cygpath -w "$T/npm-global")" PATH="$T/npm-global:$PATH" export HOME="$T/home" USERPROFILE="$(cygpath -w "$T/home")" CODING_SH_HOME="$T/home/.coding.sh" sh install.sh coding.sh --version coding.sh init --agent claude-code --yes --no-mcp grep 'hook.mjs\\" claude-code' "$HOME/.claude/settings.json" coding.sh doctor --agent claude-code # The one slow job a push to main runs too (the comment at the top says why). real-capture: name: real capture / ${{ matrix.os }}${{ matrix.os == 'ubuntu-latest' && format(' ({0}/2)', matrix.part) || '' }} if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ready') }} # Always GitHub-hosted: it installs third-party agents from npm and runs them. Each script # installs into its own temp home, so the Linux steps split across two jobs of about equal time. runs-on: ${{ matrix.os }} timeout-minutes: 30 strategy: fail-fast: false matrix: include: - { os: ubuntu-latest, part: 1 } - { os: ubuntu-latest, part: 2 } - { os: windows-latest, part: 1 } defaults: run: shell: bash steps: - uses: actions/checkout@v5 - uses: actions/setup-node@v5 with: node-version: 24.x cache: npm - name: Install run: npm ci - name: Build working-directory: packages/coding.sh run: npm run build # Aider installs through uv (it fetches a Python aider supports). - uses: astral-sh/setup-uv@v7 # coding.sh from its tarball plus pinned OpenCode, Pi, Factory Droid and GitHub Copilot CLI # from npm, Goose from its GitHub release and Aider from PyPI, under a temp HOME, each # driven through one session against a local stub model. - name: Real agents against a stub model if: matrix.part == 1 run: node scripts/e2e/real-capture.mjs # Harness compare on real Claude Code and Codex sessions against a scripted stub model: the # one-item gate gives KEEP, KILL and REJECT from captured exposure, and failed runs split by own tests. - name: Harness compare with real agents if: runner.os == 'Linux' && matrix.part == 1 run: node scripts/e2e/compare-capture.mjs # compare --repair with real Claude Code and Codex: a failed first attempt is told "Verification # failed." with the failing test's name in its own resumed session, fixes it in round 1, never # sees the hidden tests, and the report shows the Seq@k curve and the rounds that changed nothing. - name: Repair rounds with real agents if: runner.os == 'Linux' && matrix.part == 1 run: node scripts/e2e/repair-capture.mjs # eval trigger with real Claude Code and Codex: a stub router loads the skill whose installed # description names a word of the prompt. Checks exact precision/recall counts from captured # sessions, the sibling route, a description A/B with its flips, and --min-f1. - name: Skill trigger eval with real agents if: runner.os == 'Linux' && matrix.part == 2 run: node scripts/e2e/trigger-capture.mjs # eval init then eval run with real Claude Code, one suite per source: a repo task from a # failed session (pass@2 100%, pass^2 0%, flaky, the failed trial's session), drafted cases # graded exactly (a regression suite that exits 1), and drafted skill trigger prompts. - name: eval init and eval run with real Claude Code if: runner.os == 'Linux' && matrix.part == 1 run: node scripts/e2e/eval-init-capture.mjs # exp ab on real Claude Code sessions that claim done without testing: mines the decision # points, replays the next turn with the committed and the edited CLAUDE.md, and checks the # verdict, the noise floor, the cost cap, the cache and --watch. - name: exp ab on real Claude Code sessions if: runner.os == 'Linux' && matrix.part == 1 run: node scripts/e2e/exp-ab-capture.mjs # Experiment sessions and exp run --watch on a command agent and an LLM judge answered by # the stub: the delta vs the previous run, a rubric-only edit regrades with 0 agent calls, # a save mid-run aborts it and stores nothing, a broken spec keeps watching, Ctrl-C exits 0. - name: Experiment sessions and exp run --watch if: runner.os == 'Linux' && matrix.part == 2 run: node scripts/e2e/exp-session-watch.mjs # exp run with a 2x2 matrix spec on a command agent answered by the stub: --plan stores # nothing, the cells run interleaved and are stored one experiment each, the p-values are # Holm-adjusted, exp compare on two cells matches, and a vary violation exits 2 before any call. - name: Experiment matrix if: runner.os == 'Linux' && matrix.part == 2 run: node scripts/e2e/exp-matrix.mjs # The refine loop on synthetic sessions, with the proposer, the next turn and the judge # answered by the stub: one edit is kept and applied, a later edit that drops it wins its own # decision points but is rejected by the guard, and a rollback restores CLAUDE.md byte for byte. - name: Refine loop keep and guard reject if: runner.os == 'Linux' && matrix.part == 2 run: node scripts/e2e/refine-loop.mjs # refine run --suite with real Claude Code against the stub, which bills input tokens by # request size: the proposer's cut of a bloated CLAUDE.md is kept on cost with the pass rate # held, a cheaper cut that drops a line the tasks need is rejected, and a cut of a safety # line is rejected before any run. - name: Refine climb on an eval suite with real Claude Code if: runner.os == 'Linux' && matrix.part == 2 run: node scripts/e2e/refine-suite-capture.mjs # The model API proxy with real Claude Code and Codex: init --proxy routes both through it to a # stub model; checks streaming, the agent's own key reaching the API, calls linked into the # hook sessions, show/diff-calls/exp replay, no key stored, and a byte-for-byte uninstall. - name: Model API proxy with real agents if: runner.os == 'Linux' && matrix.part == 2 run: node scripts/e2e/proxy-capture.mjs # Fork from step N with real Claude Code against the stub: a session whose CLAUDE.md says npm # test fails its test step; replay --fork with a fixed CLAUDE.md sends that step's stored call # again and takes another action, --run reruns the agent from the step in a rolled-back copy # and its verifier passes, and --save-case makes a case eval run fails until CLAUDE.md is fixed. - name: Fork from a replay step with real Claude Code if: runner.os == 'Linux' && matrix.part == 2 run: node scripts/e2e/fork-capture.mjs # What the proxy adds to a streamed call, direct and proxied interleaved; fails over budget # (2 ms per chunk, 5 ms to first byte). - name: Proxy latency budget if: runner.os == 'Linux' && matrix.part == 2 run: node scripts/dogfood/bench.mjs proxy --check cli-suite: name: cli-suite (dev tasks, ${{ matrix.part }}/2) if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'ready') }} # The CLI effectiveness suite's own health, on the committed dev tasks only: held-out tasks # live outside the repo, so this never passes --split all or heldout. It needs no API key and # starts no agent. Always GitHub-hosted. Every task builds its own fixture home twice, which # is most of the time, so the tasks split across two jobs. runs-on: ubuntu-latest timeout-minutes: 15 strategy: fail-fast: false matrix: part: [1, 2] steps: - uses: actions/checkout@v5 - uses: actions/setup-node@v5 with: node-version: 24.x cache: npm # The fixture world holds git repos with commits. - name: Git identity run: | git config --global user.name "coding.sh CI" git config --global user.email "ci@coding.sh.invalid" - name: Install run: npm ci - name: Build working-directory: packages/coding.sh run: npm run build # For every dev task: the fixture home builds, the checks fail before any work, the scripted # reference passes them, and the instruction does not leak the answer. Exits 1 on any # invalid task. - name: Validate the dev tasks run: node scripts/dogfood/cli-suite.mjs validate --split dev --shard ${{ matrix.part }}/2 --json # The CLI guidelines over the same fixture home. The score is printed, not gated; the step # fails when the fixture home does not build. - name: CLI guidelines on the fixture home if: matrix.part == 1 run: node scripts/dogfood/cli-guidelines.mjs # Site perf (site-perf.yml: Lighthouse A/B, budgets) on a pull request, as a job of this # workflow so the full suite needs its verdict. The full suite always runs it: the measured # pages also read CHANGELOG.md, the docs and the CLI's registry, so no path list proves a diff # cannot move them. Without the `ready` label it runs when scripts/ci-changes.mjs says the # diff touches the site's measured paths. A push to main is measured by site-perf.yml's own # push trigger, on the desk, so this job is skipped there. site-perf: name: site perf needs: changes if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.changes.outputs.site_perf != 'false' }} uses: ./.github/workflows/site-perf.yml # The one check main requires. Its name is "full suite" only when the full suite ran, so a pull # request without the `ready` label reports no check of that name and cannot merge: GitHub # lists the skipped job under the name expression itself, unevaluated. A skipped job counts as # passed for a required check, so two things keep this one honest. The condition below reads # the event, not another job's output: with the label this job always runs, and it fails # unless every other job ended in success (a skipped or cancelled job fails it). And it is # `always()`, not `!cancelled()`: a cancelled run ends with this check failed, never skipped. # A manual run does not measure Site perf here (MAY_SKIP below), so it reports under another # name: a manual run on a pull request's branch cannot stand in for the required check. A push # to main does not run this job: it runs the quick jobs and the real captures, not the suite. full-suite: name: ${{ github.event_name == 'workflow_dispatch' && 'full suite without site perf' || contains(github.event.pull_request.labels.*.name, 'ready') && 'full suite' || 'full suite (not run, add the ready label)' }} needs: [changes, test, build, site, gate-static, stores, python-sdk, runtime-floor, action, image, windows, real-capture, cli-suite, site-perf] if: ${{ always() && (github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'ready')) }} runs-on: ubuntu-latest timeout-minutes: 5 steps: - uses: actions/checkout@v5 - name: Every job passed env: NEEDS: ${{ toJSON(needs) }} # The one job that may end skipped, and only in a manual run, which has no pull request # to compare with. This reads the event, not a job's output. MAY_SKIP: ${{ github.event_name == 'workflow_dispatch' && 'site-perf' || '' }} run: node scripts/ci-full-suite.mjs