#!/usr/bin/env bash # preflight.sh — run everything CI runs, locally, before pushing. # # The whole point: failures get found HERE, not in a GitHub CI round-trip. # Mirrors .github/workflows/ci.yml and security.yml as of 2026-08-15, with # two deliberate strictness additions (go vet; svelte-check) that CI lacks. # # ./preflight.sh full run (~13-19 min; long pole is # `go test -race ./...` — CI's own # comment on that step measured 581s on # a loaded machine, #329) # PREFLIGHT_FAST=1 ./preflight.sh skip `npm ci` when node_modules exists # PREFLIGHT_COVERAGE=1 ./preflight.sh also run the CI coverage pass (re-runs suite) # PREFLIGHT_SCAN_PROCS=N ./preflight.sh CPU cap for gosec/govulncheck/ # go-licenses while they run in the # background alongside the test suite # (default 2 — see the note above the # gosec docker run below for why this # exists) # # Requires: go (toolchain per go.mod), node (20 preferred — warns on drift), # docker (for CI-identical gosec; skipped loudly if absent), jq. # # CPU note: unlike CI, which runs lint/test/build/security as five # separate jobs on five separate VMs (zero contention by construction), # every stage here shares one machine's cores. `go test -race ./...` is # deliberately left at its default (uncapped) parallelism, matching # ci.yml's own invocation exactly — it is the long pole and the # correctness-critical step. The background gosec/govulncheck/go-licenses # scans are capped via PREFLIGHT_SCAN_PROCS so they can make progress # concurrently with the test suite (preserving the wall-clock win of not # running them sequentially) without fighting it for every core. set -euo pipefail cd "$(dirname "$0")" LOGDIR=.preflight mkdir -p "$LOGDIR" FAILED=0 START=$(date +%s) PREFLIGHT_SCAN_PROCS="${PREFLIGHT_SCAN_PROCS:-2}" say() { printf '\n\033[1m== %s ==\033[0m\n' "$*"; } pass() { printf '\033[32mPASS\033[0m %s (%ss)\n' "$1" "$(( $(date +%s) - STAGE_START ))"; } fail() { printf '\033[31mFAIL\033[0m %s — see %s\n' "$1" "$2"; FAILED=1; } # shellcheck source=scripts/preflight-govulncheck-verdict.sh source scripts/preflight-govulncheck-verdict.sh stage() { STAGE_START=$(date +%s); say "$1"; } # ---- 1. gofmt — CI parity: UNFILTERED repo root (stricter than lint.sh) ---- stage "gofmt (unfiltered, CI parity)" UNFORMATTED=$(gofmt -l .) if [ -n "$UNFORMATTED" ]; then echo "$UNFORMATTED" echo "run: gofmt -w ." exit 1 fi pass gofmt # ---- 2. outbound HTTP policy (CI parity: ci.yml's lint job) ---- # # This was missing while CI ran it, so a violation cost a full CI # round-trip to discover -- exactly what this script exists to prevent. # It is cheap and it guards ADR-022, so it runs before the long stages # rather than alongside them. stage "outbound HTTP policy (netguardcheck)" go run ./internal/cmd/netguardcheck ./... 2>&1 | tee "$LOGDIR/netguard.log" pass netguard # ---- 3. go vet (not in CI's lint job — deliberate extra rigor) ---- stage "go vet" go vet ./... 2>&1 | tee "$LOGDIR/vet.log" pass vet # ---- background security stages (don't need the UI build) ---- say "launching parallel security scans (gosec / govulncheck / licenses)" ( if command -v docker >/dev/null 2>&1; then # gosec's own image/digest is pinned deliberately (see below) and NOT # what makes this slow. `--rm` gives every invocation a brand-new # container with an empty $GOPATH/pkg/mod and empty build cache, so # without the two named volumes below, gosec re-downloads and # re-compiles this module's ENTIRE dependency graph from scratch on # every single preflight run, fighting go test -race for CPU the # whole time it does. Named volumes persist across --rm (only the # container is ephemeral, not the volume), so the first run after # this change is still cold; every run after that reuses them. This # is a separate, container-local cache -- never the host's own # GOMODCACHE/GOCACHE -- specifically to avoid a root-in-container # write leaving host-owned cache files an unprivileged host `go` # invocation can't later read or overwrite. docker run --rm -v "$PWD":/src -w /src \ -v brokoli-preflight-gosec-gomodcache:/go/pkg/mod \ -v brokoli-preflight-gosec-gocache:/gocache \ -e GOCACHE=/gocache \ -e GOMAXPROCS="$PREFLIGHT_SCAN_PROCS" --cpus="$PREFLIGHT_SCAN_PROCS" \ ghcr.io/securego/gosec@sha256:4342ad119a7c69f3f4e4ce78d81ba183dc774a70a7a4c6eeb15fe9e511f214f0 \ -no-fail -fmt=json -out=/src/$LOGDIR/gosec.json -exclude-dir=.preflight ./... >/dev/null 2>&1 # Assert the scan happened before trusting its verdict: a baseline # comparison treats "0 findings" as the best possible outcome, so a # gosec run that inspected nothing passes while suggesting the # baseline be lowered to 0. Mirrors ci parity with security.yml, and # the reason both now check is scripts/check-licenses.sh. files=$(jq -r '.Stats.files // 0' "$LOGDIR/gosec.json" 2>/dev/null || echo 0) errs=$(jq -r '(."Golang errors" // {}) | length' "$LOGDIR/gosec.json" 2>/dev/null || echo 0) if [ "$files" -lt 1 ]; then echo "gosec scanned $files files — the scan inspected nothing" > "$LOGDIR/gosec.verdict"; exit 1 fi if [ "$errs" -gt 0 ]; then echo "gosec hit $errs Go build error(s) — partial scan, verdict not comparable to the baseline" > "$LOGDIR/gosec.verdict"; exit 1 fi findings=$(jq '[.Issues[]] | length' "$LOGDIR/gosec.json") baseline=$(jq -r '.issues' security/gosec-baseline.json) echo "gosec findings: $findings (baseline: $baseline) across $files files" > "$LOGDIR/gosec.verdict" if [ "$findings" -gt "$baseline" ]; then exit 1; fi if [ "$findings" -lt "$baseline" ]; then echo "note: baseline can be reduced to $findings" >> "$LOGDIR/gosec.verdict"; fi else echo "SKIP: docker not available — CI WILL run gosec; findings there may differ" > "$LOGDIR/gosec.verdict" fi ) > "$LOGDIR/gosec.log" 2>&1 & GOSEC_PID=$! # Pin GOTOOLCHAIN to the module's toolchain: `go run pkg@version` is # module-less, so it only upgrades to x/vuln's own minimum (1.25), which # then can't analyze packages that use newer language features. CI never # hits this because setup-go installs the go.mod version outright. MOD_TOOLCHAIN=$(awk '/^toolchain /{print $2}' go.mod) # Pinned exact versions, not @latest: a tool version bump on any given day # would otherwise force a fresh, uncached compile (defeating the point of # a warm build cache) and make results non-deterministic run to run. # `go install` once into the host's own module/build cache (already 2.9G/ # 2.7G warm on a machine that's built this module before) rather than # `go run` on every invocation -- `go run` would keep re-resolving the # module each time. Bump these deliberately, the same way the gosec # digest above is a deliberate, reviewed bump. GOVULNCHECK_VERSION=v1.7.0 # The runner prefers a matching installed binary (including a customized # GOBIN) and only installs when the pinned version is absent. ( export GOVULNCHECK_VERSION MOD_TOOLCHAIN GOMAXPROCS="$PREFLIGHT_SCAN_PROCS" bash scripts/run-preflight-govulncheck.sh ) > "$LOGDIR/govulncheck.log" 2>&1 & VULN_PID=$! ( # scripts/check-licenses.sh, not go-licenses: go-licenses cannot load a # Go 1.24+ stdlib (every package trips "does not have module info"), # exits fatally having printed nothing, and the old `|| true` + grep # over that empty output passed unconditionally. See the script header. GOMAXPROCS="$PREFLIGHT_SCAN_PROCS" bash scripts/check-licenses.sh > "$LOGDIR/licenses.txt" bash scripts/check-licenses_test.sh ) > "$LOGDIR/licenses.log" 2>&1 & LIC_PID=$! # ---- 3. node version guard (CI pins 20) ---- stage "node version" NODE_MAJOR=$(node -v | sed 's/^v//' | cut -d. -f1) if [ "$NODE_MAJOR" != "20" ]; then if [ -s "$HOME/.nvm/nvm.sh" ]; then # Sourcing nvm.sh mutates PATH to nvm's default — if we then fail to # land on 20, RESTORE the original PATH: a half-switched node broke # npm's native optional deps the first time this script ran. PREFLIGHT_OLD_PATH=$PATH # shellcheck disable=SC1091 . "$HOME/.nvm/nvm.sh" if nvm use 20 >/dev/null 2>&1 || { nvm install 20 >/dev/null 2>&1 && nvm use 20 >/dev/null 2>&1; }; then echo "using node $(node -v) via nvm (CI parity)" else PATH=$PREFLIGHT_OLD_PATH echo "WARN: node $NODE_MAJOR (CI pins 20); nvm couldn't provide 20 — build may differ from CI" fi else echo "WARN: node v$NODE_MAJOR, CI pins v20 — build may differ from CI" fi else echo "node $(node -v) matches CI" fi pass node-guard # ---- 3b. go toolchain parity with the workflows ---- # The workflows pin setup-go to an exact version so Go does not download # a SECOND toolchain per job on top of the one setup-go installed (which # it does whenever that version differs from go.mod's toolchain # directive). Same class of drift the node guard above catches, so the # same treatment: notice it here rather than in a CI bill. stage "go toolchain parity (advisory)" mod_toolchain=$(awk '/^toolchain /{print substr($2,3)}' go.mod) wf_toolchain=$(grep -ho 'go-version: "[^"]*"' .github/workflows/ci.yml | head -1 | sed 's/.*"\(.*\)"/\1/') if [ -n "$mod_toolchain" ] && [ "$mod_toolchain" != "$wf_toolchain" ]; then printf '\033[33mWARN\033[0m go.mod toolchain (%s) != ci.yml go-version (%s) -- CI will download a second toolchain per job\n' \ "$mod_toolchain" "$wf_toolchain" else echo "go toolchain $mod_toolchain matches the workflows" fi pass go-toolchain-guard # ---- 4. UI build (go:embed prerequisite) ---- stage "UI build" pushd ui >/dev/null if [ "${PREFLIGHT_FAST:-}" = "1" ] && [ -d node_modules ]; then echo "PREFLIGHT_FAST: skipping npm ci" else npm ci --silent fi popd >/dev/null # The UI check chain + prettier need only node_modules + source, not the # Vite build output. They have no ordering dependency on anything else in # this script, so they run in the background (started right after `npm # ci`, alongside the Vite build and the security scans) instead of # blocking serially before the long test run, and are collected in the # final wait block the same way gosec/govulncheck/go-licenses are. # # This block used to run svelte-check and compare its error count against # ui/svelte-check-baseline.json. The UI is React (brokoli#645) -- there is # no svelte-check, no ui/src, and no baseline file. Two consequences, both # live until now: # # `npm run check ... || true` threw the real result away, so a UI # typecheck or unit-test failure could not fail preflight. The only # thing gating was an error count scraped from output that no longer # contains it. # # `jq -r '.errors' svelte-check-baseline.json` read a file that does not # exist, which failed the stage on every run no matter what the UI did. # # So the stage was simultaneously vacuous and always-red. It now gates on # the exit status of ui/package.json's own "check" script, which is the # thing that actually means the UI is sound (typecheck, unit tests, # colour tokens, build, import boundaries). ( cd ui if ! npm run check > "../$LOGDIR/ui-check.log" 2>&1; then echo "npm run check failed -- see ui-check.log" > "../$LOGDIR/ui-checks.verdict" exit 1 fi echo "npm run check: clean (typecheck, tests, colors, build, boundaries)" > "../$LOGDIR/ui-checks.verdict" # Prettier gate scoped to CHANGED files only: the tree has never been # fully prettier-formatted and CI has no format gate — wholesale # reformatting belongs in its own dedicated PR, not as preflight fallout. # Sources live under ui/apps/*/src and ui/packages/*/src. CHANGED_UI=$( (git -C .. diff --name-only origin/main...HEAD -- ui; git -C .. diff --name-only -- ui; git -C .. ls-files --others --exclude-standard -- ui) | sort -u | grep -E '^ui/(apps|packages)/[^/]+/src/.*\.(ts|tsx|css)$' | sed 's|^ui/||' || true) if [ -n "$CHANGED_UI" ]; then # shellcheck disable=SC2086 npx prettier --check $CHANGED_UI > "../$LOGDIR/ui-format.log" 2>&1 || exit 1 echo "prettier: $(echo "$CHANGED_UI" | wc -l) changed file(s) clean" >> "../$LOGDIR/ui-checks.verdict" else echo "prettier: no changed UI files" >> "../$LOGDIR/ui-checks.verdict" fi ) > "$LOGDIR/ui-checks.log" 2>&1 & UI_CHECK_PID=$! pushd ui >/dev/null npm run build > "../$LOGDIR/ui-build.log" 2>&1 || { popd >/dev/null; fail ui-build "$LOGDIR/ui-build.log"; exit 1; } popd >/dev/null pass ui-build # ---- 5. tests — CI parity: -v -race ---- # Live-database tests skip unless these are set, and skipping quietly is how # the differential tests that gate streaming and pushdown correctness went a # long time without running anywhere but one machine. Provision them the way # the EE preflight provisions Redis: use what the operator already has, # otherwise bring up the compose file, otherwise say plainly that the run is # weaker than it looks. if [ -z "${BROKOLI_TEST_POSTGRES_URL:-}" ] || [ -z "${BROKOLI_TEST_MYSQL_URL:-}" ] || [ -z "${BROKOLI_TEST_CLICKHOUSE_URL:-}" ]; then if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then say "starting test databases (docker-compose.test.yml)" docker compose -f docker-compose.test.yml up -d >/dev/null 2>&1 || true for _ in $(seq 1 40); do healthy=$(docker compose -f docker-compose.test.yml ps --format '{{.Health}}' 2>/dev/null | grep -c healthy || true) [ "$healthy" -ge 3 ] && break sleep 3 done : "${BROKOLI_TEST_POSTGRES_URL:=postgres://brokoli:brokoli@localhost:55532/brokoli_test?sslmode=disable}" : "${BROKOLI_TEST_MYSQL_URL:=mysql://brokoli:br@k:li/pw#1@tcp(localhost:55533)/brokoli_test}" : "${BROKOLI_TEST_MYSQL_ROOT_URL:=mysql://root:rootpw@tcp(localhost:55533)/brokoli_test}" : "${BROKOLI_TEST_CLICKHOUSE_URL:=clickhouse://brokoli:brokoli@localhost:55534/brokoli_test}" # The azure_blob tests opt into loopback per test, so unlike the S3 # ones they need no process-wide policy override and can run here too. : "${BROKOLI_TEST_AZURE_BLOB_ENDPOINT:=http://127.0.0.1:55537/devstoreaccount1}" export BROKOLI_TEST_POSTGRES_URL BROKOLI_TEST_MYSQL_URL BROKOLI_TEST_MYSQL_ROOT_URL BROKOLI_TEST_CLICKHOUSE_URL BROKOLI_TEST_AZURE_BLOB_ENDPOINT else say "WARNING: no docker compose — live-database tests will SKIP, and they are" say " the ones that check what a real server does with a literal." fi fi # -timeout 15m, not Go's 10m default: a package measured 581s under -race # on a loaded machine (Tnsor-Labs/brokoli#329), so the default panics # under load and blames whichever test held the baton. # # The engine package is excluded here and run below with its own budget, # because that is what .github/workflows/ci.yml does: CI runs # `$(go list ./... | grep -v '/engine$')` at 15m and gives engine a # separate job at -timeout 25m. This script ran everything at 15m while # its comment claimed CI parity, so preflight failed on a package CI was # giving 10 more minutes -- measured at 1037s (17m17s) on an idle # machine, which is over the old ceiling before any load at all. The # panic then named whichever test held the baton, which is the exact # misdiagnosis the 15m ceiling was introduced to stop. stage "go test -race ./... minus engine" # Full output to a log, not just a tail: piping straight into `tail` # throws away the "--- FAIL" lines and the failing package name, which # left a failure showing as a bare "FAIL" with nothing to act on. if ! go test -race -timeout 15m $(go list ./... | grep -v '/engine$') > "$LOGDIR/tests.log" 2>&1; then grep -E "^(FAIL|--- FAIL|panic:)" "$LOGDIR/tests.log" | head -20 fail tests "$LOGDIR/tests.log" exit 1 fi tail -3 "$LOGDIR/tests.log" pass tests # ---- 5a1. the engine package, on CI's budget ---- # Matches the dedicated "Test (engine)" job: -timeout 25m inside a 30m # job limit. It is the most expensive package in the repo and the only # one that has ever needed this. stage "go test -race ./engine/ (the long pole)" if ! go test -race -timeout 25m ./engine/ > "$LOGDIR/tests-engine.log" 2>&1; then grep -E "^(FAIL|--- FAIL|panic:)" "$LOGDIR/tests-engine.log" | head -20 fail tests-engine "$LOGDIR/tests-engine.log" exit 1 fi tail -3 "$LOGDIR/tests-engine.log" pass tests-engine # ---- 5a2. code-node legacy-path leg (ADR-029 dual mode) ---- # The pool is the default; the legacy spawn path stays green until its # removal one release later. Matches the CI leg. stage "engine code-node tests, legacy path (BROKOLI_CODE_POOL=0)" if ! env BROKOLI_CODE_POOL=0 go test -timeout 10m -run 'TestCodeNode|TestLazyRows|TestPipeline_LazyRows|TestPipeline_EmitZeroRows|TestPipeline_Streamed|TestInstanceWorker|TestExpansion' ./engine/ > "$LOGDIR/tests-legacy-code.log" 2>&1; then grep -E "^(FAIL|--- FAIL|panic:)" "$LOGDIR/tests-legacy-code.log" | head -10 fail tests-legacy-code "$LOGDIR/tests-legacy-code.log" exit 1 fi pass tests-legacy-code # ---- 5b. code-node wrapper contract tests (pytest, ADR-029) ---- # Real Python tested as Python, matching CI. Skipped only when pytest # is genuinely unavailable — the wrapper tests matter exactly as much # as the Go tests that pin the same contract from the other side. stage "pytest pkg/codeexec/pywrapper/tests" if python3 -m pytest --version >/dev/null 2>&1; then if ! python3 -m pytest pkg/codeexec/pywrapper/tests/ -q > "$LOGDIR/pytest.log" 2>&1; then tail -20 "$LOGDIR/pytest.log" fail pytest "$LOGDIR/pytest.log" exit 1 fi tail -1 "$LOGDIR/pytest.log" pass pytest else echo "pytest not installed for python3 — skipping (CI still runs it)" fi # ---- 5c. code-node JavaScript wrapper contract tests (ADR-030) ---- stage "node --test pkg/codeexec/jswrapper/tests" if ! node --test pkg/codeexec/jswrapper/tests/*.test.mjs > "$LOGDIR/jswrapper-node.log" 2>&1; then tail -20 "$LOGDIR/jswrapper-node.log" fail jswrapper-node "$LOGDIR/jswrapper-node.log" exit 1 fi tail -3 "$LOGDIR/jswrapper-node.log" pass jswrapper-node # ---- 6. optional coverage pass (CI runs it; adds minutes; opt-in) ---- if [ "${PREFLIGHT_COVERAGE:-}" = "1" ]; then stage "coverage (CI parity)" go test -coverprofile="$LOGDIR/coverage.out" ./... > "$LOGDIR/coverage.log" 2>&1 go tool cover -func="$LOGDIR/coverage.out" | tail -1 pass coverage fi # ---- 7. build + smoke — CI parity ---- stage "CGO_ENABLED=0 build + --help smoke" CGO_ENABLED=0 go build -tags minicore_disabled -o "$LOGDIR/brokoli-preflight" . "$LOGDIR/brokoli-preflight" --help >/dev/null pass build-smoke # ---- collect parallel stages ---- say "waiting on parallel security scans + UI checks" wait "$GOSEC_PID" || fail gosec "$LOGDIR/gosec.log" cat "$LOGDIR/gosec.verdict" 2>/dev/null || true set +e wait "$VULN_PID" vuln_status=$? set -e if ! govulncheck_verdict "$vuln_status" "$LOGDIR/govulncheck.log"; then FAILED=1 fi wait "$LIC_PID" || fail licenses "$LOGDIR/licenses.log" wait "$UI_CHECK_PID" || fail ui-checks "$LOGDIR/ui-checks.log" cat "$LOGDIR/ui-checks.verdict" 2>/dev/null || true [ "$FAILED" = 0 ] && { [ -f "$LOGDIR/gosec.verdict" ] && grep -q SKIP "$LOGDIR/gosec.verdict" || true; } TOTAL=$(( $(date +%s) - START )) if [ "$FAILED" = 1 ]; then printf '\n\033[31mPREFLIGHT FAILED\033[0m after %ss — fix locally, do not push.\n' "$TOTAL" exit 1 fi printf '\n\033[32mPREFLIGHT GREEN\033[0m in %ss — safe to push.\n' "$TOTAL"