name: Backend Services CI/CD on: # Nightly run against main. The push/pull_request triggers below are # path-filtered, so a stretch of docs- or frontend-only merges leaves the # backend gauntlet unexercised — that let seven gates rot silently on main # (lock freshness, repo-root RLS suite, ingestion suite, admin tests, ...) # until the next backend-touching PR lit them all up at once (#2566). # A scheduled run converts silent drift into a red run within 24h. schedule: - cron: '0 5 * * *' workflow_dispatch: push: branches: [main, develop] paths: # frontend/src/lib/public-openapi.ts and frontend/package-lock.json are # included because repo-root-suite runs the published OpenAPI surface # contract, which shells out to node and reads that spec (#3248). A # spec-only change must trigger the gate that compares it. The rest of # frontend/** stays out so frontend-only merges do not wait on this job. - 'frontend/src/lib/public-openapi.ts' - 'frontend/package-lock.json' - 'alembic/**' - 'server/**' - 'services/**' - 'shared/**' - 'kernel/**' - 'plugins/fsma/**' - 'Dockerfile' - 'railway.toml' - 'requirements.in' - 'requirements.lock' - 'scripts/**' - 'tests/**' - 'docs/security/route-security-manifest.json' - 'docs/security/release-evidence/**' - '.github/workflows/backend-ci.yml' - '.github/workflows/deploy.yml' pull_request: branches: [main] paths: # Same frontend spec inclusion as push above (#3248 OpenAPI contract). - 'frontend/src/lib/public-openapi.ts' - 'frontend/package-lock.json' - 'alembic/**' - 'server/**' - 'services/**' - 'shared/**' - 'kernel/**' - 'plugins/fsma/**' - 'Dockerfile' - 'railway.toml' - 'requirements.in' - 'requirements.lock' - 'scripts/**' - 'tests/**' - 'docs/security/route-security-manifest.json' - 'docs/security/release-evidence/**' - '.github/workflows/backend-ci.yml' - '.github/workflows/deploy.yml' # Supersede older PR runs only. Give push, schedule, and manual runs distinct # groups so no pending main check can be replaced before deploy.yml sees it. concurrency: group: backend-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: PYTHON_VERSION: '3.11' permissions: contents: read jobs: # ── Dependency Lock Freshness ──────────────────────────── dependency-lock: name: Dependency Lock Freshness runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: # Generate, freshness-validate, install, and runtime must all agree on # one interpreter so the --require-hashes lock stays reproducible. # Dockerfile (python:3.11-slim) and PYTHON_VERSION below both use 3.11, # so this gate runs on 3.11 too — was 3.12, which let the 3.12-resolved # lock be installed on 3.11 everywhere else (#2527). requirements.lock # must be regenerated under 3.11 (header reads "with Python 3.11"); the # freshness diff below makes any leftover 3.12 resolution a hard fail. python-version: ${{ env.PYTHON_VERSION }} - name: Verify requirements.lock freshness run: | python -m pip install "pip==24.3.1" pip-tools==7.5.2 CUSTOM_COMPILE_COMMAND="pip-compile --allow-unsafe --generate-hashes --output-file=requirements.lock requirements.in" \ pip-compile --allow-unsafe --generate-hashes --output-file=requirements.lock requirements.in git diff --exit-code -- requirements.lock # ── Test Matrix ────────────────────────────────────────── test: name: Test ${{ matrix.service }} needs: dependency-lock runs-on: ubuntu-latest # Ingestion's ~5,000 coverage tests reached 98% before the old 10-minute # job deadline, which also includes service startup and dependency install. # By 2026-10-14 the suite had 6,139 tests: the test step took 10m23s on # one runner and only reached 74% at 15 minutes on slower runners, so # ingestion gets 25 minutes. The deadline still stops a hung test. timeout-minutes: ${{ matrix.service == 'ingestion' && 25 || 10 }} strategy: fail-fast: false matrix: service: - admin - compliance - graph - ingestion - nlp - scheduler - shared services: postgres: image: postgres:16 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres ports: - 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 redis: image: redis:7-alpine ports: - 6379:6379 options: >- --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 neo4j: image: ${{ matrix.service == 'graph' && 'neo4j:5.24-community' || '' }} env: NEO4J_AUTH: neo4j/test-password-for-ci ports: - 7687:7687 - 7474:7474 options: >- --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:7474 || exit 1" --health-interval 10s --health-timeout 5s --health-retries 10 localstack: image: localstack/localstack:3 env: SERVICES: s3 ports: - 4566:4566 options: >- --health-cmd "curl -f http://localhost:4566/_localstack/health" --health-interval 10s --health-timeout 5s --health-retries 10 steps: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Create additional databases env: PGPASSWORD: postgres run: | psql -h localhost -U postgres -c "CREATE DATABASE regengine_admin;" psql -h localhost -U postgres -c "CREATE DATABASE test_db;" - name: Cache pip packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 with: path: ~/.cache/pip # Key off the root requirements.lock — the single source of truth. # Per-service requirements.txt files are DEPRECATED (see #1429) # and match root, so caching on root invalidates correctly. key: pip-${{ hashFiles('requirements.lock') }} restore-keys: | pip- - name: Install dependencies run: | # Install from the root requirements.lock (pip-compile output; see # requirements.in for the human-edited top-level spec). --require-hashes # guarantees reproducible builds and rejects CVE-shifted artifacts. # Per-service stubs are DEPRECATED and will be removed with # docker-compose (#1429). pip install --require-hashes -r requirements.lock - name: Run tests with coverage env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db COMPLIANCE_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db ADMIN_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/regengine_admin REDIS_URL: redis://localhost:6379/0 JWT_SECRET: test-secret-key-for-ci SUPABASE_URL: http://localhost:54321 SUPABASE_KEY: test-key-for-ci NEO4J_URI: bolt://localhost:7687 REGENGINE_REQUIRE_NEO4J: ${{ matrix.service == 'graph' && '1' || '0' }} NEO4J_USER: neo4j NEO4J_PASSWORD: test-password-for-ci OBJECT_STORAGE_ACCESS_KEY_ID: ci-test-key OBJECT_STORAGE_SECRET_ACCESS_KEY: ci-test-secret OBJECT_STORAGE_ENDPOINT_URL: http://localhost:4566 PYTHONDONTWRITEBYTECODE: 1 run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${PYTHONPATH}" cd services/${{ matrix.service }} # Per-service coverage thresholds — ratcheted up 2026-04-20 (#1348). # Actual coverage measured 2026-04-20; thresholds set at (actual - 5%) # to give a meaningful floor without immediately breaking CI. # compliance: actual 85% → floor 80% # scheduler: actual 69% → floor 65% # admin: actual 75% → floor 70% # graph: actual 64% → floor 60% # nlp: actual 68% → floor 65% # ingestion: actual ~50% on CI (collect errors block local run) → floor 48% THRESHOLD=60 # services/shared has no `app/` subdir — its source IS the # top-level *.py and subpackages of services/shared/. After # `cd services/shared` we cover ``.`` (current directory); for # everything else we cover the conventional ``app`` package. COV_TARGET=app case "${{ matrix.service }}" in compliance) THRESHOLD=80 ;; # was 30; actual 85% scheduler) THRESHOLD=65 ;; # was 35; actual 69% admin) THRESHOLD=70 ;; # was 50; actual 75% graph) THRESHOLD=60 ;; # was 50; actual 64% ingestion) THRESHOLD=48 ;; # was 42; actual ~50% on CI nlp) THRESHOLD=65 ;; # was 50; actual 68% shared) THRESHOLD=35; COV_TARGET=. ;; # measured ~44% locally; floor 35 absorbs CI-vs-local variance, ratchet to 40 after a stable run. esac # NOTE: --ignore=tests/integration/ is intentionally removed so that # integration tests contribute to the coverage report when they run. # Tests that require live external services should be marked with # @pytest.mark.integration and can be skipped via -m "not integration" # in environments without those services. INTEGRATION_ARGS=() if [ "${{ matrix.service }}" = "graph" ]; then INTEGRATION_ARGS+=(--run-integration) fi pytest tests/ "${INTEGRATION_ARGS[@]}" -v --cov=$COV_TARGET --cov-report=term \ --cov-report=xml:coverage.xml \ --cov-fail-under=$THRESHOLD \ -o "testpaths=tests" - name: Upload coverage if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: coverage-${{ matrix.service }} path: services/${{ matrix.service }}/coverage.xml retention-days: 14 if-no-files-found: ignore # ── Lint ───────────────────────────────────────────────── lint: name: Lint ${{ matrix.service }} runs-on: ubuntu-latest timeout-minutes: 5 strategy: fail-fast: false matrix: service: [admin, compliance, graph, ingestion, nlp, scheduler] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Run linting # F821 (undefined name) is now load-bearing — it caught the runtime # NameError bugs that were hidden in services/nlp/app/extractors/ # llm_extractor.py for months before PR #1482 fixed them. Scope # expanded from ``app/`` only to the whole service directory so # ``main.py`` / ``api.py`` / ``conftest.py`` at service root are # also covered, plus this job now gates merges (see ci-status). run: | pip install ruff==0.15.5 cd services/${{ matrix.service }} ruff check . --select E9,F63,F7,F82 --line-length 120 \ --extend-exclude __pycache__,.venv,venv,build,dist \ --statistics # ── Lint shared/ ───────────────────────────────────────── # ``services/shared/`` isn't in the service matrix but is imported by # every service. F821 regressions there silently break downstream # callers at runtime — lint it explicitly. lint-shared: name: Lint shared/ runs-on: ubuntu-latest timeout-minutes: 3 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Run linting run: | pip install ruff==0.15.5 cd services/shared ruff check . --select E9,F63,F7,F82 --line-length 120 \ --extend-exclude __pycache__,.venv,venv,build,dist \ --statistics # ── Lint repo roots (scripts/, cli/, server/, alembic/, …) ─ # The per-service matrix + lint-shared only cover services/. Python # outside services/ was ungated — that gap let a NameError (missing # ``import time``) sit in scripts/verify_compliance_lineage.py. This job # runs the same bug-gate over everything EXCEPT services/, so the gap # closes for current and future top-level Python without enumerating # roots by hand. lint-repo-roots: name: Lint repo roots runs-on: ubuntu-latest timeout-minutes: 3 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Run linting run: | pip install ruff==0.15.5 ruff check . --select E9,F63,F7,F82 --line-length 120 \ --extend-exclude __pycache__,.venv,venv,build,dist,services \ --statistics # ── RLS fallback-UUID guard (hard gate) ────────────────── # Scans migrations for the fail-open RLS pattern that #1091 fixed. # Any new migration that reintroduces `COALESCE(..., )` # or `COALESCE(current_setting(...), tenant_id)` will fail CI. rls-fallback-check: name: RLS fail-open migration guard runs-on: ubuntu-latest timeout-minutes: 2 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Check for fail-open RLS fallbacks in migrations run: python scripts/check_rls_fallback.py # ── tenant_id UUID-only guard (hard gate) ──────────────── # Five tables drifted to ``tenant_id TEXT/VARCHAR`` instead of UUID, # silently breaking RLS policies that compare against # ``get_tenant_context()`` (which returns UUID). v067 standardized # the live schema; this guard rejects new migrations that would # reintroduce the drift. tenant-id-uuid-check: name: tenant_id UUID-only guard runs-on: ubuntu-latest timeout-minutes: 2 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Check for tenant_id TEXT/VARCHAR drift in migrations run: python scripts/check_tenant_id_uuid_only.py # ── Alembic revision ID guard (hard gate) ──────────────── # Rejects new migrations that use handcrafted sequential revision IDs or # introduce duplicate revision IDs (either would corrupt the migration DAG). # Pre-existing sequential IDs (v043–v072) and known duplicate IDs are # grandfathered — see scripts/check_alembic_revisions.py and #1296. alembic-revision-check: name: Alembic revision ID guard runs-on: ubuntu-latest timeout-minutes: 2 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Check for sequential or duplicate Alembic revision IDs run: python scripts/check_alembic_revisions.py # ── Orphan migration guard (hard gate) ──────────────── # Rejects new Flyway ``services/*/migrations/V*.sql`` files (Railway # runs alembic only — these never execute in prod) and new files # under ``docs/abandoned_migrations_*`` (quarantined during the # 2026-04-20 consolidation incident). Current orphans are # grandfathered via ``scripts/orphan_migrations_allowlist.txt``; # the allowlist must shrink monotonically — see #1864, #1872. - name: Guard against new orphan migrations run: python scripts/check_orphan_migrations.py # ── Dep version drift guard (hard gate) ───────────────── # Scans services/*/requirements*.txt to ensure no per-service file # lowers the floor for shared packages below pyproject.toml. Prevents # the pydantic-settings 2.1 vs 2.6 class of drift — see #1886. dep-version-check: name: Dep version drift guard runs-on: ubuntu-latest timeout-minutes: 2 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Check for dependency version drift run: python scripts/check_dep_versions.py # ── kernel.control retirement guard (hard gate) ───────── # ``kernel/control/`` (codegen + RCE surface) was retired by the # ``epic-g-retire-kernel`` epic. Re-introducing an import in # services/ or server/ is forbidden. See PR #2205 for the # broader migration plan (other kernel/ subpackages still have # live consumers and are tracked separately). - name: Guard against kernel.control imports run: python scripts/check_no_kernel_control_imports.py # ── Ingestion full suite with xdist isolation (hard gate) ─ # Restores real signal from services/ingestion/tests. The existing # ``test`` matrix runs the ingestion suite single-process, where # module-level ``sys.modules[...]`` stubbing in 6 test files cascades # into ~144F/35E across ~25 other files. Under ``--dist=loadfile`` # each test file runs in its own worker process with a fresh # sys.modules, so the cascade can't form. # # The allowlist of excluded files lives in # ``services/ingestion/tests/suite-runs.toml`` and is rendered into # ``--ignore=`` flags by ``scripts/render_ingestion_ignores.py``. # That config splits exclusions into two buckets: 6 module-level # ``sys.modules`` poisoners (still covered by the per-service ``test`` # matrix above), and a small set of pre-existing real failures that # fail solo too (broken mock targets after the routes.py split in # PR #2200). Both buckets must shrink — see the README in that dir. # # Hard gate: this job must block merges if ingestion regresses. # See ``project_test_suite_isolation_2026_04_19.md`` for the original # root-cause table. ingestion-full-suite: name: Ingestion full suite (xdist signal) runs-on: ubuntu-latest # 10 → 15: this job now also migrates a second database to alembic head # and runs the remediation integration suite (see the last two steps). timeout-minutes: 15 services: postgres: image: postgres:16 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres ports: - 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 redis: image: redis:7-alpine ports: - 6379:6379 options: >- --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Create test database env: PGPASSWORD: postgres run: psql -h localhost -U postgres -c "CREATE DATABASE test_db;" - name: Prepare remediation database (roles + pre-baseline function) env: PGPASSWORD: postgres # pragma: allowlist secret run: | # The supplier-remediation integration tests # (services/ingestion/tests/remediation/) exercise FORCE RLS, the # append-only trigger, and the SECURITY DEFINER token lookup # against a real migrated Postgres. Reproduce the production role # layout: migrations remain owned by postgres while the app connects # directly as the exact NOBYPASSRLS runtime role. v111 applies the # production grants; no test-only grant block is permitted. psql -h localhost -U postgres -c "CREATE ROLE regengine LOGIN PASSWORD 'regengine' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION;" # pragma: allowlist secret — throwaway CI-service password psql -h localhost -U postgres -c "CREATE ROLE regengine_sysadmin LOGIN PASSWORD 'regengine_sysadmin' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION;" # pragma: allowlist secret — throwaway CI-service password psql -h localhost -U postgres -c "CREATE DATABASE remediation_test OWNER postgres;" # get_tenant_context() is referenced by baseline RLS policies but # created by NO migration — production Supabase carries it from # pre-baseline state. Seed it exactly as scripts/init-postgres.sh # does for local databases, or `alembic upgrade head` fails. psql -h localhost -U postgres -d remediation_test <<'EOSQL' CREATE OR REPLACE FUNCTION get_tenant_context() RETURNS UUID AS $fn$ DECLARE tid TEXT; BEGIN tid := NULLIF(current_setting('app.tenant_id', TRUE), ''); IF tid IS NULL THEN RAISE EXCEPTION 'app.tenant_id not set - tenant context required for RLS'; END IF; RETURN tid::UUID; END; $fn$ LANGUAGE plpgsql STABLE; EOSQL - name: Cache pip packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 with: path: ~/.cache/pip key: pip-${{ hashFiles('requirements.lock') }} restore-keys: | pip- - name: Install dependencies run: | pip install --require-hashes -r requirements.lock - name: Migrate remediation database to alembic head env: MIGRATIONS_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/remediation_test # pragma: allowlist secret DATABASE_URL: postgresql://postgres:postgres@localhost:5432/remediation_test # pragma: allowlist secret run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" alembic upgrade head - name: Render allowlist ignores id: ignores run: | # ``render_ingestion_ignores.py`` exits non-zero if any # listed file is missing on disk — the suite-runs.toml stays # honest after deletes/renames. IGNORES=$(python3 scripts/render_ingestion_ignores.py) echo "Allowlist ignores: ${IGNORES}" echo "flags=${IGNORES}" >> "$GITHUB_OUTPUT" - name: Run ingestion suite with per-file worker isolation env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db REDIS_URL: redis://localhost:6379/0 JWT_SECRET: test-secret-key-for-ci SUPABASE_URL: http://localhost:54321 SUPABASE_KEY: test-key-for-ci PYTHONDONTWRITEBYTECODE: 1 # ``continue-on-error: false`` is the GA default; stating it # explicitly so the gate's blocking nature is grep-able. continue-on-error: false run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${PYTHONPATH}" # --dist=loadfile pins all tests from one file to one worker # process. Each worker has its own sys.modules, so module-level # stubbing can't poison other files. ``--ignore`` flags come # from the allowlist config so the policy lives in one file. pytest services/ingestion/tests \ --dist=loadfile -n auto \ -q --no-cov \ ${{ steps.ignores.outputs.flags }} - name: Run ingestion live-Postgres tests against alembic head env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/remediation_test # pragma: allowlist secret REDIS_URL: redis://localhost:6379/0 JWT_SECRET: test-secret-key-for-ci SUPABASE_URL: http://localhost:54321 SUPABASE_KEY: test-key-for-ci PYTHONDONTWRITEBYTECODE: 1 REMEDIATION_TEST_SUPERUSER_DSN: postgresql://postgres:postgres@localhost:5432/remediation_test # pragma: allowlist secret REMEDIATION_EXCHANGE_PG_ADMIN_DSN: postgresql://postgres:postgres@localhost:5432/postgres # pragma: allowlist secret # An unreachable DB must FAIL this step, never skip it green (#2530). REGENGINE_REQUIRE_DB: "1" run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${PYTHONPATH}" # The suite step above points at the unmigrated test_db and leaves # REMEDIATION_TEST_SUPERUSER_DSN unset, so every *_pg.py file skips # there. Setting the DSN on that step instead would also un-skip # services/ingestion/tests/remediation/, which production excludes # (see the step below). The glob is deliberately non-recursive: it # takes the top-level *_pg.py files only. If it ever matches nothing # pytest exits non-zero on the literal pattern, so the step cannot # pass by running no files. pytest services/ingestion/tests/test_*_pg.py -v --no-cov - name: Verify disabled remediation production contract env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/remediation_test REDIS_URL: redis://localhost:6379/0 JWT_SECRET: test-secret-key-for-ci SUPABASE_URL: http://localhost:54321 SUPABASE_KEY: test-key-for-ci PYTHONDONTWRITEBYTECODE: 1 REMEDIATION_TEST_SUPERUSER_DSN: postgresql://postgres:postgres@localhost:5432/remediation_test # pragma: allowlist secret REMEDIATION_TEST_APP_DSN: postgresql://regengine:regengine@localhost:5432/remediation_test # pragma: allowlist secret # An unreachable DB must FAIL this step, never skip it green (#2530). REGENGINE_REQUIRE_DB: "1" run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${PYTHONPATH}" # Production excludes remediation and v111 denies its legacy table # and token-function grants. Keep the feature tests for promotion # work, but gate this release on actual-role denial, never synthetic # runtime grants that contradict the production inventory. pytest tests/security/test_remediation_disabled_contract_pg.py -v --no-cov # ── Docker Build (consolidated monolith) ───────────────── # Production deploys a single consolidated monolith built from the # repo-root Dockerfile — see server/main.py and railway.toml. The per- # service Dockerfiles under services// are kept only so the local # docker-compose files still work; they're not built or deployed anywhere # and aren't gated here. See #1429 for the full drift cleanup plan. docker-build: name: Docker Build (monolith) runs-on: ubuntu-latest timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - name: Build root Dockerfile run: docker build -t regengine:test -f Dockerfile . - name: Verify image CMD and import surface # #2890: do not override CMD with `python -c`. Prove the image # actually ships the production entrypoint, then import it. run: | cmd=$(docker inspect --format '{{json .Config.Cmd}}' regengine:test) echo "CMD=${cmd}" echo "${cmd}" | grep -q gunicorn docker run --rm --entrypoint python \ -e REGENGINE_ENV=test -e ADMIN_FALLBACK_SQLITE=sqlite:///:memory: \ regengine:test -c "import server.main" # ── Security (Hard Gate) ───────────────────────────────── security: name: Security Audit ${{ matrix.service }} runs-on: ubuntu-latest # 15 min: pip-audit itself takes ~1 min, but runner post-job cleanup has hung past 5 min (#3599, run 40176059711). timeout-minutes: 15 strategy: fail-fast: false matrix: service: [admin, compliance, graph, ingestion, nlp, scheduler] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Run pip-audit run: | # Root requirements.lock is the single source of truth (per #1429 / # #1433 which removed per-service requirements.txt files; #1139 moved # to pip-compile lockfile). The matrix still fans out by service name # to preserve historical check names on branch-protection rules; # every job audits the same consolidated dependency set. # # Temporary documented ignores: # - PYSEC-2025-183 / PyJWT: no fixed version is published; RegEngine # signs HS256 session JWTs with generated high-entropy AUTH_SECRET_KEY. # - PYSEC-2024-278 / langchain-community: pip-audit flags current # 0.4.1 even though advisory metadata says fixed in 0.0.27. # - PYSEC-2026-161 / starlette: Host header path poisoning. Fix is # starlette 1.0.1 but prometheus-fastapi-instrumentator 7.1.0 # (latest) pins starlette<1.0.0. RegEngine uses decorator-based # auth, not request.url.path reconstruction, so not exploitable # on current routes. Revisit once prometheus-fastapi-instrumentator # supports starlette 1.x. # - CVE-2026-48818 / starlette: Windows-only StaticFiles UNC SSRF. # RegEngine production runs Linux containers; revisit once FastAPI # supports the Starlette 1.x fixed line. # - CVE-2026-48817 / starlette: HTTPEndpoint non-standard method # dispatch. RegEngine does not register Starlette HTTPEndpoint # subclasses; revisit once FastAPI supports the fixed line. # - CVE-2026-54282 and CVE-2026-54283 / starlette: request URL/form # parser hardening fixed in Starlette 1.x. FastAPI 0.136 still # resolves Starlette 0.52.x, so keep this explicit until the # framework dependency allows 1.x. pip_audit_ignores=( --ignore-vuln PYSEC-2025-183 --ignore-vuln PYSEC-2024-278 --ignore-vuln PYSEC-2026-161 --ignore-vuln CVE-2026-48818 --ignore-vuln CVE-2026-48817 --ignore-vuln CVE-2026-54282 --ignore-vuln CVE-2026-54283 ) python -m pip install pip-audit==2.10.0 pip-audit -r requirements.lock --desc on "${pip_audit_ignores[@]}" # ── Repo-root security / RLS / isolation suite (hard gate) ─ # The highest-value correctness suites live in the repo-root tests/ dir # (cross-tenant isolation, RLS coverage + hardening, hash-chain audit # integrity, canonical-event invariants). The per-service `test` matrix # cd's into services/ and never runs them; test-suite-check only # `--collect-only`s them; qa-health runs them continue-on-error with no DB. # So a regression in tenant isolation or the evidence hash chain merged # green (#2502). The RLS behavioral tests additionally only knew how to # spin up Docker-in-Docker testcontainers and skipped silently in every # blocking job (#2530). # # This job provisions Postgres + Redis, sets DATABASE_URL so the # ``skipif(not DATABASE_URL)`` / ``rls_postgres_or_skip`` guards activate, # and sets REGENGINE_REQUIRE_DB=1 so the RLS suites FAIL (not skip) if the # DB is unreachable — a missing DB layer can never turn this gate green. # Each DB-mutating suite gets its own fresh database to avoid schema # interference (alembic head vs the hardening test's self-built tables). repo-root-suite: name: Repo-root security/RLS/isolation suite runs-on: ubuntu-latest # Raised from 15 for #3248: this job now runs `npm ci` plus a production # app import child with its own 300s timeout, on top of ten new test # directories. A timeout fails the status gate and blocks deploy, so the # cap must leave headroom for a slow registry or a slow import. timeout-minutes: 25 services: postgres: image: postgres:16 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres ports: - 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 redis: image: redis:7-alpine ports: - 6379:6379 options: >- --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 env: REDIS_URL: redis://localhost:6379/0 JWT_SECRET: test-secret-key-for-ci # Make missing/unreachable Postgres a hard failure for the RLS suites # so the behavioral gate can't pass green by skipping (#2530). REGENGINE_REQUIRE_DB: "1" PYTHONDONTWRITEBYTECODE: 1 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Cache pip packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 with: path: ~/.cache/pip key: pip-${{ hashFiles('requirements.lock') }} restore-keys: | pip- - name: Install dependencies run: pip install --require-hashes -r requirements.lock - name: Route-complete security manifest and disabled-router gate run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" # Imports the broad inventory app to execute every operation's auth # adapter, then imports the exact production allowlist in a clean # child process. Every omitted operation is safely concretized with # its actual method and must return exact 404 against that table. python scripts/generate_route_security_manifest.py --check - name: Enabled outbound HTTP and queue-handler guard run: | python scripts/check_enabled_outbound_http.py pytest tests/security/test_enabled_outbound_http_guard.py \ tests/security/test_disabled_router_task_handlers.py \ -v --no-cov -o "testpaths=tests" - name: Sanitized release-evidence gate run: | python scripts/validate_release_evidence.py pytest tests/security/test_release_evidence.py -v --no-cov -o "testpaths=tests" - name: Anti-stub canonical guard (#2989) # Catches the incident class that merged green on 2026-09-17: a # cross-module call site whose backing function was dropped by a # merge resolution (0c6dfd8b^). Guard fires: call site exists, # function missing. Behavioral drivers re-prove the load-bearing # modules in fresh subprocesses. Bite proof: PR #3129. run: | pytest tests/test_guard_canonical.py -v --no-cov -o "testpaths=tests" - name: Create isolated test databases env: PGPASSWORD: postgres run: | # Separate DBs so alembic-migrated suites and the self-bootstrapping # RLS hardening suite never collide on schema. psql -h localhost -U postgres -c "CREATE DATABASE rls_migrated;" psql -h localhost -U postgres -c "CREATE DATABASE rls_hardening;" - name: Offline invariants (no DB) run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" # These execute pure-Python assertions (hash-chain audit integrity, # canonical-event normalization). No DB, no skips. # test_logging_binding_guard: AST scan banning structlog-style # kwargs on stdlib logging.Logger bindings — the defect class that # shipped twice (sandbox/router.py, then request_workflow/#2625). # test_visibility_timeout_2918 (#2918) and test_secret_validation # (SEC-004) ran in NO blocking job until the QA suite review found # them ignored in test-suite-check's fast shard: pure-offline # regression tests with no gate. They run here, where a failure # blocks the merge. # The four shared-auth-kernel qa4 files (#3252) are pure-offline # (in-memory stores, fakes, in-process TestClient) and likewise # ran only in test-suite-check, which is not a required check. # They run here, where a failure blocks the merge. pytest tests/security/test_audit_integrity.py tests/test_canonical_event.py \ tests/test_logging_binding_guard.py \ tests/test_guard_text_bind_casts.py \ tests/test_ci_qa_gates_3248.py \ tests/shared/test_visibility_timeout_2918.py \ tests/shared/test_secret_validation.py \ tests/shared/test_shared_auth_contract_qa4.py \ tests/shared/test_secrets_manager_qa4.py \ tests/shared/test_brute_force_limiter_qa4.py \ tests/shared/test_tenant_context_qa4.py \ -v --no-cov -o "testpaths=tests" - name: Operator database tooling tests run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" # These scripts change database roles, credentials and grants, so # their tests must block a merge. The only other workflow that # collects tests/scripts is test-suite-check.yml, which is not a # required check and does not trigger on scripts/** changes. # test_bootstrap_staging_database drives the staging bootstrap with # fake railway and psql binaries. Its membership-revoke tests also # start a disposable postgres:17 through testcontainers and roll # every scenario back; with the job's REGENGINE_REQUIRE_DB=1 they # fail rather than skip if Docker is unavailable. # test_v126_operator_guidance pins the v126 refusal message and the # read-only, search_path-pinned verify and pre-deploy runbooks. # test_check_supply_chain guards the supply-chain pinning gate: # a checker-only weakening under scripts/** must fail here, # because test-suite-check.yml does not trigger on that path. # test_prod_default_guards pins the localhost-default and # non-localhost-refusal guards on the operator helper scripts. pytest tests/scripts/test_bootstrap_staging_database.py \ tests/scripts/test_v126_operator_guidance.py \ tests/scripts/test_check_supply_chain.py \ tests/scripts/test_prod_default_guards.py \ -v --no-cov -o "testpaths=tests" - name: Setup Node.js (published-spec contract needs the TS export) uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6 with: node-version-file: .nvmrc cache: 'npm' cache-dependency-path: frontend/package-lock.json - name: Install frontend dependencies (published-spec contract) run: | cd frontend npm ci - name: In-process security & FSMA-204 contract suites (#2503) run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" # Formerly live-HTTP suites that silently skipped on ConnectError # because no CI job booted the services (#2503). Rewritten to run # in-process against the consolidated monolith via TestClient. # Needs the Redis service (job env REDIS_URL) for the job-status # tenant-scoping tests; no Postgres — every assertion resolves # before DB access. An unreachable Redis FAILS these tests. # The repo-root offline mirrors (tests/admin, tests/billing, ...) # and the published OpenAPI surface contract executed in NO # blocking job (QA suite review): they are asserted here, in the # same process composition that was rehearsed green locally. # NOT included, deliberately: tests/e2e and tests/vertical_services # (need a live stack; skip without one), tests/load (locust/k6 # perf harnesses, not merge-gate material), tests/nlp (whole file # skips without the full NLP pipeline), # tests/contract/test_service_contracts.py (needs live services), # tests/contract/test_admin_api_contract.py (needs AUDIT_INTEGRITY_KEY # plus a generated docs/openapi/admin-api.json that no CI step # produces — it would only skip), and # tests/contract/test_error_envelope_contract_2863.py (fails on the # documented flat 409 idempotency shape; needs a product decision). # tests/audit is NOT listed: it holds only manual helper scripts # (audit_trace_back.py, verify_chain.py), which do not match # python_files = "test_*.py", so pytest collects nothing there and # the gate comment must not claim they are asserted. Hash-chain # audit integrity is gated by # tests/security/test_audit_integrity.py in the offline step above. # tests/qa_sweep_20261002/test_portal_export_disclosure.py is # excluded below: BE-011 is still a strict xfail (the redacted FDA # CSV still embeds the portal link token via leftover KDEs), so an # expected failure there exits 0 and proves nothing. Keep it out # of the blocking argv until the export stops embedding portal_id; # when that is fixed, delete the xfail and drop this --ignore so a # regression fails the job. pytest tests/test_router_policy.py \ tests/test_server_import_smoke.py \ tests/test_request_workflow_unit.py \ tests/test_compliance_control_plane_e2e.py \ tests/test_rls_probe_helper.py \ tests/contract/test_ingest_nlp_schema_contract.py \ tests/security/test_account_erasure_live_db_guard.py \ tests/security/test_global_admin_context_allowlist.py \ tests/security/test_security_audit_fixes.py \ tests/security/test_auth_coverage.py \ tests/security/test_tenant_route_isolation_matrix.py \ tests/security/test_route_security_manifest.py \ tests/security/test_staging_release_gate_runner.py \ tests/data_integrity/test_fsma_204_verification_phase2b.py \ tests/admin \ tests/billing \ tests/compliance \ tests/docs \ tests/graph \ tests/infra \ tests/ingestion \ tests/recall_reconstruction \ tests/qa_sweep_20261002 \ tests/contract/test_openapi_production_surface_2861.py \ --ignore=tests/qa_sweep_20261002/test_portal_export_disclosure.py \ -v --no-cov -o "testpaths=tests" - name: RLS coverage + tenant-isolation e2e (alembic head) env: MIGRATIONS_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/rls_migrated DATABASE_URL: postgresql://postgres:postgres@localhost:5432/rls_migrated ADMIN_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/rls_migrated run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" # test_rls_coverage runs `alembic upgrade head` into rls_migrated and # asserts every fsma.* table is RLS-hardened; the e2e isolation test # reuses the same migrated schema. Both FAIL (not skip) here because # REGENGINE_REQUIRE_DB=1 and DATABASE_URL is reachable. # test_client_role_lockdown_v092 asserts the v092 client-role # lockdown (Supabase anon/authenticated exposure) holds at head. pytest tests/migrations/test_rls_coverage.py \ tests/migrations/test_client_role_lockdown_v092.py \ tests/test_e2e_tenant_isolation.py \ tests/invariants/test_dual_write_parity.py \ tests/invariants/test_fda_export_v1_golden.py \ tests/invariants/test_chain_verify_golden.py \ tests/invariants/test_no_third_party_regengine_domains.py \ tests/migrations/test_v097_pin_get_tenant_context.py \ tests/migrations/test_v099_set_tenant_context.py \ tests/migrations/test_v120_rule_citations.py \ -v --no-cov -o "testpaths=tests" - name: RLS enforcement e2e (live Postgres — #2461 fail-closed proof) env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/rls_migrated ADMIN_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/rls_migrated run: | set -o pipefail repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" # services/shared/tests is covered by NO other job — `lint-shared` # runs ruff only — so until now all 12 tests in # test_rls_enforcement_live.py skipped everywhere: locally for want of # DATABASE_URL, and in CI because the step above only passes tests/… # paths. They are the sole end-to-end proof that an unset # app.tenant_id FAILS CLOSED (raises) rather than silently returning # zero rows — the entire premise of the #2461 cutover. # # rls_migrated is already at alembic head (migrated by # test_rls_coverage above), so the suite's schema guard passes and the # tests RUN. Kept as its own step because the root conftest remaps # sys.path per service between collectors, making a mixed # tests/… + services/… invocation fragile. pytest services/shared/tests/test_rls_enforcement_live.py \ services/shared/tests/test_supplier_data_quality_live.py \ -v --no-cov -rs -o "testpaths=services/shared/tests" \ | tee /tmp/rls_live.log # A skip is visually identical to a pass in the checks UI, and a # skipped RLS suite is exactly how this went unvalidated through the # whole #2461 build-out. Treat any skip as a failure. if grep -qE '[0-9]+ skipped' /tmp/rls_live.log; then echo "::error::live RLS tests SKIPPED — they must execute, not skip (see #2461)" exit 1 fi - name: Request-workflow e2e (deadline breach, #2625) env: PGPASSWORD: postgres DATABASE_URL: postgresql://postgres:postgres@localhost:5432/request_workflow_e2e ADMIN_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/request_workflow_e2e run: | set -o pipefail repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" # The repo-root request-workflow e2e suites ran in NO job, which is # how services/shared/request_workflow shipped with structlog-style # kwargs on stdlib loggers — every deadline-breach log call raised # TypeError in production and nothing red appeared anywhere (#2625). # # rls_migrated is at alembic head (migrated by test_rls_coverage # above); clone it so e2e case data never leaks into the RLS, # ORM-drift, or schema-parity gates that share this job. psql -h localhost -U postgres \ -c "CREATE DATABASE request_workflow_e2e TEMPLATE rls_migrated;" # All five suites, one invocation, one fresh DB. fda_request / # identity_ambiguity / stale_evaluations previously failed on any # fresh alembic-head DB (unseeded rule_definitions fixtures, the # matching.py aborted-transaction bug, uuid=ANY(text[]) in bulk # merge) — fixed in #2628; rehearsed combined against a fresh # template clone: 44/44. test_e2e_package_manifest.py is the # #3245 blocking-defect proof (it calls the real submit_package # gate); it ran in no blocking job until #3245 review found it # ignored, so a gutted gate stayed green. Rehearsed 12/12 with # zero skips against a fresh alembic-head DB. pytest tests/test_e2e_deadline_breach.py \ tests/test_e2e_fda_request.py \ tests/test_e2e_identity_ambiguity.py \ tests/test_e2e_stale_evaluations.py \ tests/test_e2e_package_manifest.py \ -v --no-cov -o "testpaths=tests" | tee /tmp/rw_e2e.log # Same skip-hardening as the RLS live step: a skipped suite is # invisible in the checks UI, and invisible is how this gap lived # for the package's entire life. if grep -qE '[0-9]+ skipped' /tmp/rw_e2e.log; then echo "::error::request-workflow e2e SKIPPED — must execute, not skip (see #2625)" exit 1 fi - name: ORM drift gate (alembic check, WS-2b) env: MIGRATIONS_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/rls_migrated DATABASE_URL: postgresql://postgres:postgres@localhost:5432/rls_migrated run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" # rls_migrated is at alembic head (migrated by test_rls_coverage in # the previous step). `alembic check` compares EVERY registered ORM # Base (admin, agent-operator, api-key store, scheduler state — see # alembic/env.py target_metadata) against that schema and fails on # structural drift: an ORM model whose table/column/index/FK no # migration created, or whose FK options (ondelete, deferrability) # disagree with the migrated schema. FK comparison used to be off # (include_object excluded every foreign_key_constraint); turning # it on caught 20 ORM FKs missing their ondelete plus 5 FKs missing # from the receipts model. This is the gate that would have caught # scheduler_seen_items existing only via runtime create_all (v093). # Raw-SQL-managed tables without ORM models are exempt by design # (include_object in alembic/env.py); type-level alignment is # enforced separately by verify_orm_db_type_alignment. alembic check # Behavioral proof the gate is not vacuous: on a throwaway scratch # database (created and dropped by the test, never the shared # rls_migrated) assert autogenerate produces no diff at head, then # drift one FK option and assert the gate fails. With # REGENGINE_REQUIRE_DB=1 (job env) this fails instead of skipping # when Postgres is unreachable. pytest tests/migrations/test_alembic_fk_autogenerate_no_diff.py \ -v --no-cov -o "testpaths=tests" - name: Schema parity audit (WS-2c) env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/rls_migrated run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" # Compares the alembic-head schema against the checked-in contract # (docs/schema/expected_schema.json) — EVERY table, including the # raw-SQL canonical FSMA tables the ORM gate cannot see. A # migration that changes the schema must regenerate the contract # in the same PR (scripts/schema_parity_audit.py snapshot), so # schema changes are always deliberate, reviewable diffs. python scripts/schema_parity_audit.py check - name: RLS hardening behavioral assertions env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/rls_hardening run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" # Cross-tenant SELECT denial, unset-context fail-hard, sandbox-UUID # sees no real rows. Self-bootstraps its schema into rls_hardening. pytest tests/migrations/test_rls_hardening.py \ -v --no-cov -o "testpaths=tests" # ── Seeded mid-chain migration rehearsal (#2588 follow-up) ── seeded-migration-rehearsal: # Every other migration gate upgrades an EMPTY database, which is how # v090's audit_logs backfill shipped green and then failed both # production deploys (the v066 append-only trigger only fires on rows). # This job upgrades to v089, seeds a representative dataset (tenants, # users, audit_logs, fsma.cte_events, fsma.request_signoffs), then runs # `alembic upgrade head` over the populated tables. Migrations are also # role-sensitive (v051/v056/v092 branch on Supabase client roles), so a # matrix leg replays the chain on a Supabase-like cluster too. name: Seeded migration rehearsal (${{ matrix.cluster }}, ${{ matrix.revision }}) runs-on: ubuntu-latest timeout-minutes: 15 strategy: fail-fast: false matrix: cluster: [app-roles-only, supabase-client-roles] revision: [267d8053ced5, 38e02636e340] services: postgres: image: postgres:17 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres ports: - 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 env: # No-Postgres must FAIL the rehearsal, never skip it (#2530). REGENGINE_REQUIRE_DB: "1" REGENGINE_SEED_REVISION: ${{ matrix.revision }} AUTH_OTP_HMAC_SECRET: synthetic-migration-rehearsal-hmac-secret-2026 # pragma: allowlist secret PGOPTIONS: "-c timezone=UTC -c search_path=public,fsma,extensions" PYTHONDONTWRITEBYTECODE: 1 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Cache pip packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 with: path: ~/.cache/pip key: pip-${{ hashFiles('requirements.lock') }} restore-keys: | pip- - name: Install dependencies run: pip install --require-hashes -r requirements.lock - name: Create dedicated-owner rehearsal database env: PGPASSWORD: postgres run: | # The rehearsal drives the DB from empty through v089 + seed to # head itself. Migrations run as a NOCREATEROLE/NOBYPASSRLS owner; # the provider URL is exposed only to the fixture seeding seam. psql -v ON_ERROR_STOP=1 -h localhost -U postgres -d postgres <<'EOSQL' CREATE ROLE regengine LOGIN PASSWORD 'seeded-runtime-password' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; CREATE ROLE regengine_sysadmin LOGIN PASSWORD 'seeded-admin-password' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; CREATE ROLE regengine_migrator_seeded LOGIN PASSWORD 'seeded-migration-password' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; CREATE DATABASE seeded_migrated OWNER regengine_migrator_seeded; GRANT CONNECT ON DATABASE seeded_migrated TO regengine, regengine_sysadmin; EOSQL - name: Seeded mid-chain upgrade rehearsal env: MIGRATIONS_DATABASE_URL: postgresql://regengine_migrator_seeded:seeded-migration-password@localhost:5432/seeded_migrated # pragma: allowlist secret REGENGINE_MIGRATIONS_ROLE: regengine_migrator_seeded REGENGINE_ENFORCE_MIGRATION_ROLE: "1" DATABASE_URL: postgresql://regengine_migrator_seeded:seeded-migration-password@localhost:5432/seeded_migrated # pragma: allowlist secret ROLE_ADMIN_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/seeded_migrated # pragma: allowlist secret REGENGINE_SEED_CLIENT_ROLES: ${{ matrix.cluster == 'supabase-client-roles' && '1' || '0' }} run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" pytest tests/migrations/test_seeded_mid_chain_upgrade.py \ -v --no-cov -o "testpaths=tests" # ── Live-DB regression suite (tests/shared/*_live.py) ── live-db-suite: # Two prod incidents (data_archival's phantom fsma.fda_exports table, # 2026-07-24; the least-privilege auth-bootstrap RLS 500s, 2026-07-27) # shipped through green CI because their defects were schema-level and # the tests/shared/*_live.py regression tests that catch them silently # skip without a designated database (#2646 guard). This job provisions # a scratch Postgres, bootstraps the prod role topology, migrates to # alembic head, and runs every live module — hard-failing if ANY module # skips, so the suite can never rot back into silence. name: Live DB suite (migrated scratch) runs-on: ubuntu-latest timeout-minutes: 15 services: postgres: image: postgres:17 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres ports: - 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 env: # No-designated-DB must FAIL the suite, never skip it (#2530): # guarded_live_database_url() pytest.fails when this is set and no # usable URL survives its reuse policy. The skip-grep on the final # step is the backstop for runtime skips (e.g. a missing role). REGENGINE_REQUIRE_DB: "1" PYTHONDONTWRITEBYTECODE: 1 PGOPTIONS: "-c timezone=UTC -c search_path=public,fsma,extensions" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - name: Setup Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: Cache pip packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 with: path: ~/.cache/pip key: pip-${{ hashFiles('requirements.lock') }} restore-keys: | pip- - name: Install dependencies run: pip install --require-hashes -r requirements.lock - name: Create scratch database (dedicated migration + runtime roles) env: PGPASSWORD: postgres run: | # pragma: allowlist secret # Database name must NOT be regengine/regengine_admin: the #2646 # guard (tests/migrations/rls_db.py) refuses loopback URLs with # those names even under CI=true, and the whole job would skip. # The provider credential creates roles and the database only. The # full Alembic chain then runs as the exact NOCREATEROLE, # NOBYPASSRLS schema owner used by production migration jobs. psql -v ON_ERROR_STOP=1 -h localhost -U postgres -d postgres <<'EOSQL' CREATE ROLE regengine LOGIN PASSWORD 'runtime-test-password' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; CREATE ROLE regengine_sysadmin LOGIN PASSWORD 'admin-runtime-test-password' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; CREATE ROLE regengine_migrator_ci LOGIN PASSWORD 'migration-test-password' NOSUPERUSER NOBYPASSRLS NOCREATEDB NOCREATEROLE NOREPLICATION; CREATE DATABASE live_suite OWNER regengine_migrator_ci; GRANT CONNECT ON DATABASE live_suite TO regengine, regengine_sysadmin; EOSQL # get_tenant_context() is referenced by baseline RLS policies but # created by NO migration — production Supabase carries it from # pre-baseline state. Seed it exactly as scripts/init-postgres.sh # does for local databases, or `alembic upgrade head` fails. Create # it as the migration owner so no provider-owned function is hidden # in an otherwise least-privilege rehearsal. PGPASSWORD=migration-test-password psql -v ON_ERROR_STOP=1 \ -h localhost -U regengine_migrator_ci -d live_suite <<'EOSQL' CREATE OR REPLACE FUNCTION get_tenant_context() RETURNS UUID AS $fn$ DECLARE tid TEXT; BEGIN tid := NULLIF(current_setting('app.tenant_id', TRUE), ''); IF tid IS NULL THEN RAISE EXCEPTION 'app.tenant_id not set - tenant context required for RLS'; END IF; RETURN tid::UUID; END; $fn$ LANGUAGE plpgsql STABLE; EOSQL - name: Run fresh and managed migration-runner paths env: MIGRATIONS_DATABASE_URL: postgresql://regengine_migrator_ci:migration-test-password@localhost:5432/live_suite # pragma: allowlist secret REGENGINE_MIGRATIONS_ROLE: regengine_migrator_ci DATABASE_URL: postgresql://regengine:runtime-test-password@localhost:5432/live_suite # pragma: allowlist secret ADMIN_DATABASE_URL: postgresql://regengine_sysadmin:admin-runtime-test-password@localhost:5432/live_suite # pragma: allowlist secret MIGRATION_LOCK_TIMEOUT_S: "10" run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" # First invocation proves the full chain as the dedicated # NOCREATEROLE migration owner. The second proves the managed-head # path and convergent v111 repair. bash scripts/run-migrations.sh bash scripts/run-migrations.sh - name: Refuse an unversioned application schema without stamping env: PGPASSWORD: postgres # pragma: allowlist secret run: | # pragma: allowlist secret psql -v ON_ERROR_STOP=1 -h localhost -U postgres -d postgres \ -c "CREATE DATABASE unmanaged_suite OWNER regengine_migrator_ci;" PGPASSWORD=migration-test-password psql -v ON_ERROR_STOP=1 \ -h localhost -U regengine_migrator_ci -d unmanaged_suite \ -c "CREATE SCHEMA fsma; CREATE TABLE fsma.preexisting_data (id integer);" set +e MIGRATIONS_DATABASE_URL='postgresql://regengine_migrator_ci:migration-test-password@localhost:5432/unmanaged_suite' \ MIGRATION_LOCK_TIMEOUT_S=10 \ bash scripts/run-migrations.sh > /tmp/unmanaged-migration.log 2>&1 runner_status=$? set -e if [ "$runner_status" -ne 65 ]; then cat /tmp/unmanaged-migration.log echo "::error::unmanaged schema must fail with exit 65; got ${runner_status}" exit 1 fi grep -q "Refusing to stamp an unknown schema to head" /tmp/unmanaged-migration.log test "$(psql -XAt -h localhost -U postgres -d unmanaged_suite -c \ "SELECT to_regclass('public.alembic_version') IS NULL")" = "t" - name: Prove v111 grants with actual runtime credentials env: MIGRATIONS_DATABASE_URL: postgresql://regengine_migrator_ci:migration-test-password@localhost:5432/live_suite # pragma: allowlist secret REGENGINE_MIGRATIONS_ROLE: regengine_migrator_ci REGENGINE_ENFORCE_MIGRATION_ROLE: "1" DATABASE_URL: postgresql://regengine:runtime-test-password@localhost:5432/live_suite # pragma: allowlist secret ADMIN_DATABASE_URL: postgresql://regengine_sysadmin:admin-runtime-test-password@localhost:5432/live_suite # pragma: allowlist secret ROLE_ADMIN_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/live_suite # pragma: allowlist secret REGENGINE_REQUIRE_RLS_ENFORCEMENT: "1" run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" set -o pipefail # The cutover lock-timeout test runs last: it drifts a grant, holds # a lock on public.api_keys, and restores both before it returns. pytest tests/migrations/test_v111_runtime_role_grants.py \ tests/security/test_account_erasure_identity_isolation_pg.py \ tests/migrations/test_runtime_role_cutover_lock_timeout.py \ -v -rs --no-cov -o "testpaths=tests" | tee /tmp/v111-role-cutover.log if grep -qE '[0-9]+ skipped' /tmp/v111-role-cutover.log; then echo "::error::v111 role-cutover proof skipped with all four direct credentials configured" exit 1 fi - name: Verify owner-only rule backfills and rollback env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/live_suite # pragma: allowlist secret run: | export PYTHONPATH="${PWD}:${PWD}/services:${PYTHONPATH}" # v126 runs its real upgrade/downgrade as this job's NOBYPASSRLS # schema owner over populated, FORCE-RLS tables, then rolls back. pytest tests/migrations/test_rule_backfill_owner_rls.py \ tests/migrations/test_v120_rule_citations.py \ tests/migrations/test_v126_tenant_composite_run_fks.py \ -v -rs --no-cov -o "testpaths=tests" - name: Run live-DB suite (skips are failures) env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/live_suite # pragma: allowlist secret ADMIN_DATABASE_URL: postgresql://regengine_sysadmin:admin-runtime-test-password@localhost:5432/live_suite # pragma: allowlist secret run: | repo_root="$(pwd)" export PYTHONPATH="${repo_root}:${repo_root}/services:${PYTHONPATH}" set -o pipefail # Glob, not an explicit list: a future tests/shared/*_live.py is # included automatically instead of silently joining the gap this # job exists to close. pytest tests/shared/test_*_live.py \ -v -rs --no-cov -o "testpaths=tests" | tee /tmp/live-db-suite.log if grep -qE '[0-9]+ skipped' /tmp/live-db-suite.log; then echo "::error::live-DB tests skipped — this job exists so they always RUN (check the #2646 guard env contract)" exit 1 fi - name: RLS deny-all guard + tenantless enqueue idempotency (skips are failures) env: # Provider credential: v104 inspects every role's grants/policies # and probes with a temp table; #2852 creates a disposable # database on this server. Both ran in no workflow until now, so # v104 went stale against v111's closed-grant tables and #2852 hid # a broken enqueue_task insert. DATABASE_URL: postgresql://postgres:postgres@localhost:5432/live_suite # pragma: allowlist secret run: | export PYTHONPATH="${PWD}:${PWD}/services:${PYTHONPATH}" set -o pipefail pytest tests/migrations/test_v104_no_deny_all_rls_tables.py \ tests/migrations/test_task_queue_tenantless_idempotency_2852.py \ -v -rs --no-cov -o "testpaths=tests" | tee /tmp/rls-guard.log if grep -qE '[0-9]+ skipped' /tmp/rls-guard.log; then echo "::error::RLS deny-all guard or #2852 idempotency suite skipped" exit 1 fi - name: Verify ingestion and exports with the restricted runtime role env: # Separate process from the provider-credential live suite above. DATABASE_URL: postgresql://regengine:runtime-test-password@localhost:5432/live_suite # pragma: allowlist secret REGENGINE_EVIDENCE_TEST_DATABASE_URL: postgresql://regengine:runtime-test-password@localhost:5432/live_suite # pragma: allowlist secret REGENGINE_EVIDENCE_ADMIN_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/live_suite # pragma: allowlist secret PYTHON_DOTENV_DISABLED: "1" REGENGINE_ENV: test ENV: test ENVIRONMENT: test ENABLE_NEO4J_SYNC: "false" RULES_ENGINE_ENFORCE: "off" FDA_EXPORT_V1_READ_CANONICAL: "0" API_KEY: local-synthetic-measurement-key # pragma: allowlist secret REGENGINE_API_KEY: local-synthetic-measurement-key # pragma: allowlist secret WEBHOOK_HMAC_SECRET: local-synthetic-measurement-signature-only # pragma: allowlist secret REDIS_URL: redis://127.0.0.1:1/0 WEBHOOK_INGEST_RATE_LIMIT_RPM: "1000" WEBHOOK_READ_RATE_LIMIT_RPM: "1000" INGESTION_RBAC_RATE_LIMIT_DEFAULT_RPM: "1000" run: | export PYTHONPATH="${PWD}:${PWD}/services:${PWD}/services/ingestion:${PYTHONPATH}" set -o pipefail pytest tests/shared/test_postgres_evidence_workflow_pg.py \ tests/shared/test_email_ingest_restricted_role_pg.py \ -v -rs --no-cov -o "testpaths=tests" | tee /tmp/evidence-workflow.log if grep -qE '[0-9]+ skipped' /tmp/evidence-workflow.log; then echo "::error::restricted-role evidence workflow skipped" exit 1 fi # ── CI Summary ─────────────────────────────────────────── ci-status: # Distinct name from the frontend gate ("Frontend CI Status Gate"). The # Railway deploy ships the backend image, so deploy.yml waits on THIS gate # specifically — a shared "CI Status Gate" name previously let a # frontend-only SHA satisfy the deploy gate with no backend run (#2537). name: Backend CI Status Gate # Run when a needed job failed so this gate still posts failure (#2537: # cancelled or skipped dependencies must not look green). Skip only when # the workflow itself was cancelled by a newer push. In that case the # replacement run owns the gate; this run must not fail the SHA for # jobs that were aborted mid-flight. `!cancelled()` does not relax the # script below: every dependency still has to be success. if: always() && !cancelled() needs: [dependency-lock, test, lint, lint-shared, lint-repo-roots, ingestion-full-suite, repo-root-suite, seeded-migration-rehearsal, live-db-suite, docker-build, security, rls-fallback-check, alembic-revision-check, dep-version-check, tenant-id-uuid-check] runs-on: ubuntu-latest steps: - name: Check results run: | echo "Dependency lock: ${{ needs.dependency-lock.result }}" echo "Test: ${{ needs.test.result }}" echo "Lint: ${{ needs.lint.result }}" echo "Lint shared/: ${{ needs.lint-shared.result }}" echo "Lint repo roots: ${{ needs.lint-repo-roots.result }}" echo "Ingestion suite: ${{ needs.ingestion-full-suite.result }}" echo "Repo-root suite: ${{ needs.repo-root-suite.result }}" echo "Seeded rehearsal: ${{ needs.seeded-migration-rehearsal.result }}" echo "Live DB suite: ${{ needs.live-db-suite.result }}" echo "Docker: ${{ needs.docker-build.result }}" echo "Security: ${{ needs.security.result }}" echo "RLS fallback: ${{ needs.rls-fallback-check.result }}" echo "Alembic rev IDs: ${{ needs.alembic-revision-check.result }}" echo "Dep drift guard: ${{ needs.dep-version-check.result }}" echo "tenant_id UUID: ${{ needs.tenant-id-uuid-check.result }}" # Lint is now blocking — ruff F821 (undefined name) would # have caught PR #1482's NLP NameError class of bugs if this # gate had been enforcing. Promoting lint to a hard gate so # the next occurrence trips on PR open rather than in prod. if [[ "${{ needs.dependency-lock.result }}" != "success" \ || "${{ needs.test.result }}" != "success" \ || "${{ needs.lint.result }}" != "success" \ || "${{ needs.lint-shared.result }}" != "success" \ || "${{ needs.lint-repo-roots.result }}" != "success" \ || "${{ needs.ingestion-full-suite.result }}" != "success" \ || "${{ needs.repo-root-suite.result }}" != "success" \ || "${{ needs.seeded-migration-rehearsal.result }}" != "success" \ || "${{ needs.live-db-suite.result }}" != "success" \ || "${{ needs.docker-build.result }}" != "success" \ || "${{ needs.security.result }}" != "success" \ || "${{ needs.rls-fallback-check.result }}" != "success" \ || "${{ needs.alembic-revision-check.result }}" != "success" \ || "${{ needs.dep-version-check.result }}" != "success" \ || "${{ needs.tenant-id-uuid-check.result }}" != "success" ]]; then echo "::error::One or more required backend checks failed, were cancelled, or were skipped" exit 1 fi echo "All checks passed"