name: Deploy Dev

# ADR-009 Phase 3: GitHub Actions → GKE commonly-dev via Workload Identity Federation.
# No long-lived service account JSON in repo secrets; a fresh OIDC token is minted
# per run and exchanged for a short-lived GCP access token.
#
# Trigger is intentionally workflow_dispatch only for the first rollout — once one
# successful run lands, a follow-up PR flips to `push: branches: [main]` for
# auto-deploy-on-merge. Phase 4 layers on post-deploy smoke probes + rollback.

on:
  workflow_dispatch:
    inputs:
      image_tag:
        description: 'Image tag override (defaults to the short SHA of the deployed commit)'
        required: false
        default: ''
      ref:
        description: 'Full 40-character commit SHA to deploy (defaults to main HEAD). It needs a green staging-smoke.'
        required: false
        default: ''
      allow_without_staging_smoke:
        description: 'EMERGENCY: deploy prod even without a green staging-smoke on this SHA'
        type: boolean
        required: false
        default: false

permissions:
  id-token: write     # required to mint the OIDC token for google-github-actions/auth
  contents: read
  statuses: read      # read the staging-smoke commit status for the prod-promote gate

concurrency:
  group: deploy-dev
  cancel-in-progress: false

env:
  # PROJECT_ID read from DEV_GCP_PROJECT_ID secret (not committed per sensitive-data policy).
  # Exported per-job via a setup step below.
  REGION: us-central1
  AR_REPO: docker
  CLUSTER: commonly-dev
  NAMESPACE: commonly-dev

jobs:
  # Prod-promote gate (Sam, 2026-10-23): no prod deploy without a green `staging-smoke`
  # commit status on this exact SHA. Deploy Staging runs the Tier-1 UI smoke and posts that
  # status; this job reads it and blocks the deploy if it is not success. The
  # allow_without_staging_smoke input is the explicit emergency override.
  #
  # The run is always dispatched on main, which the deploy identity trusts. `inputs.ref` picks
  # which commit it builds: a commit that is not main HEAD, e.g. a release commit holding main's
  # reviewed changes up to a point plus a cherry-picked fix. Only a full SHA is accepted, and the
  # deploy job checks out the SHA this job resolved, never a branch name.
  require_staging_smoke:
    name: Require green staging-smoke on this SHA
    runs-on: ubuntu-latest
    outputs:
      sha: ${{ steps.resolve.outputs.sha }}
    steps:
      - name: Resolve the commit to deploy
        id: resolve
        env:
          GH_TOKEN: ${{ github.token }}
          REF: ${{ inputs.ref }}
          HEAD_SHA: ${{ github.sha }}
          REPO: ${{ github.repository }}
        run: |
          set -euo pipefail
          if [ -z "$REF" ]; then
            echo "sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
            echo "Deploying main HEAD $HEAD_SHA"
            exit 0
          fi
          if ! printf '%s' "$REF" | grep -Eq '^[0-9a-f]{40}$'; then
            echo "::error title=ref::ref must be a full 40-character lowercase commit SHA (got: $REF)"
            exit 1
          fi
          FOUND=$(gh api "repos/$REPO/commits/$REF" --jq .sha 2>/dev/null || echo "")
          if [ "$FOUND" != "$REF" ]; then
            echo "::error title=ref::commit $REF is not in $REPO"
            exit 1
          fi
          echo "sha=$REF" >> "$GITHUB_OUTPUT"
          echo "::notice title=ref::Deploying $REF instead of main HEAD $HEAD_SHA"

      - name: Gate on the staging-smoke commit status
        env:
          GH_TOKEN: ${{ github.token }}
          SHA: ${{ steps.resolve.outputs.sha }}
          REPO: ${{ github.repository }}
          ALLOW: ${{ inputs.allow_without_staging_smoke }}
        run: |
          set -uo pipefail
          if [ "$ALLOW" = "true" ]; then
            echo "::warning title=staging-smoke gate::bypassed via allow_without_staging_smoke=true"
            exit 0
          fi
          # Latest staging-smoke by created_at, tiebreak id; if a failure and a success share the
          # newest created_at (same-second race) prefer failure, so the gate fails closed.
          STATE=$(gh api "repos/$REPO/commits/$SHA/statuses" \
            --jq '[.[]|select(.context=="staging-smoke")] | if length==0 then "" else ((max_by(.created_at).created_at) as $m | [.[]|select(.created_at==$m)] | (if any(.[]; .state=="failure") then "failure" else (sort_by(.id)|last|.state) end)) end' 2>/dev/null || echo "")
          echo "staging-smoke on $SHA = ${STATE:-<none>}"
          if [ "$STATE" != "success" ]; then
            echo "::error title=staging-smoke gate::no green staging-smoke on $SHA (got: ${STATE:-none}). Deploy Staging on this SHA first (it runs the Tier-1 smoke and posts the status), or re-dispatch with allow_without_staging_smoke=true for an emergency."
            exit 1
          fi

  deploy:
    name: Build + push + helm upgrade
    needs: require_staging_smoke
    runs-on: ubuntu-latest
    environment: dev
    timeout-minutes: 30

    steps:
      - name: Checkout (with submodules for _external/clawdbot)
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.require_staging_smoke.outputs.sha }}
          submodules: recursive

      - name: Compute image tag
        id: tag
        run: |
          if [ -n "${{ inputs.image_tag }}" ]; then
            echo "tag=${{ inputs.image_tag }}" >> "$GITHUB_OUTPUT"
          else
            echo "tag=$(git rev-parse --short=8 HEAD)" >> "$GITHUB_OUTPUT"
          fi

      - name: Export PROJECT_ID from secret
        env:
          DEV_GCP_PROJECT_ID: ${{ secrets.DEV_GCP_PROJECT_ID }}
        run: echo "PROJECT_ID=${DEV_GCP_PROJECT_ID}" >> "$GITHUB_ENV"

      - name: Authenticate to GCP via WIF
        uses: google-github-actions/auth@v2
        with:
          workload_identity_provider: ${{ secrets.WIF_PROVIDER }}
          service_account: ${{ secrets.WIF_SERVICE_ACCOUNT }}

      - name: Set up gcloud
        uses: google-github-actions/setup-gcloud@v2

      - name: Configure Docker for Artifact Registry
        run: gcloud auth configure-docker ${REGION}-docker.pkg.dev --quiet

      - name: Install Helm
        uses: azure/setup-helm@v4
        with:
          version: v3.14.0

      - name: Materialize private values overlay (runner-only)
        # DEV_HELM_VALUES_PRIVATE holds real GCP project ID / PG host / AR repos
        # / ESO SA email. Kept out of the OSS repo per sensitive-data policy.
        # Written to $RUNNER_TEMP, auto-cleaned when the runner teardowns.
        run: |
          printf '%s\n' "${DEV_HELM_VALUES_PRIVATE}" > "$RUNNER_TEMP/values-private.yaml"
          echo "Wrote $(wc -l < "$RUNNER_TEMP/values-private.yaml") lines to $RUNNER_TEMP/values-private.yaml"
        env:
          DEV_HELM_VALUES_PRIVATE: ${{ secrets.DEV_HELM_VALUES_PRIVATE }}

      # clawdbot-gateway is parked (agents.clawdbot.enabled: false since #1051),
      # and the chart then renders no gateway Deployment at all. Building it
      # anyway cost 3.3 of the ~6.4 build minutes per deploy and pushed a
      # ~1.1 GB image nothing ran: 709 such versions by 2026-10-14, about two
      # thirds of the Artifact Registry bill. Ask the release itself, with the
      # same three values files the upgrade uses, so the build returns the
      # moment the flag does. "could not find template" is helm's answer for a
      # template that rendered empty; any other helm error fails the job rather
      # than reading as "disabled".
      - name: Is clawdbot-gateway enabled in this release?
        id: gateway
        run: |
          set -uo pipefail
          if err=$(helm template commonly-dev k8s/helm/commonly \
              -f k8s/helm/commonly/values.yaml \
              -f k8s/helm/commonly/values-dev.yaml \
              -f "$RUNNER_TEMP/values-private.yaml" \
              --show-only templates/agents/clawdbot-deployment.yaml 2>&1 >/dev/null); then
            echo "enabled=true" >> "$GITHUB_OUTPUT"
          elif printf '%s' "$err" | grep -q 'could not find template'; then
            echo "enabled=false" >> "$GITHUB_OUTPUT"
            echo "clawdbot-gateway is disabled in this release; skipping its build and tag."
          else
            printf '%s\n' "$err"
            exit 1
          fi

      - name: Build + push backend image
        env:
          TAG: ${{ steps.tag.outputs.tag }}
        run: |
          REPO=${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/commonly-backend
          docker build backend -t "$REPO:$TAG"
          docker push "$REPO:$TAG"

      - name: Build + push frontend image
        env:
          TAG: ${{ steps.tag.outputs.tag }}
          REACT_APP_SENTRY_DSN: ${{ secrets.DEV_SENTRY_FRONTEND_DSN }}
        run: |
          REPO=${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/commonly-frontend
          docker build frontend \
            --build-arg REACT_APP_API_URL=https://api.commonly.me \
            --build-arg REACT_APP_SHOWCASE_POD_ID=6a507c9b792f1ed2cbfec648 \
            --build-arg REACT_APP_COMMUNITY_POD_ID=6a5fe677306155f677c26abf \
            --build-arg REACT_APP_COMMUNITY_INVITE_TOKEN=7b91255f18ae3c0ae3721707a6613731 \
            --build-arg REACT_APP_TELEGRAM_BOT_HANDLE=Commonly240Bot \
            --build-arg "REACT_APP_VERSION=$TAG" \
            --build-arg "REACT_APP_SENTRY_DSN=$REACT_APP_SENTRY_DSN" \
            -t "$REPO:$TAG"
          docker push "$REPO:$TAG"

      - name: Build + push clawdbot-gateway image
        if: steps.gateway.outputs.enabled == 'true'
        env:
          TAG: ${{ steps.tag.outputs.tag }}
        run: |
          # Stage commonly-bundled-skills/ into the gateway build context
          # so the Dockerfile's COPY commonly-bundled-skills /opt/... layer
          # picks up the latest skill content. Sidesteps the kubectl-exec
          # ARG_MAX limit that the runtime sync would otherwise hit when
          # bundles carry sub-files (officecli specialized sub-skills, etc.).
          rm -rf _external/clawdbot/commonly-bundled-skills
          cp -r backend/commonly-bundled-skills _external/clawdbot/commonly-bundled-skills

          REPO=${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/clawdbot-gateway
          docker build _external/clawdbot \
            --build-arg OPENCLAW_EXTENSIONS=acpx \
            --build-arg OPENCLAW_INSTALL_GH_CLI=1 \
            --build-arg OPENCLAW_INSTALL_DOC_TOOLCHAIN=1 \
            -t "$REPO:$TAG"
          docker push "$REPO:$TAG"

          # Cleanup so a re-run sees a deterministic build context.
          rm -rf _external/clawdbot/commonly-bundled-skills

      - name: Build + push commonly-bot image
        env:
          TAG: ${{ steps.tag.outputs.tag }}
        run: |
          REPO=${REGION}-docker.pkg.dev/${PROJECT_ID}/${AR_REPO}/commonly-bot
          docker build external/commonly-agent-services -t "$REPO:$TAG"
          docker push "$REPO:$TAG"

      - name: Get GKE credentials
        uses: google-github-actions/get-gke-credentials@v2
        with:
          cluster_name: ${{ env.CLUSTER }}
          location: ${{ env.REGION }}

      # `--wait` is deliberately NOT used here. It waits for EVERY resource in
      # the release, so any long-unhealthy auxiliary workload fails a deploy
      # that actually succeeded. On 2026-08-04 that produced five consecutive
      # false failures: `litellm` (CrashLoopBackOff, 561 restarts over 2d10h)
      # and `cloud-codex-cody` (Init:CrashLoopBackOff) never become ready, so
      # `--wait` burned the full 10m timeout and exited non-zero while
      # backend/frontend/clawdbot-gateway/commonly-bot had all rolled out
      # correctly minutes earlier.
      #
      # The error it surfaces — `client rate limiter Wait returned an error:
      # context deadline exceeded` — reads like throttling and is not: the
      # release already runs --burst-limit 200 --qps 100 (#245). It is the
      # context deadline expiring while waiting on pods that will never be
      # ready. Raising the limiter cannot fix it.
      #
      # The cost of the false failure was not the red check. It was that helm
      # records a failed upgrade, so the newest revision helm considers
      # `deployed` went stale by days — and a bare `helm rollback` during an
      # incident silently reverts every deploy since, which by then included
      # two auth fixes. Rollback safety depends on this job telling the truth.
      - name: Helm upgrade commonly-dev
        env:
          TAG: ${{ steps.tag.outputs.tag }}
          GATEWAY_ENABLED: ${{ steps.gateway.outputs.enabled }}
        run: |
          # Point the gateway at this run's tag only when this run built it.
          GATEWAY_TAG=()
          if [ "$GATEWAY_ENABLED" = "true" ]; then
            GATEWAY_TAG=(--set agents.clawdbot.image.tag="$TAG")
          fi
          helm upgrade commonly-dev k8s/helm/commonly -n "$NAMESPACE" \
            -f k8s/helm/commonly/values.yaml \
            -f k8s/helm/commonly/values-dev.yaml \
            -f "$RUNNER_TEMP/values-private.yaml" \
            --set backend.image.tag="$TAG" \
            --set frontend.image.tag="$TAG" \
            "${GATEWAY_TAG[@]}" \
            --set agents.commonlyBot.image.tag="$TAG" \
            --timeout 10m \
            --burst-limit 200 \
            --qps 100

      # Verify exactly what this job deploys: the four workloads whose image
      # tags it sets above. Scoping the check to them is the point — an
      # unrelated unhealthy workload is a real problem, but it is not THIS
      # job's failure, and conflating the two is what made the signal
      # worthless. Anything outside this list is deliberately not gated here;
      # it needs its own alerting rather than a deploy that cries wolf.
      #
      # `kubectl rollout status` is the honest instrument: it returns non-zero
      # on a genuine stuck rollout, so a real regression still fails the job.
      #
      # The list is DERIVED from the cluster, never hardcoded. The hardcoded
      # version waited 8 minutes for clawdbot-gateway after #1051 parked it
      # (enabled: false removes the Deployment entirely), which made every
      # deploy go red while succeeding — alarm fatigue over the exact signal
      # this step exists to give. Deriving also means a future workload joins
      # the gate the moment its chart ships it, and a parked one leaves it the
      # moment it is disabled — the gate follows the release, not a memory of
      # it. A deployment scaled to zero passes instantly (0 of 0 updated),
      # which is correct: parked is not stuck.
      - name: Verify rollout of the deployed workloads
        id: rollout
        run: |
          set -uo pipefail
          FAILED=""
          DEPLOYS=$(kubectl get deploy -n "$NAMESPACE" -o name)
          if [ -z "$DEPLOYS" ]; then
            echo "::error title=Rollout failed::no deployments found in $NAMESPACE — nothing to verify is itself a failure"
            exit 1
          fi
          for d in $DEPLOYS; do
            echo "::group::rollout status $d"
            if kubectl rollout status "$d" -n "$NAMESPACE" --timeout=8m; then
              echo "$d: rolled out"
            else
              echo "::error title=Rollout failed::$d did not become ready within 8m"
              FAILED="$FAILED $d"
            fi
            echo "::endgroup::"
          done
          if [ -n "$FAILED" ]; then
            echo "Deploy failed for:$FAILED"
            exit 1
          fi
          echo "All four deployed workloads are live."

      # A pod can be Ready and still unable to serve chat. The backend decides
      # once at boot whether Postgres is available, and /ready answers 200 in
      # that degraded state. On 2026-09-25 a transient connect timeout at boot
      # left /api/pg/messages unmounted on the only replica, chat history 404
      # for minutes, and this job green. Until readiness itself asserts it
      # (TASK-168), ask the newest backend pod from the inside. Unauthenticated,
      # a mounted /api/pg/messages answers 401 and an unmounted one 404.
      - name: Verify the new backend serves chat
        # Deliberately not `always()` / `!cancelled()`: if an earlier step failed
        # (image push, helm upgrade, GKE credentials) the cluster is unreachable
        # or unchanged, and this step would then report "Chat unavailable on the
        # new backend" for a reason that has nothing to do with chat. The case
        # that needs it is the one below — the rollout gate failing on its own.
        #
        # 2026-09-27: a stuck rollout (readiness refusing a pod that was in fact
        # serving 401s) exited 1 inside the rollout step, so this step was
        # skipped and the operator got "backend did not become ready within 8m"
        # with no 401-vs-404 discriminator attached. That discriminator is what
        # separates "readiness is lying about a working pod" from "the pod is
        # broken", and it was absent exactly when it was needed.
        if: ${{ success() || steps.rollout.outcome == 'failure' }}
        run: |
          set -uo pipefail
          POD=$(kubectl get pod -n "$NAMESPACE" -l app=backend \
            -o jsonpath='{range .items[*]}{.status.startTime}{" "}{.metadata.name}{" "}{.metadata.deletionTimestamp}{"\n"}{end}' \
            | awk 'NF == 2' | sort | tail -1 | awk '{print $2}')
          if [ -z "$POD" ]; then
            echo "::error title=Chat check failed::no running backend pod found"
            exit 1
          fi
          echo "Checking $POD"
          for attempt in 1 2 3 4 5 6; do
            STATUS=$(kubectl exec -n "$NAMESPACE" "$POD" -c backend -- curl -s http://localhost:5000/api/pg/status || true)
            ROUTE=$(kubectl exec -n "$NAMESPACE" "$POD" -c backend -- curl -s -o /dev/null -w '%{http_code}' http://localhost:5000/api/pg/messages/000000000000000000000000 || true)
            echo "attempt $attempt: pg/status=$STATUS pg/messages=$ROUTE"
            if [[ "$STATUS" == *'"available":true'* ]] && [ "$ROUTE" = "401" ]; then
              echo "$POD serves chat."
              exit 0
            fi
            sleep 10
          done
          echo "::error title=Chat unavailable on the new backend::$POD has pg/status=$STATUS and pg/messages=$ROUTE (401 means mounted, 404 unmounted). Chat history fails until the pod restarts: kubectl rollout restart deploy/backend -n $NAMESPACE"
          exit 1

      # The stranger journey walks signup -> Scout's welcome -> a real question
      # -> the connect card against the release just deployed
      # (docs/runbooks/alerting.md#stranger-journey). It runs as a Job from the
      # same CronJob as the daily run, so its result lands in Cloud Logging and
      # pages the same way. This step waits so the deploy log carries the
      # verdict, but never fails the deploy on it: the release is already
      # serving, and the alert is the response path.
      - name: Run the stranger journey against this release
        if: success()
        env:
          TAG: ${{ steps.tag.outputs.tag }}
        run: |
          set -uo pipefail
          if ! kubectl get cronjob stranger-journey -n "$NAMESPACE" >/dev/null 2>&1; then
            echo "No stranger-journey CronJob in this release; skipping."
            exit 0
          fi
          JOB="stranger-journey-deploy-${TAG}-${GITHUB_RUN_ATTEMPT}"
          kubectl create job --from=cronjob/stranger-journey "$JOB" -n "$NAMESPACE"
          for _ in $(seq 1 60); do
            DONE=$(kubectl get job "$JOB" -n "$NAMESPACE" -o jsonpath='{.status.succeeded}{.status.failed}')
            [ -n "$DONE" ] && break
            sleep 10
          done
          kubectl logs "job/$JOB" -n "$NAMESPACE" --tail=12 || true
          if [ "$(kubectl get job "$JOB" -n "$NAMESPACE" -o jsonpath='{.status.succeeded}')" = "1" ]; then
            echo "::notice title=Stranger journey::passed on $TAG"
          else
            echo "::warning title=Stranger journey did not pass on $TAG::read the job log above; the journey alert pages on a failed step"
          fi

      # Unhealthy workloads outside the deployed set are surfaced, never
      # failed on. Silence here is how litellm accumulated 561 restarts
      # without anyone noticing; failing on it is how five good deploys got
      # marked bad. A notice is the correct middle.
      - name: Note any unhealthy workloads outside the deployed set
        if: always()
        run: |
          set -uo pipefail
          UNHEALTHY=$(kubectl get deploy -n "$NAMESPACE" \
            -o jsonpath='{range .items[?(@.spec.replicas>0)]}{.metadata.name}{" "}{.status.readyReplicas}{"\n"}{end}' \
            | awk '$2 == "" || $2 == 0 { printf "%s ", $1 }')
          if [ -n "$UNHEALTHY" ]; then
            echo "::notice title=Unhealthy workloads (not deployed by this job)::$UNHEALTHY"
          else
            echo "No unhealthy workloads in $NAMESPACE."
          fi

      - name: Report deploy outcome
        if: always()
        env:
          TAG: ${{ steps.tag.outputs.tag }}
          STATUS: ${{ job.status }}
        run: |
          HELM_REV=$(helm history commonly-dev -n "$NAMESPACE" -o json 2>/dev/null | jq -r '.[-1].revision // "unknown"')
          echo "::notice title=Dev deploy::status=$STATUS tag=$TAG helm_revision=$HELM_REV"