name: Triggered Integration Test

# ------------------------------------------------------------------------------
# Maintainer-triggered integration test of Grid main, a pull request, or a branch,
# plus nightly GLB quick and combined-site full runs against main. Manual runs
# and reusable workflow calls can select a supported release qualification.
#
# Runs the selected qualification against the tested revision. GLB verifies
# multi-cluster routing, SWIM discovery, mTLS, overlay hot-reload, and provider
# selection; the other choices exercise their release-specific topology.
# ------------------------------------------------------------------------------

on:
  workflow_dispatch:
    inputs:
      pr_number:
        description: Grid pull-request number; mutually exclusive with branch
        required: false
        default: ""
        type: string
      branch:
        description: Branch in this repository to test; leave blank for a PR or main
        required: false
        default: ""
        type: string
      release_qualification:
        description: Release qualification and supported mode to run
        required: true
        default: glb-quick
        type: choice
        options:
          - glb-quick
          - glb-full
          - static-weighted
          - combined-site
          - combined-site-local
          - llmd-pool-metrics-quick
          - llmd-pool-metrics-full
          - operator-routing
      operator_image_mode:
        description: Operator image profile; FIPS runtime requires a FIPS-enabled runner
        required: false
        default: standard
        type: choice
        options:
          - standard
          - fips
  workflow_call:
    inputs:
      release_qualification:
        description: Release qualification and supported mode to run
        required: true
        type: string
      pr_number:
        description: Grid pull-request number; mutually exclusive with branch
        required: false
        type: string
        default: ""
      branch:
        description: Branch in this repository to test; leave blank for a PR or main
        required: false
        type: string
        default: ""
      operator_image_mode:
        description: Operator image profile; FIPS runtime requires a FIPS-enabled runner
        required: false
        type: string
        default: standard
  schedule:
    # Nightly GLB quick qualification.
    - cron: "0 18 * * *"
    # Nightly combined-site qualification (mapped to full mode below).
    - cron: "0 19 * * *"

concurrency:
  group: grid-triggered-integration-${{ github.workflow }}-${{ inputs.release_qualification || 'scheduled' }}-${{ inputs.pr_number != '' && format('pr-{0}', inputs.pr_number) || inputs.branch != '' && format('branch-{0}', inputs.branch) || 'main' }}-${{ github.event_name }}-${{ github.event_name == 'schedule' && github.event.schedule || '' }}
  cancel-in-progress: true

permissions: {}

env:
  CARGO_TERM_COLOR: always
  RUST_VERSION: "1.96.0"
  KIND_VERSION: v0.32.0
  KUBECTL_VERSION: v1.32.3
  # The gateway image MUST be pinned by digest for reproducible evidence.
  # To obtain the digest for a new tag:
  #   docker pull ghcr.io/praxis-proxy/ai:<tag>
  #   docker inspect --format='{{index .RepoDigests 0}}' ghcr.io/praxis-proxy/ai:<tag>
  GATEWAY_IMAGE_REF: ghcr.io/praxis-proxy/ai:0.7.0
  GATEWAY_IMAGE_DIGEST: sha256:550661eea7e392a9f4415b836e0b2393299f26856bc02cbbc8d3301d49b759f3
  # Forge topologies run with imagePullPolicy=Never, so pull immutable simulator
  # references and materialize the expected local tags before loading Kind.
  SIM_IMAGE_REF: ghcr.io/neuralmagic/vllm-vcr@sha256:cf98f91f12210893f63b3afd3b1ea5bd3c35640379ce2bbf702a4cefddf9e5a4
  SIM_IMAGE_TAG: ghcr.io/neuralmagic/vllm-vcr:vllm0.23
  STATIC_WEIGHTED_SIM_IMAGE_REF: ghcr.io/llm-d/llm-d-inference-sim@sha256:7f3a1f72875c5dd5d00299ad358dc1f6e17041a5124609a43aa17ad318bbed32
  STATIC_WEIGHTED_SIM_IMAGE_TAG: ghcr.io/llm-d/llm-d-inference-sim:v0.10.2
  COMBINED_SITE_SIM_IMAGE_REF: ghcr.io/neuralmagic/vllm-vcr@sha256:cf98f91f12210893f63b3afd3b1ea5bd3c35640379ce2bbf702a4cefddf9e5a4
  COMBINED_SITE_SIM_IMAGE_TAG: ghcr.io/neuralmagic/vllm-vcr:vllm0.23
  LLMD_SIM_IMAGE_REF: ghcr.io/llm-d/llm-d-inference-sim@sha256:7f3a1f72875c5dd5d00299ad358dc1f6e17041a5124609a43aa17ad318bbed32
  LLMD_SIM_IMAGE_TAG: ghcr.io/llm-d/llm-d-inference-sim:v0.10.2
  LLMD_EPP_IMAGE_REF: ghcr.io/llm-d/llm-d-router-endpoint-picker@sha256:39257d24552d16d8f215d3de25cc734fd44af3b282b67032db01aff900c610cf
  LLMD_EPP_IMAGE_TAG: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.11.0
  # Pin the linux/amd64 child manifest. Kind imports every platform from a
  # multi-platform index, while this workflow runs amd64 nodes and only pulls
  # the runner's platform into Docker.
  PROBE_IMAGE_REF: curlimages/curl@sha256:fcff5cf7a4b895da7bd2933c914938db2b05d2113fa0d6c55b6d29930408f661
  PROBE_IMAGE_TAG: curlimages/curl:8.10.1
  # Pinned identity/date/message-suffix for the local PR-merge reconstruction
  # in resolve-source's "Resolve source" step and qualification-e2e's "Reconstruct PR
  # merge" step (see grid#66). Both jobs build a merge commit from the same
  # base_sha/head_sha pair and must produce a byte-identical SHA, which
  # requires byte-identical author/committer identity, date, and message --
  # a fixed, meaningless-but-consistent value is fine since this commit is a
  # CI-internal integrity check, never published or citable.
  MERGE_COMMIT_NAME: grid-ci-merge-check
  MERGE_COMMIT_EMAIL: grid-ci-merge-check@users.noreply.github.com
  MERGE_COMMIT_DATE: "1970-01-01T00:00:00Z"
  MERGE_COMMIT_MESSAGE_SUFFIX: grid-ci deterministic merge check

jobs:
  # ============================================================================
  # Resolve the source revision to test
  # ============================================================================

  resolve-source:
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    permissions:
      contents: read
      pull-requests: read
    outputs:
      source_kind: ${{ steps.source.outputs.source_kind }}
      source_label: ${{ steps.source.outputs.source_label }}
      tested_sha: ${{ steps.source.outputs.tested_sha }}
      checkout_sha: ${{ steps.source.outputs.checkout_sha }}
      branch_name: ${{ steps.source.outputs.branch_name }}
      pr_number: ${{ steps.pr.outputs.pr_number }}
      title: ${{ steps.pr.outputs.title }}
      url: ${{ steps.pr.outputs.url }}
      author: ${{ steps.pr.outputs.author }}
      draft: ${{ steps.pr.outputs.draft }}
      head_repo: ${{ steps.pr.outputs.head_repo }}
      head_branch: ${{ steps.pr.outputs.head_branch }}
      head_sha: ${{ steps.pr.outputs.head_sha }}
      base_sha: ${{ steps.source.outputs.base_sha }}
      release_qualification: ${{ steps.selection.outputs.release_qualification }}
      qualification: ${{ steps.mode.outputs.qualification }}
      mode: ${{ steps.mode.outputs.mode }}
      membership: ${{ steps.mode.outputs.membership }}
      operator_image_mode: ${{ steps.operator_image.outputs.mode }}
    steps:
      - name: Select release qualification
        id: selection
        env:
          EVENT_NAME: ${{ github.event_name }}
          SCHEDULE: ${{ github.event.schedule }}
          INPUT_RELEASE_QUALIFICATION: ${{ inputs.release_qualification }}
        run: |
          set -euo pipefail
          if [[ "${EVENT_NAME}" == "schedule" ]]; then
            case "${SCHEDULE}" in
              "0 18 * * *")
                RELEASE_QUALIFICATION=glb-quick
                ;;
              "0 19 * * *")
                RELEASE_QUALIFICATION=combined-site
                ;;
              *)
                echo "::error::No release qualification is mapped for schedule '${SCHEDULE}'"
                exit 1
                ;;
            esac
          else
            RELEASE_QUALIFICATION="${INPUT_RELEASE_QUALIFICATION:-glb-quick}"
          fi
          echo "release_qualification=${RELEASE_QUALIFICATION}" >> "${GITHUB_OUTPUT}"
          echo "Release qualification: ${RELEASE_QUALIFICATION}"

      - name: Resolve qualification mode
        id: mode
        env:
          RELEASE_QUALIFICATION: ${{ steps.selection.outputs.release_qualification }}
        run: |
          set -euo pipefail
          MEMBERSHIP=swim
          case "${RELEASE_QUALIFICATION}" in
            glb-quick)
              QUALIFICATION=glb
              MODE=quick
              ;;
            glb-full)
              QUALIFICATION=glb
              MODE=full
              ;;
            static-weighted)
              QUALIFICATION=static-weighted
              MODE=quick
              ;;
            combined-site)
              QUALIFICATION=combined-site
              MODE=full
              ;;
            combined-site-local)
              QUALIFICATION=combined-site
              MODE=full
              MEMBERSHIP=local
              ;;
            llmd-pool-metrics-quick)
              QUALIFICATION=llmd-pool-metrics
              MODE=quick
              ;;
            llmd-pool-metrics-full)
              QUALIFICATION=llmd-pool-metrics
              MODE=full
              ;;
            operator-routing)
              QUALIFICATION=operator-routing MODE=quick
              ;;
            *)
              echo "::error::Unknown release qualification '${RELEASE_QUALIFICATION}'"
              exit 1
              ;;
          esac
          {
            echo "qualification=${QUALIFICATION}"
            echo "mode=${MODE}"
            echo "membership=${MEMBERSHIP}"
          } >> "${GITHUB_OUTPUT}"
          echo "Release qualification: ${RELEASE_QUALIFICATION} -> ${QUALIFICATION} (${MODE})"

      - name: Select operator image mode
        id: operator_image
        env:
          EVENT_NAME: ${{ github.event_name }}
          INPUT_OPERATOR_IMAGE_MODE: ${{ inputs.operator_image_mode }}
          QUALIFICATION: ${{ steps.mode.outputs.qualification }}
        run: |
          set -euo pipefail
          if [[ "${EVENT_NAME}" == "schedule" ]]; then
            OPERATOR_IMAGE_MODE=standard
          else
            OPERATOR_IMAGE_MODE="${INPUT_OPERATOR_IMAGE_MODE:-standard}"
          fi
          case "${OPERATOR_IMAGE_MODE}" in
            standard|fips) ;;
            *)
              echo "::error::Unsupported operator image mode '${OPERATOR_IMAGE_MODE}' (expected standard or fips)"
              exit 1
              ;;
          esac
          if [[ "${OPERATOR_IMAGE_MODE}" == "fips" && "${QUALIFICATION}" != "combined-site" ]]; then
            echo "::error::FIPS operator mode is currently supported only for the combined-site qualification."
            exit 1
          fi
          echo "mode=${OPERATOR_IMAGE_MODE}" >> "${GITHUB_OUTPUT}"
          echo "Operator image mode: ${OPERATOR_IMAGE_MODE}"

      - name: Validate source selector
        env:
          PR_NUM: ${{ inputs.pr_number }}
          BRANCH_NAME: ${{ inputs.branch }}
        run: |
          set -euo pipefail
          if [[ -n "${PR_NUM}" && -n "${BRANCH_NAME}" ]]; then
            echo "::error::Choose either pr_number or branch, not both"
            exit 1
          fi
          if [[ -n "${PR_NUM}" ]] && ! [[ "${PR_NUM}" =~ ^[1-9][0-9]*$ ]]; then
            echo "::error::pr_number must be a positive decimal integer, got: '${PR_NUM}'"
            exit 1
          fi
          if [[ -n "${BRANCH_NAME}" ]]; then
            if ! [[ "${BRANCH_NAME}" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] \
              || ! git check-ref-format --branch "${BRANCH_NAME}" >/dev/null; then
              echo "::error::branch must be a valid simple branch name in this repository"
              exit 1
            fi
          fi

      - name: Query PR metadata
        if: inputs.pr_number != ''
        id: pr
        env:
          GH_TOKEN: ${{ github.token }}
          PR_NUM: ${{ inputs.pr_number }}
        run: |
          set -euo pipefail
          PR_JSON=""
          PR_JSON=$(gh api "repos/${{ github.repository }}/pulls/${PR_NUM}" 2>&1) || {
            echo "::error::Failed to fetch PR #${PR_NUM}. Does the PR exist?"
            echo "::error::API response: ${PR_JSON}"
            exit 1
          }

          STATE=$(echo "${PR_JSON}" | jq -r '.state')
          if [[ "${STATE}" != "open" ]]; then
            echo "::error::PR #${PR_NUM} is '${STATE}', expected 'open'"
            exit 1
          fi

          BASE_BRANCH=$(echo "${PR_JSON}" | jq -r '.base.ref')
          if [[ "${BASE_BRANCH}" != "main" ]]; then
            echo "::error::PR #${PR_NUM} targets '${BASE_BRANCH}', expected 'main'"
            exit 1
          fi

          TITLE=$(echo "${PR_JSON}" | jq -r '.title' | head -c 200 | tr -d '\n\r')
          {
            echo "pr_number=${PR_NUM}"
            echo "title=${TITLE}"
            echo "url=$(echo "${PR_JSON}" | jq -r '.html_url')"
            echo "author=$(echo "${PR_JSON}" | jq -r '.user.login')"
            echo "draft=$(echo "${PR_JSON}" | jq -r '.draft')"
            echo "head_repo=$(echo "${PR_JSON}" | jq -r '.head.repo.full_name')"
            echo "head_branch=$(echo "${PR_JSON}" | jq -r '.head.ref')"
            echo "head_sha=$(echo "${PR_JSON}" | jq -r '.head.sha')"
          } >> "${GITHUB_OUTPUT}"

          # NOTE: intentionally not capturing `.base.sha` here as the
          # verified base: GitHub only refreshes this REST field on PR
          # synchronize events (e.g. a push to the head branch), so it can
          # lag `main`'s actual tip by hours if the PR sits idle while main
          # moves forward. The base used for verification is instead
          # re-resolved live from refs/heads/main in the "Resolve source"
          # step below (see grid#66 for why that step also stopped trusting
          # GitHub's precomputed refs/pull/<n>/merge for the same reason).
          echo "${PR_JSON}" | jq '{
            number: .number,
            title: .title,
            html_url: .html_url,
            state: .state,
            draft: .draft,
            author: .user.login,
            head_repo: .head.repo.full_name,
            head_branch: .head.ref,
            head_sha: .head.sha,
            base_branch: .base.ref,
            base_sha_at_pr_query: .base.sha
          }' > source-metadata.json

          echo "::group::PR metadata"
          cat source-metadata.json
          echo "::endgroup::"

      - name: Resolve source
        id: source
        env:
          PR_NUM: ${{ inputs.pr_number }}
          BRANCH_NAME: ${{ inputs.branch }}
          HEAD_SHA: ${{ steps.pr.outputs.head_sha }}
        run: |
          set -euo pipefail
          REPO_URL="https://github.com/${{ github.repository }}.git"

          if [[ -n "${PR_NUM}" ]]; then
            # Resolve refs/heads/main live instead of trusting the PR REST
            # resource's `.base.sha` (see note in "Query PR metadata"),
            # which can lag main's actual tip by hours while a PR sits idle.
            BASE_SHA=$(git ls-remote "${REPO_URL}" refs/heads/main | awk '{print $1}')
            if [[ -z "${BASE_SHA}" ]]; then
              echo "::error::Cannot resolve refs/heads/main for base verification"
              exit 1
            fi

            # Construct the merge locally from the two pinned commit SHAs
            # instead of trusting GitHub's asynchronously-regenerated
            # refs/pull/<n>/merge: that ref is only recomputed on a push to
            # the PR branch, a merge-base change, or a 12h max-age timer,
            # and can be observed lagging main's live tip well past the
            # window either of those give (see grid#66 for a live-reproduced
            # example -- unchanged 49+ minutes after main advanced). A full
            # (unshallow) clone guarantees `git merge` can find the true
            # merge-base between base and head regardless of how far apart
            # they are; qualification-e2e's "Reconstruct PR merge" step repeats this
            # exact reconstruction (same inputs, same pinned identity/date/
            # message below) to independently arrive at the identical SHA
            # this step verifies it against.
            git clone --quiet "${REPO_URL}" merge-check
            git -C merge-check fetch --quiet origin "${BASE_SHA}" "${HEAD_SHA}"
            git -C merge-check checkout --quiet "${BASE_SHA}"

            MERGE_COMMIT_MESSAGE="Merge PR #${PR_NUM} head ${HEAD_SHA} into base ${BASE_SHA} (${MERGE_COMMIT_MESSAGE_SUFFIX})"
            if ! GIT_AUTHOR_NAME="${MERGE_COMMIT_NAME}" GIT_AUTHOR_EMAIL="${MERGE_COMMIT_EMAIL}" GIT_AUTHOR_DATE="${MERGE_COMMIT_DATE}" \
                 GIT_COMMITTER_NAME="${MERGE_COMMIT_NAME}" GIT_COMMITTER_EMAIL="${MERGE_COMMIT_EMAIL}" GIT_COMMITTER_DATE="${MERGE_COMMIT_DATE}" \
                 git -C merge-check merge --no-ff --quiet -m "${MERGE_COMMIT_MESSAGE}" "${HEAD_SHA}"; then
              echo "::error::PR #${PR_NUM} (head ${HEAD_SHA}) does not merge cleanly onto main (${BASE_SHA})"
              exit 1
            fi
            MERGE_SHA=$(git -C merge-check rev-parse HEAD)

            {
              echo "source_kind=pr"
              echo "source_label=PR #${PR_NUM}"
              echo "tested_sha=${MERGE_SHA}"
              echo "checkout_sha=${BASE_SHA}"
              echo "base_sha=${BASE_SHA}"
            } >> "${GITHUB_OUTPUT}"
            echo "Resolved refs/heads/main -> ${BASE_SHA} (base)"
            echo "Constructed deterministic merge of PR #${PR_NUM} head ${HEAD_SHA} onto ${BASE_SHA} -> ${MERGE_SHA}"
          elif [[ -n "${BRANCH_NAME}" ]]; then
            BRANCH_REF="refs/heads/${BRANCH_NAME}"
            BRANCH_SHA=$(git ls-remote "${REPO_URL}" "${BRANCH_REF}" | awk -v ref="${BRANCH_REF}" '$2 == ref {print $1}')
            if [[ -z "${BRANCH_SHA}" ]]; then
              echo "::error::Cannot resolve ${BRANCH_REF} in ${REPO_URL}"
              exit 1
            fi
            {
              echo "source_kind=branch"
              echo "source_label=branch/${BRANCH_NAME}"
              echo "branch_name=${BRANCH_NAME}"
              echo "tested_sha=${BRANCH_SHA}"
              echo "checkout_sha=${BRANCH_SHA}"
            } >> "${GITHUB_OUTPUT}"
            echo "Resolved ${BRANCH_REF} -> ${BRANCH_SHA}"
          else
            MAIN_SHA=$(git ls-remote "${REPO_URL}" refs/heads/main | awk '{print $1}')
            if [[ -z "${MAIN_SHA}" ]]; then
              echo "::error::Cannot resolve refs/heads/main"
              exit 1
            fi
            {
              echo "source_kind=main"
              echo "source_label=main"
              echo "tested_sha=${MAIN_SHA}"
              echo "checkout_sha=${MAIN_SHA}"
            } >> "${GITHUB_OUTPUT}"
            echo "Resolved refs/heads/main -> ${MAIN_SHA}"
          fi

      - name: Restrict FIPS mode to repository sources
        if: steps.operator_image.outputs.mode == 'fips' && steps.source.outputs.source_kind == 'pr'
        run: |
          echo "::error::FIPS mode runs on a self-hosted FIPS runner and cannot execute PR merge code. Use a trusted repository branch or select standard mode."
          exit 1

      - name: Write main metadata
        if: inputs.pr_number == '' && inputs.branch == ''
        run: |
          set -euo pipefail
          jq -n \
            --arg source "main" \
            --arg repository "${{ github.repository }}" \
            --arg branch "main" \
            --arg sha "${{ steps.source.outputs.tested_sha }}" \
            '{source: $source, repository: $repository, branch: $branch, sha: $sha}' \
            > source-metadata.json
          echo "::group::Source metadata"
          cat source-metadata.json
          echo "::endgroup::"

      - name: Write branch metadata
        if: inputs.branch != ''
        env:
          BRANCH_NAME: ${{ inputs.branch }}
          BRANCH_SHA: ${{ steps.source.outputs.tested_sha }}
        run: |
          set -euo pipefail
          jq -n \
            --arg repository "${{ github.repository }}" \
            --arg branch "${BRANCH_NAME}" \
            --arg sha "${BRANCH_SHA}" \
            '{source: "branch", repository: $repository, branch: $branch, sha: $sha}' \
            > source-metadata.json
          echo "::group::Source metadata"
          cat source-metadata.json
          echo "::endgroup::"

      - name: Upload source metadata
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: source-metadata
          path: source-metadata.json
          retention-days: 14

  # ============================================================================
  # Selected release qualification against the tested Grid revision
  # ============================================================================

  qualification-e2e:
    name: ${{ needs.resolve-source.outputs.source_label }} ${{ needs.resolve-source.outputs.release_qualification }} E2E
    needs: resolve-source
    runs-on: ${{ needs.resolve-source.outputs.operator_image_mode == 'fips' && fromJSON('["self-hosted","linux","fips"]') || 'ubuntu-24.04' }}
    permissions:
      contents: read
      packages: read
    timeout-minutes: 120
    env:
      SOURCE_KIND: ${{ needs.resolve-source.outputs.source_kind }}
      SOURCE_LABEL: ${{ needs.resolve-source.outputs.source_label }}
      TESTED_SHA: ${{ needs.resolve-source.outputs.tested_sha }}
      PR_NUMBER: ${{ needs.resolve-source.outputs.pr_number }}
      HEAD_SHA: ${{ needs.resolve-source.outputs.head_sha }}
      BASE_SHA: ${{ needs.resolve-source.outputs.base_sha }}
      BRANCH_NAME: ${{ needs.resolve-source.outputs.branch_name }}
      QUALIFICATION: ${{ needs.resolve-source.outputs.qualification }}
      MODE: ${{ needs.resolve-source.outputs.mode }}
      MEMBERSHIP: ${{ needs.resolve-source.outputs.membership }}
      OPERATOR_IMAGE_MODE: ${{ needs.resolve-source.outputs.operator_image_mode }}
      EVIDENCE_DIR: ${{ github.workspace }}/evidence/${{ needs.resolve-source.outputs.release_qualification }}
      DIAGNOSTICS_DIR: ${{ github.workspace }}/diagnostics/${{ needs.resolve-source.outputs.release_qualification }}
    steps:
      # ----------------------------------------------------------------------
      # Checkout
      # ----------------------------------------------------------------------

      - name: Checkout Grid at base revision
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: ${{ github.repository }}
          ref: ${{ needs.resolve-source.outputs.checkout_sha }}
          path: grid
          fetch-depth: 0
          persist-credentials: false

      - name: Reconstruct PR merge
        if: needs.resolve-source.outputs.source_kind == 'pr'
        run: |
          set -euo pipefail
          REPO="${GITHUB_WORKSPACE}/grid"
          git -C "${REPO}" fetch --quiet origin "${HEAD_SHA}"

          # Must reproduce resolve-source's "Resolve source" step exactly
          # (same base_sha/head_sha, same pinned identity/date/message) so
          # the resulting commit SHA matches TESTED_SHA byte-for-byte --
          # that match is what "Verify PR merge checkout" below asserts.
          MERGE_COMMIT_MESSAGE="Merge PR #${PR_NUMBER} head ${HEAD_SHA} into base ${BASE_SHA} (${MERGE_COMMIT_MESSAGE_SUFFIX})"
          if ! GIT_AUTHOR_NAME="${MERGE_COMMIT_NAME}" GIT_AUTHOR_EMAIL="${MERGE_COMMIT_EMAIL}" GIT_AUTHOR_DATE="${MERGE_COMMIT_DATE}" \
               GIT_COMMITTER_NAME="${MERGE_COMMIT_NAME}" GIT_COMMITTER_EMAIL="${MERGE_COMMIT_EMAIL}" GIT_COMMITTER_DATE="${MERGE_COMMIT_DATE}" \
               git -C "${REPO}" merge --no-ff --quiet -m "${MERGE_COMMIT_MESSAGE}" "${HEAD_SHA}"; then
            echo "::error::Reconstructing the merge of PR #${PR_NUMBER} head ${HEAD_SHA} onto base ${BASE_SHA} failed"
            echo "::error::This mirrors resolve-source's own merge, which already succeeded earlier in this run"
            exit 1
          fi

      - name: Verify PR merge checkout
        if: needs.resolve-source.outputs.source_kind == 'pr'
        run: |
          set -euo pipefail
          REPO="${GITHUB_WORKSPACE}/grid"
          ACTUAL=$(git -C "${REPO}" rev-parse HEAD)
          PARENT_COUNT=$(git -C "${REPO}" rev-list --parents -n 1 HEAD | awk '{print NF - 1}')
          ACTUAL_BASE=$(git -C "${REPO}" rev-parse HEAD^1)
          ACTUAL_HEAD=$(git -C "${REPO}" rev-parse HEAD^2)

          echo "::group::Verified Grid source checkout"
          echo "Repository:       ${{ github.repository }}"
          echo "Requested PR:     #${PR_NUMBER}"
          echo "PR head branch:   ${{ needs.resolve-source.outputs.head_repo }}:${{ needs.resolve-source.outputs.head_branch }}"
          echo "Expected PR head: ${HEAD_SHA}"
          echo "Expected base:    ${BASE_SHA}"
          echo "Tested merge SHA: ${TESTED_SHA}"
          echo "Checked-out HEAD: ${ACTUAL}"
          echo "Merge parent 1:   ${ACTUAL_BASE} (base)"
          echo "Merge parent 2:   ${ACTUAL_HEAD} (PR head)"
          git -C "${REPO}" status --short --branch
          git -C "${REPO}" log -1 --format='Commit: %H%nParents: %P%nSubject: %s'
          echo "::endgroup::"

          if [[ "${ACTUAL}" != "${TESTED_SHA}" ]]; then
            echo "::error::Merge SHA changed between resolve and checkout"
            echo "::error::Expected ${TESTED_SHA}, got ${ACTUAL}"
            exit 1
          fi
          if [[ "${PARENT_COUNT}" -ne 2 ]]; then
            echo "::error::Expected a two-parent PR merge commit, found ${PARENT_COUNT} parent(s)"
            exit 1
          fi
          if [[ "${ACTUAL_BASE}" != "${BASE_SHA}" || "${ACTUAL_HEAD}" != "${HEAD_SHA}" ]]; then
            echo "::error::Checked-out merge does not combine the expected base and PR head"
            echo "::error::Expected parents: ${BASE_SHA} ${HEAD_SHA}"
            echo "::error::Actual parents:   ${ACTUAL_BASE} ${ACTUAL_HEAD}"
            exit 1
          fi
          if [[ "${ACTUAL}" == "${BASE_SHA}" || "${ACTUAL}" == "${HEAD_SHA}" ]]; then
            echo "::error::Checked-out commit is not the PR merge commit"
            exit 1
          fi
          echo "::notice::PASS: testing PR #${PR_NUMBER} merge ${TESTED_SHA}, containing PR head ${HEAD_SHA}; this is not main/base ${BASE_SHA}"

      - name: Verify main checkout
        if: needs.resolve-source.outputs.source_kind == 'main'
        run: |
          set -euo pipefail
          ACTUAL=$(git -C "${GITHUB_WORKSPACE}/grid" rev-parse HEAD)

          echo "::group::Verified Grid source checkout"
          echo "Source:           main"
          echo "Expected SHA:     ${TESTED_SHA}"
          echo "Checked-out HEAD: ${ACTUAL}"
          git -C "${GITHUB_WORKSPACE}/grid" log -1 --format='Commit: %H%nSubject: %s'
          echo "::endgroup::"

          if [[ "${ACTUAL}" != "${TESTED_SHA}" ]]; then
            echo "::error::Main SHA changed between resolve and checkout"
            echo "::error::Expected ${TESTED_SHA}, got ${ACTUAL}"
            exit 1
          fi
          echo "::notice::PASS: testing main at ${ACTUAL}"

      - name: Verify branch checkout
        if: needs.resolve-source.outputs.source_kind == 'branch'
        run: |
          set -euo pipefail
          ACTUAL=$(git -C "${GITHUB_WORKSPACE}/grid" rev-parse HEAD)

          echo "::group::Verified Grid branch checkout"
          echo "Repository:       ${{ github.repository }}"
          echo "Requested branch: ${BRANCH_NAME}"
          echo "Expected SHA:     ${TESTED_SHA}"
          echo "Checked-out HEAD: ${ACTUAL}"
          git -C "${GITHUB_WORKSPACE}/grid" status --short --branch
          git -C "${GITHUB_WORKSPACE}/grid" log -1 --format='Commit: %H%nSubject: %s'
          echo "::endgroup::"

          if [[ "${ACTUAL}" != "${TESTED_SHA}" ]]; then
            echo "::error::Branch SHA changed between resolve and checkout"
            echo "::error::Expected ${TESTED_SHA}, got ${ACTUAL}"
            exit 1
          fi
          echo "::notice::PASS: testing ${BRANCH_NAME} at ${ACTUAL}"

      - name: Validate private backend settings in the checked-in topology
        run: |
          set -euo pipefail
          case "${QUALIFICATION}" in
            glb)
              CONFIGS=(
                "${GITHUB_WORKSPACE}/grid/tests/e2e/topologies/grid-glb-demo/configs/east-provider/praxis.yaml"
                "${GITHUB_WORKSPACE}/grid/tests/e2e/topologies/grid-glb-demo/configs/west-provider/praxis.yaml"
              )
              ;;
            static-weighted|combined-site|llmd-pool-metrics)
              CONFIGS=(
                "${GITHUB_WORKSPACE}/grid/tests/e2e/topologies/grid-${QUALIFICATION}/configs/consumer/praxis.yaml"
                "${GITHUB_WORKSPACE}/grid/tests/e2e/topologies/grid-${QUALIFICATION}/configs/provider/praxis.yaml"
              )
              ;;
            operator-routing)
              TOPOLOGY="${GITHUB_WORKSPACE}/grid/tests/e2e/topologies/grid-operator-routing/forge.yaml"
              MOCK_PROVIDER_MANIFEST="${GITHUB_WORKSPACE}/grid/tests/e2e/topologies/grid-operator-routing/resources/mock-providers.yaml"
              QUALIFICATION_SOURCE="${GITHUB_WORKSPACE}/grid/crates/xtask/src/env/operator_routing_qualification.rs"
              for endpoint in \
                'endpoint: "http://provider-a.grid-system.svc:8080"' \
                'endpoint: "http://provider-b-old.grid-system.svc:8080"'; do
                if ! grep -Fq "${endpoint}" "${TOPOLOGY}"; then
                  echo "::error::Operator-routing topology is missing expected local Service endpoint ${endpoint}"
                  exit 1
                fi
              done
              if ! grep -Fq 'name: provider-b-replacement' "${MOCK_PROVIDER_MANIFEST}" \
                || ! grep -Fq 'http://provider-b-replacement.grid-system.svc:8080' "${QUALIFICATION_SOURCE}"; then
                echo "::error::Operator-routing replacement endpoint is missing its run-owned Service or runtime spec update"
                exit 1
              fi
              echo "Validated operator-routing uses only the run-owned cluster-local provider Services"
              CONFIGS=()
              ;;
            *)
              echo "::error::No private-backend validation mapping for ${QUALIFICATION}"
              exit 1
              ;;
          esac

          for config in "${CONFIGS[@]}"; do
            if [[ ! -f "${config}" ]]; then
              echo "::error::Missing qualification provider config: ${config}"
              exit 1
            fi

            if ! grep -Eq '^  allow_private_endpoints:[[:space:]]*true[[:space:]]*(#.*)?$' "${config}" \
              || ! grep -Eq '^  allow_private_upstreams:[[:space:]]*true[[:space:]]*(#.*)?$' "${config}"; then
              echo "::error::${config} must explicitly allow the private Kind endpoint and upstream addresses"
              exit 1
            fi
            echo "Validated private Kind backend allowances in ${config#"${GITHUB_WORKSPACE}/grid/"}"
          done

      - name: Validate immutable gateway input
        if: env.QUALIFICATION != 'operator-routing'
        run: |
          set -euo pipefail
          if ! [[ "${GATEWAY_IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
            echo "::error::gateway_image_digest must be a sha256 digest (sha256 followed by 64 lowercase hex characters)"
            exit 1
          fi
          echo "Gateway input: ${GATEWAY_IMAGE_REF}@${GATEWAY_IMAGE_DIGEST}"

      # ----------------------------------------------------------------------
      # Toolchain
      # ----------------------------------------------------------------------

      - name: Install Rust
        run: |
          rustup toolchain install "${RUST_VERSION}" --profile minimal
          rustup default "${RUST_VERSION}"
          rustc --version
          cargo --version

      - name: Cache Cargo registry
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: |
            ~/.cargo/registry/index
            ~/.cargo/registry/cache
            ~/.cargo/git/db
          key: ${{ runner.os }}-rust-${{ env.RUST_VERSION }}-triggered-e2e-${{ hashFiles('grid/**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-rust-${{ env.RUST_VERSION }}-triggered-e2e-
          save-always: true

      - name: Install Kind
        run: |
          curl -fsSL -o /tmp/kind \
            "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
          echo "50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54  /tmp/kind" | sha256sum -c
          sudo install -m 0755 /tmp/kind /usr/local/bin/kind
          rm /tmp/kind
          kind version

      - name: Install kubectl
        run: |
          curl -fsSL -o /tmp/kubectl \
            "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
          curl -fsSL -o /tmp/kubectl.sha256 \
            "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256"
          echo "$(cat /tmp/kubectl.sha256)  /tmp/kubectl" | sha256sum -c
          sudo install -m 0755 /tmp/kubectl /usr/local/bin/kubectl
          rm /tmp/kubectl /tmp/kubectl.sha256
          kubectl version --client

      - name: Verify system tools
        run: |
          docker version --format 'Docker {{.Server.Version}}'
          openssl version
          curl --version | head -1
          jq --version

      - name: Record tool versions
        run: |
          {
            echo "rust=$(rustc --version)"
            echo "cargo=$(cargo --version)"
            echo "docker=$(docker version --format '{{.Server.Version}}')"
            echo "kind=$(kind version)"
            echo "kubectl=$(kubectl version --client -o json | jq -r '.clientVersion.gitVersion')"
            echo "helm=$(helm version --template '{{.Version}}')"
            echo "openssl=$(openssl version)"
            echo "curl=$(curl --version | head -1)"
            echo "jq=$(jq --version)"
            echo "os=$(uname -sr)"
            echo "runner=${{ runner.os }}-${{ runner.arch }}"
          } | tee "${GITHUB_WORKSPACE}/versions.txt"

      # ----------------------------------------------------------------------
      # Free disk space for five Kind clusters
      # ----------------------------------------------------------------------

      - name: Free disk space
        run: |
          echo "=== Before cleanup ==="
          df -h /
          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc \
            /usr/local/share/boost /usr/local/graalvm /usr/local/share/chromium \
            /usr/local/.ghcup 2>/dev/null || true
          sudo docker image prune -af 2>/dev/null || true
          echo "=== After cleanup ==="
          df -h /

      # ----------------------------------------------------------------------
      # Build images from the tested Grid revision
      # ----------------------------------------------------------------------

      - name: Compute image tags
        id: tags
        run: |
          SHORT_SHA="${TESTED_SHA:0:7}"
          if [[ "${SOURCE_KIND}" == "pr" ]]; then
            TAG="pr-${PR_NUMBER}-${SHORT_SHA}"
            ARTIFACT_PREFIX="grid-pr-${PR_NUMBER}-${SHORT_SHA}"
          elif [[ "${SOURCE_KIND}" == "branch" ]]; then
            TAG="branch-${SHORT_SHA}"
            ARTIFACT_PREFIX="grid-branch-${SHORT_SHA}"
          else
            TAG="main-${SHORT_SHA}"
            ARTIFACT_PREFIX="grid-main-${SHORT_SHA}"
          fi
          TAG="${TAG}-${QUALIFICATION}"
          case "${OPERATOR_IMAGE_MODE}" in
            standard) OPERATOR_IMAGE="grid-operator:${TAG}" ;;
            fips) OPERATOR_IMAGE="grid-operator-fips:${TAG}" ;;
            *) echo "::error::Unknown operator image mode '${OPERATOR_IMAGE_MODE}'"; exit 1 ;;
          esac
          {
            echo "tag=${TAG}"
            echo "operator_image=${OPERATOR_IMAGE}"
            echo "operator_image_mode=${OPERATOR_IMAGE_MODE}"
            echo "short_sha=${SHORT_SHA}"
            echo "artifact_prefix=${ARTIFACT_PREFIX}"
          } >> "${GITHUB_OUTPUT}"
          echo "Image tag: ${TAG}"
          echo "Operator image: ${OPERATOR_IMAGE}"

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1

      - name: Require a FIPS-enabled host for FIPS runtime qualification
        if: env.OPERATOR_IMAGE_MODE == 'fips'
        run: |
          set -euo pipefail
          flag=/proc/sys/crypto/fips_enabled
          if [[ ! -r "${flag}" || "$(<"${flag}")" != "1" ]]; then
            echo "::error::FIPS operator runtime qualification requires a runner whose kernel reports ${flag}=1; use operator_image_mode=standard on ordinary GitHub-hosted runners."
            exit 1
          fi

      - name: Build Grid Operator image
        if: env.OPERATOR_IMAGE_MODE == 'standard'
        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
        with:
          context: grid
          file: grid/deploy/operator/Containerfile
          push: false
          load: true
          tags: grid-operator:${{ steps.tags.outputs.tag }}
          labels: org.opencontainers.image.revision=${{ env.TESTED_SHA }}
          cache-from: type=gha,scope=triggered-e2e-operator
          cache-to: type=gha,mode=max,scope=triggered-e2e-operator

      - name: Build Grid Operator FIPS image
        if: env.OPERATOR_IMAGE_MODE == 'fips'
        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
        with:
          context: grid
          file: grid/deploy/operator/Containerfile.fips
          push: false
          load: true
          tags: grid-operator-fips:${{ steps.tags.outputs.tag }}
          labels: org.opencontainers.image.revision=${{ env.TESTED_SHA }}
          cache-from: type=gha,scope=triggered-e2e-operator-fips
          cache-to: type=gha,mode=max,scope=triggered-e2e-operator-fips

      - name: Build source-matched Grid Gateway image
        if: (env.QUALIFICATION == 'llmd-pool-metrics' && env.MODE == 'full') || env.QUALIFICATION == 'operator-routing'
        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
        with:
          context: grid
          file: grid/deploy/gateway/Containerfile
          push: false
          load: true
          tags: grid-gateway:${{ steps.tags.outputs.tag }}
          labels: org.opencontainers.image.revision=${{ env.TESTED_SHA }}
          cache-from: type=gha,scope=triggered-e2e-grid-gateway
          cache-to: type=gha,mode=max,scope=triggered-e2e-grid-gateway

      - name: Build Grid mock-provider image
        if: env.QUALIFICATION == 'operator-routing'
        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
        with:
          context: grid
          file: grid/crates/mock-providers/Containerfile
          push: false
          load: true
          tags: grid-mock-providers:${{ steps.tags.outputs.tag }}
          labels: org.opencontainers.image.revision=${{ env.TESTED_SHA }}
          cache-from: type=gha,scope=triggered-e2e-mock-providers
          cache-to: type=gha,mode=max,scope=triggered-e2e-mock-providers

      - name: Build Grid overlay-sync image
        if: env.QUALIFICATION == 'static-weighted' || env.QUALIFICATION == 'combined-site' || env.QUALIFICATION == 'llmd-pool-metrics'
        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
        with:
          context: grid
          file: grid/crates/overlay-sync/Containerfile
          push: false
          load: true
          tags: grid-overlay-sync:${{ steps.tags.outputs.tag }}
          labels: org.opencontainers.image.revision=${{ env.TESTED_SHA }}
          cache-from: type=gha,scope=triggered-e2e-overlay-sync
          cache-to: type=gha,mode=max,scope=triggered-e2e-overlay-sync

      - name: Pull pinned gateway image
        if: env.QUALIFICATION != 'operator-routing'
        id: gateway
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail

          cleanup_docker_auth() { docker logout ghcr.io 2>/dev/null || true; }

          PINNED_GATEWAY="${GATEWAY_IMAGE_REF}@${GATEWAY_IMAGE_DIGEST}"
          if ! docker pull "${PINNED_GATEWAY}" 2>/dev/null; then
            echo "Anonymous pull failed; attempting authenticated pull"
            echo "${GH_TOKEN}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
            trap cleanup_docker_auth EXIT
            docker pull "${PINNED_GATEWAY}"
            trap - EXIT
            docker logout ghcr.io
          fi
          docker tag "${PINNED_GATEWAY}" "${GATEWAY_IMAGE_REF}"

          RESOLVED_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' "${GATEWAY_IMAGE_REF}" \
            | grep -oP 'sha256:[0-9a-f]+')
          echo "digest=${RESOLVED_DIGEST}" >> "${GITHUB_OUTPUT}"
          echo "Resolved gateway digest: ${RESOLVED_DIGEST}"

          if [[ "${RESOLVED_DIGEST}" != "${GATEWAY_IMAGE_DIGEST}" ]]; then
            echo "::error::Gateway image digest mismatch"
            echo "::error::Expected: ${GATEWAY_IMAGE_DIGEST}"
            echo "::error::Actual:   ${RESOLVED_DIGEST}"
            exit 1
          fi
          echo "Digest verification: PASS"

      - name: Pull pinned GLB simulator image
        if: env.QUALIFICATION == 'glb'
        run: |
          set -euo pipefail
          docker pull "${SIM_IMAGE_REF}"
          EXPECTED_DIGEST="${SIM_IMAGE_REF##*@}"
          REPO_DIGESTS=$(docker image inspect --format='{{json .RepoDigests}}' "${SIM_IMAGE_REF}")
          if ! jq -e --arg digest "${EXPECTED_DIGEST}" 'any(.[]; endswith($digest))' <<< "${REPO_DIGESTS}" > /dev/null; then
            echo "::error::Pulled simulator does not report the pinned repository digest ${EXPECTED_DIGEST}: ${REPO_DIGESTS}"
            exit 1
          fi
          docker tag "${SIM_IMAGE_REF}" "${SIM_IMAGE_TAG}"
          ACTUAL_ID=$(docker image inspect --format='{{.Id}}' "${SIM_IMAGE_REF}")
          echo "Simulator image verified: ${SIM_IMAGE_REF} (image ID ${ACTUAL_ID}), tagged locally as ${SIM_IMAGE_TAG}"

      - name: Pull static-weighted simulator image
        if: env.QUALIFICATION == 'static-weighted'
        id: static-simulator
        run: |
          set -euo pipefail
          docker pull "${STATIC_WEIGHTED_SIM_IMAGE_REF}"
          EXPECTED_DIGEST="${STATIC_WEIGHTED_SIM_IMAGE_REF##*@}"
          REPO_DIGESTS=$(docker image inspect --format='{{json .RepoDigests}}' "${STATIC_WEIGHTED_SIM_IMAGE_REF}")
          if ! jq -e --arg digest "${EXPECTED_DIGEST}" 'any(.[]; endswith($digest))' <<< "${REPO_DIGESTS}" > /dev/null; then
            echo "::error::Pulled static-weighted simulator does not report the pinned repository digest ${EXPECTED_DIGEST}: ${REPO_DIGESTS}"
            exit 1
          fi
          docker tag "${STATIC_WEIGHTED_SIM_IMAGE_REF}" "${STATIC_WEIGHTED_SIM_IMAGE_TAG}"
          IMAGE_ID=$(docker image inspect --format='{{.Id}}' "${STATIC_WEIGHTED_SIM_IMAGE_TAG}")
          {
            echo "image_id=${IMAGE_ID}"
            echo "repo_digests=${REPO_DIGESTS}"
          } >> "${GITHUB_OUTPUT}"
          echo "Static-weighted simulator verified: ${STATIC_WEIGHTED_SIM_IMAGE_REF} (image ID ${IMAGE_ID}; tagged locally as ${STATIC_WEIGHTED_SIM_IMAGE_TAG})"

      - name: Pull combined-site simulator image
        if: env.QUALIFICATION == 'combined-site'
        id: combined-simulator
        run: |
          set -euo pipefail
          docker pull "${COMBINED_SITE_SIM_IMAGE_REF}"
          EXPECTED_DIGEST="${COMBINED_SITE_SIM_IMAGE_REF##*@}"
          REPO_DIGESTS=$(docker image inspect --format='{{json .RepoDigests}}' "${COMBINED_SITE_SIM_IMAGE_REF}")
          if ! jq -e --arg digest "${EXPECTED_DIGEST}" 'any(.[]; endswith($digest))' <<< "${REPO_DIGESTS}" > /dev/null; then
            echo "::error::Pulled combined-site simulator does not report the pinned repository digest ${EXPECTED_DIGEST}: ${REPO_DIGESTS}"
            exit 1
          fi
          docker tag "${COMBINED_SITE_SIM_IMAGE_REF}" "${COMBINED_SITE_SIM_IMAGE_TAG}"
          IMAGE_ID=$(docker image inspect --format='{{.Id}}' "${COMBINED_SITE_SIM_IMAGE_TAG}")
          {
            echo "image_id=${IMAGE_ID}"
            echo "repo_digests=${REPO_DIGESTS}"
          } >> "${GITHUB_OUTPUT}"
          echo "Combined-site simulator verified: ${COMBINED_SITE_SIM_IMAGE_REF} (image ID ${IMAGE_ID}; tagged locally as ${COMBINED_SITE_SIM_IMAGE_TAG})"

      - name: Pull pinned llm-d simulator image
        if: env.QUALIFICATION == 'llmd-pool-metrics'
        id: llmd-simulator
        run: |
          set -euo pipefail
          docker pull "${LLMD_SIM_IMAGE_REF}"
          EXPECTED_DIGEST="${LLMD_SIM_IMAGE_REF##*@}"
          REPO_DIGESTS=$(docker image inspect --format='{{json .RepoDigests}}' "${LLMD_SIM_IMAGE_REF}")
          if ! jq -e --arg digest "${EXPECTED_DIGEST}" 'any(.[]; endswith($digest))' <<< "${REPO_DIGESTS}" > /dev/null; then
            echo "::error::Pulled llm-d simulator does not report pinned digest ${EXPECTED_DIGEST}: ${REPO_DIGESTS}"
            exit 1
          fi
          docker tag "${LLMD_SIM_IMAGE_REF}" "${LLMD_SIM_IMAGE_TAG}"
          IMAGE_ID=$(docker image inspect --format='{{.Id}}' "${LLMD_SIM_IMAGE_TAG}")
          {
            echo "image_id=${IMAGE_ID}"
            echo "repo_digests=${REPO_DIGESTS}"
          } >> "${GITHUB_OUTPUT}"
          echo "llm-d simulator verified: ${LLMD_SIM_IMAGE_REF} (image ID ${IMAGE_ID}; tagged locally as ${LLMD_SIM_IMAGE_TAG})"

      - name: Pull pinned llm-d EPP image
        if: env.QUALIFICATION == 'llmd-pool-metrics'
        id: llmd-epp
        run: |
          set -euo pipefail
          docker pull "${LLMD_EPP_IMAGE_REF}"
          EXPECTED_DIGEST="${LLMD_EPP_IMAGE_REF##*@}"
          REPO_DIGESTS=$(docker image inspect --format='{{json .RepoDigests}}' "${LLMD_EPP_IMAGE_REF}")
          if ! jq -e --arg digest "${EXPECTED_DIGEST}" 'any(.[]; endswith($digest))' <<< "${REPO_DIGESTS}" > /dev/null; then
            echo "::error::Pulled llm-d EPP does not report pinned digest ${EXPECTED_DIGEST}: ${REPO_DIGESTS}"
            exit 1
          fi
          docker tag "${LLMD_EPP_IMAGE_REF}" "${LLMD_EPP_IMAGE_TAG}"
          IMAGE_ID=$(docker image inspect --format='{{.Id}}' "${LLMD_EPP_IMAGE_TAG}")
          {
            echo "image_id=${IMAGE_ID}"
            echo "repo_digests=${REPO_DIGESTS}"
          } >> "${GITHUB_OUTPUT}"
          echo "llm-d EPP verified: ${LLMD_EPP_IMAGE_REF} (image ID ${IMAGE_ID}; tagged locally as ${LLMD_EPP_IMAGE_TAG})"

      - name: Pull pinned probe image
        run: |
          set -euo pipefail
          docker pull "${PROBE_IMAGE_REF}"
          EXPECTED_DIGEST="${PROBE_IMAGE_REF##*@}"
          REPO_DIGESTS=$(docker image inspect --format='{{json .RepoDigests}}' "${PROBE_IMAGE_REF}")
          if ! jq -e --arg digest "${EXPECTED_DIGEST}" 'any(.[]; endswith($digest))' <<< "${REPO_DIGESTS}" > /dev/null; then
            echo "::error::Pulled probe image does not report the pinned repository digest ${EXPECTED_DIGEST}: ${REPO_DIGESTS}"
            exit 1
          fi
          docker tag "${PROBE_IMAGE_REF}" "${PROBE_IMAGE_TAG}"
          echo "Probe image verified: ${PROBE_IMAGE_REF} (tagged locally as ${PROBE_IMAGE_TAG})"

      # ----------------------------------------------------------------------
      # Image verification
      # ----------------------------------------------------------------------

      - name: Verify locally built images
        env:
          OPERATOR_IMAGE: ${{ steps.tags.outputs.operator_image }}
          TAG: ${{ steps.tags.outputs.tag }}
        run: |
          set -euo pipefail
          FAILED=0

          operator_img="${OPERATOR_IMAGE}"
          if ! docker image inspect "${operator_img}" > /dev/null 2>&1; then
            echo "::error::Required locally-built image missing: ${operator_img}"
            FAILED=1
          else
            IMAGE_REVISION=$(docker image inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' "${operator_img}")
            echo "${operator_img}: revision label ${IMAGE_REVISION}"
            if [[ "${IMAGE_REVISION}" != "${TESTED_SHA}" ]]; then
              echo "::error::${operator_img} was not built from verified source ${TESTED_SHA}"
              FAILED=1
            fi
          fi
          IMAGE_DESCRIPTION=$(docker image inspect --format='{{index .Config.Labels "org.opencontainers.image.description"}}' "${operator_img}")
          if [[ "${OPERATOR_IMAGE_MODE}" == "fips" ]]; then
            if [[ "${IMAGE_DESCRIPTION}" != *"FIPS build"* ]]; then
              echo "::error::FIPS mode selected but image is not labeled as a FIPS build: ${operator_img}"
              FAILED=1
            fi
          elif [[ "${IMAGE_DESCRIPTION}" == *"FIPS build"* ]]; then
            echo "::error::Standard mode selected but a FIPS operator image was selected: ${operator_img}"
            FAILED=1
          fi

          if [[ "${QUALIFICATION}" == "static-weighted" || "${QUALIFICATION}" == "combined-site" || "${QUALIFICATION}" == "llmd-pool-metrics" ]]; then
            img="grid-overlay-sync:${TAG}"
            if ! docker image inspect "${img}" > /dev/null 2>&1; then
              echo "::error::Required locally-built image missing: ${img}"
              FAILED=1
            else
              IMAGE_REVISION=$(docker image inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' "${img}")
              echo "${img}: revision label ${IMAGE_REVISION}"
              if [[ "${IMAGE_REVISION}" != "${TESTED_SHA}" ]]; then
                echo "::error::${img} was not built from verified source ${TESTED_SHA}"
                FAILED=1
              fi
              if [[ "${img}" =~ ^ghcr\.io/ ]] || [[ "${img}" =~ :v[0-9] ]] || [[ "${img}" =~ :latest$ ]]; then
                echo "::error::GRID_XTASK_OVERLAY_SYNC_IMAGE=${img} appears to be a released image, not a locally-built image"
                FAILED=1
              fi
            fi
          fi

          if [[ ( "${QUALIFICATION}" == "llmd-pool-metrics" && "${MODE}" == "full" ) || "${QUALIFICATION}" == "operator-routing" ]]; then
            img="grid-gateway:${TAG}"
            if ! docker image inspect "${img}" > /dev/null 2>&1; then
              echo "::error::Required source-built Grid Gateway image missing: ${img}"
              FAILED=1
            else
              IMAGE_REVISION=$(docker image inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' "${img}")
              echo "${img}: revision label ${IMAGE_REVISION}"
              if [[ "${IMAGE_REVISION}" != "${TESTED_SHA}" ]]; then
                echo "::error::${img} was not built from verified source ${TESTED_SHA}"
                FAILED=1
              fi
            fi
          fi

          if [[ "${QUALIFICATION}" == "operator-routing" ]]; then
            img="grid-mock-providers:${TAG}"
            if ! docker image inspect "${img}" > /dev/null 2>&1; then
              echo "::error::Required source-built mock-provider image missing: ${img}"
              FAILED=1
            else
              IMAGE_REVISION=$(docker image inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' "${img}")
              echo "${img}: revision label ${IMAGE_REVISION}"
              if [[ "${IMAGE_REVISION}" != "${TESTED_SHA}" ]]; then
                echo "::error::${img} was not built from verified source ${TESTED_SHA}"
                FAILED=1
              fi
            fi
          fi

          if [[ "${QUALIFICATION}" != "operator-routing" ]] && ! docker image inspect "${GATEWAY_IMAGE_REF}" > /dev/null 2>&1; then
            echo "::error::Pinned gateway image missing: ${GATEWAY_IMAGE_REF}"
            FAILED=1
          fi

          if [[ "${QUALIFICATION}" == "llmd-pool-metrics" ]]; then
            for img in "${LLMD_SIM_IMAGE_TAG}" "${LLMD_EPP_IMAGE_TAG}"; do
              if ! docker image inspect "${img}" > /dev/null 2>&1; then
                echo "::error::Pinned llm-d image missing: ${img}"
                FAILED=1
              fi
            done
          fi

          if [[ "${operator_img}" =~ ^ghcr\.io/ ]] || [[ "${operator_img}" =~ :v[0-9] ]] || [[ "${operator_img}" =~ :latest$ ]]; then
            echo "::error::GRID_XTASK_OPERATOR_IMAGE=${operator_img} appears to be a released image, not a locally-built image"
            FAILED=1
          fi

          if [[ "${FAILED}" -ne 0 ]]; then
            exit 1
          fi
          echo "All required images verified"

      - name: Write image manifest
        env:
          OPERATOR_IMAGE: ${{ steps.tags.outputs.operator_image }}
          TAG: ${{ steps.tags.outputs.tag }}
          RESOLVED_DIGEST: ${{ steps.gateway.outputs.digest }}
          STATIC_SIMULATOR_DIGESTS: ${{ steps.static-simulator.outputs.repo_digests || 'unknown' }}
          COMBINED_SIMULATOR_DIGESTS: ${{ steps.combined-simulator.outputs.repo_digests || 'unknown' }}
          LLMD_SIMULATOR_DIGESTS: ${{ steps.llmd-simulator.outputs.repo_digests || 'unknown' }}
          LLMD_EPP_DIGESTS: ${{ steps.llmd-epp.outputs.repo_digests || 'unknown' }}
        run: |
          set -euo pipefail
          case "${QUALIFICATION}" in
            glb)
              SIMULATOR_REF="${SIM_IMAGE_TAG}"
              SIMULATOR_SOURCE_REF="${SIM_IMAGE_REF}"
              SIMULATOR_DESCRIPTION="pinned GLB simulator materialized from ${SIM_IMAGE_REF}"
              SIMULATOR_DIGESTS=$(docker image inspect --format='{{join .RepoDigests ","}}' "${SIM_IMAGE_TAG}")
              ;;
            static-weighted)
              SIMULATOR_REF="${STATIC_WEIGHTED_SIM_IMAGE_TAG}"
              SIMULATOR_SOURCE_REF="${STATIC_WEIGHTED_SIM_IMAGE_REF}"
              SIMULATOR_DESCRIPTION="static-weighted simulator materialized from its pinned digest"
              SIMULATOR_DIGESTS="${STATIC_SIMULATOR_DIGESTS}"
              ;;
            combined-site)
              SIMULATOR_REF="${COMBINED_SITE_SIM_IMAGE_TAG}"
              SIMULATOR_SOURCE_REF="${COMBINED_SITE_SIM_IMAGE_REF}"
              SIMULATOR_DESCRIPTION="combined-site default tag materialized from its pinned digest"
              SIMULATOR_DIGESTS="${COMBINED_SIMULATOR_DIGESTS}"
              ;;
            llmd-pool-metrics)
              SIMULATOR_REF="${LLMD_SIM_IMAGE_TAG}"
              SIMULATOR_SOURCE_REF="${LLMD_SIM_IMAGE_REF}"
              SIMULATOR_DESCRIPTION="llm-d simulator materialized from its pinned digest"
              SIMULATOR_DIGESTS="${LLMD_SIMULATOR_DIGESTS}"
              ;;
            operator-routing)
              SIMULATOR_REF=""
              SIMULATOR_SOURCE_REF=""
              SIMULATOR_DESCRIPTION="not used; dedicated mock-provider image is source-built"
              SIMULATOR_DIGESTS=""
              ;;
            *)
              echo "::error::Unknown qualification '${QUALIFICATION}' while writing image manifest"
              exit 1
              ;;
          esac
          {
            echo "=== Image Manifest ==="
            echo ""
            echo "Operator image mode: ${OPERATOR_IMAGE_MODE}"
            echo "${OPERATOR_IMAGE}"
            docker inspect --format='  ID: {{.Id}}' "${OPERATOR_IMAGE}"
            docker inspect --format='  Source revision: {{index .Config.Labels "org.opencontainers.image.revision"}}' "${OPERATOR_IMAGE}"
            echo ""
            if [[ "${QUALIFICATION}" == "static-weighted" || "${QUALIFICATION}" == "combined-site" || "${QUALIFICATION}" == "llmd-pool-metrics" ]]; then
              echo "grid-overlay-sync:${TAG}"
              docker inspect --format='  ID: {{.Id}}' "grid-overlay-sync:${TAG}"
              docker inspect --format='  Source revision: {{index .Config.Labels "org.opencontainers.image.revision"}}' "grid-overlay-sync:${TAG}"
              echo ""
            fi
            if [[ ( "${QUALIFICATION}" == "llmd-pool-metrics" && "${MODE}" == "full" ) || "${QUALIFICATION}" == "operator-routing" ]]; then
              echo "grid-gateway:${TAG}  (built from tested Grid source)"
              docker inspect --format='  ID: {{.Id}}' "grid-gateway:${TAG}"
              docker inspect --format='  Source revision: {{index .Config.Labels "org.opencontainers.image.revision"}}' "grid-gateway:${TAG}"
              echo ""
            fi
            if [[ "${QUALIFICATION}" == "operator-routing" ]]; then
              echo "grid-mock-providers:${TAG}  (built from tested Grid source)"
              docker inspect --format='  ID: {{.Id}}' "grid-mock-providers:${TAG}"
              docker inspect --format='  Source revision: {{index .Config.Labels "org.opencontainers.image.revision"}}' "grid-mock-providers:${TAG}"
              echo ""
            else
              echo "${GATEWAY_IMAGE_REF}  (pinned gateway; not built by Grid)"
              docker inspect --format='  ID: {{.Id}}' "${GATEWAY_IMAGE_REF}"
              echo "  Digest: ${RESOLVED_DIGEST}"
              echo ""
            fi
            if [[ "${QUALIFICATION}" != "operator-routing" ]]; then
              echo "${SIMULATOR_REF}  (${SIMULATOR_DESCRIPTION})"
              docker inspect --format='  ID: {{.Id}}' "${SIMULATOR_REF}"
              echo "  Pinned source: ${SIMULATOR_SOURCE_REF}"
              echo "  Repository digests: ${SIMULATOR_DIGESTS}"
              echo ""
            fi
            if [[ "${QUALIFICATION}" == "llmd-pool-metrics" ]]; then
              echo "${LLMD_EPP_IMAGE_TAG}  (llm-d endpoint picker materialized from its pinned digest)"
              docker inspect --format='  ID: {{.Id}}' "${LLMD_EPP_IMAGE_TAG}"
              echo "  Pinned source: ${LLMD_EPP_IMAGE_REF}"
              echo "  Repository digests: ${LLMD_EPP_DIGESTS}"
              echo ""
            fi
            echo "${PROBE_IMAGE_REF}  (pinned GLB probe image; local tag ${PROBE_IMAGE_TAG})"
            docker inspect --format='  ID: {{.Id}}' "${PROBE_IMAGE_TAG}"
            echo ""
            echo "=== Override Environment ==="
            echo "GRID_XTASK_OPERATOR_IMAGE=${OPERATOR_IMAGE}"
            if [[ "${QUALIFICATION}" == "static-weighted" || "${QUALIFICATION}" == "combined-site" || "${QUALIFICATION}" == "llmd-pool-metrics" ]]; then
              echo "GRID_XTASK_OVERLAY_SYNC_IMAGE=grid-overlay-sync:${TAG}"
            else
              echo "GRID_XTASK_OVERLAY_SYNC_IMAGE=<unset>"
            fi
            if [[ "${QUALIFICATION}" == "operator-routing" ]]; then
              echo "GRID_XTASK_GATEWAY_IMAGE=<unset> (uses the source-built Grid Gateway)"
            else
              echo "GRID_XTASK_GATEWAY_IMAGE=${GATEWAY_IMAGE_REF}"
            fi
            if [[ ( "${QUALIFICATION}" == "llmd-pool-metrics" && "${MODE}" == "full" ) || "${QUALIFICATION}" == "operator-routing" ]]; then
              echo "GRID_XTASK_GRID_GATEWAY_IMAGE=grid-gateway:${TAG}"
            else
              echo "GRID_XTASK_GRID_GATEWAY_IMAGE=<unset>"
            fi
            if [[ "${QUALIFICATION}" == "llmd-pool-metrics" ]]; then
              echo "GRID_XTASK_EPP_IMAGE=${LLMD_EPP_IMAGE_TAG}"
            else
              echo "GRID_XTASK_EPP_IMAGE=<unset>"
            fi
            if [[ "${QUALIFICATION}" == "operator-routing" ]]; then
              echo "GRID_XTASK_MOCK_PROVIDER_IMAGE=grid-mock-providers:${TAG}"
            else
              echo "GRID_XTASK_MOCK_PROVIDER_IMAGE=<unset>"
            fi
            if [[ "${QUALIFICATION}" == "operator-routing" ]]; then
              echo "GRID_XTASK_SIM_IMAGE=<unset> (not used by operator-routing)"
            elif [[ "${QUALIFICATION}" == "combined-site" ]]; then
              echo "GRID_XTASK_SIM_IMAGE=<unset> (xtask default ${COMBINED_SITE_SIM_IMAGE_TAG}, materialized from ${COMBINED_SITE_SIM_IMAGE_REF})"
            else
              echo "GRID_XTASK_SIM_IMAGE=${SIMULATOR_REF}"
            fi
            echo "GRID_XTASK_IMAGE_PULL_POLICY=Never"
          } | tee "${GITHUB_WORKSPACE}/image-manifest.txt"

      # ----------------------------------------------------------------------
      # Prepare workspace
      # ----------------------------------------------------------------------

      - name: Download source metadata
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          name: source-metadata
          path: ${{ github.workspace }}

      - name: Create evidence and diagnostics directories
        run: |
          mkdir -p "${DIAGNOSTICS_DIR}"
          if [[ "${QUALIFICATION}" != "operator-routing" ]]; then
            mkdir -p "${EVIDENCE_DIR}"
          fi

      - name: Record llm-d cleanup baseline
        if: env.QUALIFICATION == 'llmd-pool-metrics'
        run: |
          set -euo pipefail
          mkdir -p "${DIAGNOSTICS_DIR}"
          kind get clusters 2>/dev/null | sort > "${DIAGNOSTICS_DIR}/llmd-kind-before.txt"
          docker network ls --format '{{.Name}} {{.ID}}' | sort > "${DIAGNOSTICS_DIR}/llmd-networks-before.txt"
          docker ps -a --format '{{.ID}} {{.Names}}' | sort > "${DIAGNOSTICS_DIR}/llmd-containers-before.txt"

      - name: Configure isolated llm-d partition network subnet
        if: env.QUALIFICATION == 'llmd-pool-metrics' && env.MODE == 'full'
        run: |
          set -euo pipefail
          real_docker=$(command -v docker)
          wrapper_dir="${RUNNER_TEMP}/grid-llmd-docker-bin"
          mkdir -p "${wrapper_dir}"
          cat > "${wrapper_dir}/docker" <<'EOF'
          #!/usr/bin/env bash
          set -euo pipefail
          if [[ "${1:-}" == "network" && "${2:-}" == "create" ]]; then
            network_name="${@: -1}"
            if [[ "${network_name}" == grid-llmd-pm-*-net ]]; then
              printf 'Applied subnet 10.253.0.0/16 to run-owned network %s\n' "${network_name}" >> "${DIAGNOSTICS_DIR}/docker-network-subnet-override.txt"
              "${GRID_VALIDATION_REAL_DOCKER:?}" network create --subnet 10.253.0.0/16 "${@:3}"
              "${GRID_VALIDATION_REAL_DOCKER}" network inspect "${network_name}" --format '{{range .IPAM.Config}}{{.Subnet}}{{end}}' >> "${DIAGNOSTICS_DIR}/docker-network-subnet-override.txt"
              exit 0
            fi
          fi
          exec "${GRID_VALIDATION_REAL_DOCKER:?}" "$@"
          EOF
          chmod +x "${wrapper_dir}/docker"
          echo "GRID_VALIDATION_REAL_DOCKER=${real_docker}" >> "${GITHUB_ENV}"
          echo "${wrapper_dir}" >> "${GITHUB_PATH}"

      # ----------------------------------------------------------------------
      # Run selected qualification
      # ----------------------------------------------------------------------

      - name: Run selected qualification
        id: qualification
        env:
          GRID_REPO: ${{ github.workspace }}/grid
          TAG: ${{ steps.tags.outputs.tag }}
          GRID_XTASK_OPERATOR_IMAGE: ${{ steps.tags.outputs.operator_image }}
          GRID_XTASK_GATEWAY_IMAGE: ${{ env.GATEWAY_IMAGE_REF }}
          GRID_GATEWAY_IMAGE: grid-gateway:${{ steps.tags.outputs.tag }}
          LLMD_EPP_IMAGE: ${{ env.LLMD_EPP_IMAGE_TAG }}
          LLMD_SIM_IMAGE: ${{ env.LLMD_SIM_IMAGE_TAG }}
          OVERLAY_SYNC_IMAGE: grid-overlay-sync:${{ steps.tags.outputs.tag }}
        run: |
          set -euo pipefail
          GRID_ACTUAL=$(git -C "${GRID_REPO}" rev-parse HEAD)
          if [[ "${GRID_ACTUAL}" != "${TESTED_SHA}" ]]; then
            echo "::error::Grid checkout changed before demo: expected ${TESTED_SHA}, got ${GRID_ACTUAL}"
            exit 1
          fi
          if [[ "${SOURCE_KIND}" == "pr" ]]; then
            echo "::notice::Demo will build xtask from PR merge ${GRID_ACTUAL}, not main/base ${BASE_SHA}"
          elif [[ "${SOURCE_KIND}" == "branch" ]]; then
            echo "::notice::Demo will build xtask from ${BRANCH_NAME} at ${GRID_ACTUAL}"
          else
            echo "::notice::Demo will build xtask from main at ${GRID_ACTUAL}"
          fi
          echo "Grid source:    ${GRID_REPO} (${SOURCE_LABEL})"
          echo "Operator image: ${GRID_XTASK_OPERATOR_IMAGE} (revision ${TESTED_SHA})"
          if [[ "${QUALIFICATION}" == "operator-routing" ]]; then
            unset GRID_XTASK_GATEWAY_IMAGE
            echo "Grid Gateway image: ${GRID_GATEWAY_IMAGE} (revision ${TESTED_SHA})"
          else
            echo "AI Gateway image: ${GRID_XTASK_GATEWAY_IMAGE}@${GATEWAY_IMAGE_DIGEST}"
          fi
          echo "Qualification:  ${QUALIFICATION} (${MODE})"
          echo "Evidence:       ${EVIDENCE_DIR}"
          cd "${GRID_REPO}"
          ./scripts/forge.sh install
          case "${QUALIFICATION}" in
            glb)
              GRID_XTASK_SIM_IMAGE="${SIM_IMAGE_TAG}" \
              GRID_XTASK_IMAGE_PULL_POLICY=Never \
              timeout --signal=TERM --kill-after=30s 75m cargo xtask env run-grid-glb-demo \
                --forge-config tests/e2e/topologies/grid-glb-demo/forge.yaml \
                "--${MODE}" --teardown --keep-on-failure \
                --evidence-dir "${EVIDENCE_DIR}" \
                2>&1 | tee "${GITHUB_WORKSPACE}/run-${QUALIFICATION}.log"
              ;;
            static-weighted)
              GRID_XTASK_SIM_IMAGE="${STATIC_WEIGHTED_SIM_IMAGE_TAG}" \
              GRID_XTASK_OVERLAY_SYNC_IMAGE="${OVERLAY_SYNC_IMAGE}" \
              GRID_XTASK_IMAGE_PULL_POLICY=Never \
              timeout --signal=TERM --kill-after=30s 75m cargo xtask env run-grid-static-weighted-qualification \
                --forge-config tests/e2e/topologies/grid-static-weighted/forge.yaml \
                --quick --teardown --keep-on-failure --evidence-dir "${EVIDENCE_DIR}" \
                2>&1 | tee "${GITHUB_WORKSPACE}/run-${QUALIFICATION}.log"
              ;;
            combined-site)
              GRID_XTASK_OVERLAY_SYNC_IMAGE="${OVERLAY_SYNC_IMAGE}" \
              GRID_XTASK_IMAGE_PULL_POLICY=Never \
              timeout --signal=TERM --kill-after=30s 75m cargo xtask env run-grid-combined-site-demo \
                --forge-config tests/e2e/topologies/grid-combined-site/forge.yaml \
                --membership "${MEMBERSHIP}" \
                --full --teardown --keep-on-failure --evidence-dir "${EVIDENCE_DIR}" \
                2>&1 | tee "${GITHUB_WORKSPACE}/run-${QUALIFICATION}.log"
              ;;
            llmd-pool-metrics)
              set +e
              GRID_XTASK_SIM_IMAGE="${LLMD_SIM_IMAGE}" \
              GRID_XTASK_EPP_IMAGE="${LLMD_EPP_IMAGE}" \
              GRID_XTASK_GRID_GATEWAY_IMAGE="${GRID_GATEWAY_IMAGE}" \
              GRID_XTASK_OVERLAY_SYNC_IMAGE="${OVERLAY_SYNC_IMAGE}" \
              GRID_XTASK_IMAGE_PULL_POLICY=Never \
              timeout --signal=TERM --kill-after=30s 75m cargo xtask env run-grid-llmd-pool-metrics-demo \
                --forge-config tests/e2e/topologies/grid-llmd-pool-metrics/forge.yaml \
                "--${MODE}" --teardown --keep-on-failure --evidence-dir "${EVIDENCE_DIR}" \
                2>&1 | tee "${GITHUB_WORKSPACE}/run-${QUALIFICATION}.log"
              pipeline_status=("${PIPESTATUS[@]}")
              set -e
              qualification_status=${pipeline_status[0]}
              tee_status=${pipeline_status[1]}
              printf '%s\n' "${qualification_status}" > "${EVIDENCE_DIR}/xtask-exit-status.txt"
              printf '%s\n' "${tee_status}" > "${EVIDENCE_DIR}/tee-exit-status.txt"
              if [[ "${qualification_status}" -ne 0 ]]; then
                echo "::error::llm-d pool-metrics xtask exited with status ${qualification_status}"
                exit "${qualification_status}"
              fi
              if [[ "${tee_status}" -ne 0 ]]; then
                echo "::error::llm-d pool-metrics tee exited with status ${tee_status}"
                exit "${tee_status}"
              fi
              ;;
            operator-routing)
              set +e
              GRID_XTASK_GRID_GATEWAY_IMAGE="${GRID_GATEWAY_IMAGE}" \
              GRID_XTASK_MOCK_PROVIDER_IMAGE="grid-mock-providers:${TAG}" \
              GRID_XTASK_IMAGE_PULL_POLICY=Never \
              timeout --signal=TERM --kill-after=30s 45m cargo xtask env run-grid-operator-routing-qualification \
                --forge-config tests/e2e/topologies/grid-operator-routing/forge.yaml \
                --teardown --keep-on-failure --evidence-dir "${EVIDENCE_DIR}" \
                2>&1 | tee "${GITHUB_WORKSPACE}/run-${QUALIFICATION}.log"
              pipeline_status=("${PIPESTATUS[@]}")
              set -e
              qualification_status=${pipeline_status[0]}
              tee_status=${pipeline_status[1]}
              mkdir -p "${EVIDENCE_DIR}"
              printf '%s\n' "${qualification_status}" > "${EVIDENCE_DIR}/xtask-exit-status.txt"
              printf '%s\n' "${tee_status}" > "${EVIDENCE_DIR}/tee-exit-status.txt"
              if [[ "${qualification_status}" -ne 0 ]]; then
                echo "::error::Operator-routing xtask exited with status ${qualification_status}"
                exit "${qualification_status}"
              fi
              if [[ "${tee_status}" -ne 0 ]]; then
                echo "::error::Operator-routing tee exited with status ${tee_status}"
                exit "${tee_status}"
              fi
              ;;
            *)
              echo "::error::Unknown qualification '${QUALIFICATION}'"
              exit 1
              ;;
          esac
          echo "result=success" >> "${GITHUB_OUTPUT}"

      # ----------------------------------------------------------------------
      # Diagnostics (always, even on failure)
      # ----------------------------------------------------------------------

      - name: Collect diagnostics
        if: always()
        env:
          QUALIFICATION_OUTCOME: ${{ steps.qualification.outcome }}
        run: |
          set -euo pipefail
          DIAG="${DIAGNOSTICS_DIR}"
          mkdir -p "${DIAG}" "${EVIDENCE_DIR}"

          redact_overlay_json() {
            jq '
              def redact:
                if type == "object" then
                  with_entries(
                    if (.key | test("credential|secret|token|authorization|password|private.?key|api.?key"; "i"))
                    then .value = "[REDACTED]"
                    else .value |= redact
                    end
                  )
                elif type == "array" then map(redact)
                else .
                end;
              redact
            ' "$1" > "$2"
          }

          case "${QUALIFICATION}" in
            glb) DEMO_DIR="${GITHUB_WORKSPACE}/grid/tests/e2e/topologies/grid-glb-demo" ;;
            static-weighted) DEMO_DIR="${GITHUB_WORKSPACE}/grid/tests/e2e/topologies/grid-static-weighted" ;;
            combined-site) DEMO_DIR="${GITHUB_WORKSPACE}/grid/tests/e2e/topologies/grid-combined-site" ;;
            llmd-pool-metrics) DEMO_DIR="${GITHUB_WORKSPACE}/grid/tests/e2e/topologies/grid-llmd-pool-metrics" ;;
            operator-routing) DEMO_DIR="${GITHUB_WORKSPACE}/grid/tests/e2e/topologies/grid-operator-routing" ;;
            *)
              echo "::warning::Unknown qualification '${QUALIFICATION}'; skipping Forge diagnostics"
              DEMO_DIR=""
              ;;
          esac

          echo "::group::Kind clusters"
          kind get clusters > "${DIAG}/kind-clusters.txt" 2>&1 || echo "(none)" > "${DIAG}/kind-clusters.txt"
          cat "${DIAG}/kind-clusters.txt"
          echo "::endgroup::"

          for cluster in $(kind get clusters 2>/dev/null); do
            CTX="kind-${cluster}"
            CDIR="${DIAG}/${cluster}"
            mkdir -p "${CDIR}/logs"

            echo "::group::Diagnostics: ${cluster}"

            kubectl --context "${CTX}" get nodes -o wide \
              > "${CDIR}/nodes.txt" 2>&1 || true

            kubectl --context "${CTX}" get pods -A -o wide \
              > "${CDIR}/pods.txt" 2>&1 || true

            kubectl --context "${CTX}" get deployments -A -o wide \
              > "${CDIR}/deployments.txt" 2>&1 || true

            kubectl --context "${CTX}" get statefulsets -A -o wide \
              > "${CDIR}/statefulsets.txt" 2>&1 || true

            kubectl --context "${CTX}" get daemonsets -A -o wide \
              > "${CDIR}/daemonsets.txt" 2>&1 || true

            kubectl --context "${CTX}" get services -A -o wide \
              > "${CDIR}/services.txt" 2>&1 || true

            kubectl --context "${CTX}" get events -A --sort-by='.lastTimestamp' \
              > "${CDIR}/events.txt" 2>&1 || true

            kubectl --context "${CTX}" get configmaps -A \
              > "${CDIR}/configmaps.txt" 2>&1 || true

            # Grid CRDs
            for crd in gridnetworks gridsites inferenceproviders; do
              kubectl --context "${CTX}" get "${crd}" -A -o yaml \
                > "${CDIR}/${crd}.yaml" 2>&1 || true
            done

            # Pod logs from all namespaces (GLB gateway, provider, operator,
            # metallb, and system pods are spread across several namespaces).
            for ns in $(kubectl --context "${CTX}" get namespaces \
              -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do
              mkdir -p "${CDIR}/logs/${ns}"
              for pod in $(kubectl --context "${CTX}" get pods -n "${ns}" \
                -o jsonpath='{.items[*].metadata.name}' 2>/dev/null); do
                kubectl --context "${CTX}" logs -n "${ns}" "${pod}" \
                  --all-containers --tail=2000 \
                  > "${CDIR}/logs/${ns}/${pod}.log" 2>&1 || true
              done
            done

            # Cluster info dump (sanitized)
            kubectl --context "${CTX}" cluster-info dump \
              --output-directory="${CDIR}/cluster-info" 2>&1 || true
            if [[ -d "${CDIR}/cluster-info" ]]; then
              find "${CDIR}/cluster-info" \
                \( -name "*.key" -o -name "*.pem" -o -name "*.crt" \
                   -o -name "*.cer" -o -name "*.p12" -o -name "*.pfx" \
                   -o -name "*.jks" -o -name "*secret*" \
                   -o -name "*credential*" -o -name "*token*" \) \
                -type f -exec rm -f {} + 2>/dev/null || true
            fi

            # On GLB failure, compare the operator-published ConfigMap with the
            # exact files visible to the Praxis gateway before cleanup removes
            # the clusters. Sensitive fields are redacted before artifact write.
            if [[ "${QUALIFICATION}" == "glb" && "${QUALIFICATION_OUTCOME}" == "failure" \
                && ( "${cluster}" == "grid-glb-east-edge" || "${cluster}" == "grid-glb-west-edge" ) ]]; then
              ODIR="${CDIR}/overlay-recovery"
              mkdir -p "${ODIR}"
              RAW_CM="${RUNNER_TEMP}/${cluster}-overlay-configmap.json"
              PARSE_SAFE_CM="${RUNNER_TEMP}/${cluster}-overlay-configmap-parse-safe.json"

              kubectl --context "${CTX}" -n grid-system get configmap grid-overlay-glb-demo-edge-gateway -o json \
                > "${RAW_CM}" 2> "${ODIR}/configmap-fetch-error.txt" || true
              if jq -e '.data | type == "object"' "${RAW_CM}" > /dev/null 2>&1; then
                PARSE_ERRORS='{}'
                for key in routing-config.json routing-overlay.json; do
                  if jq -e --arg key "${key}" '.data | has($key)' "${RAW_CM}" > /dev/null 2>&1; then
                    if ! jq -e --arg key "${key}" \
                      '.data[$key] | (type == "string") and (try (fromjson | true) catch false)' \
                      "${RAW_CM}" > /dev/null 2>&1; then
                      digest=$(jq -j --arg key "${key}" '.data[$key]' "${RAW_CM}" | sha256sum | awk '{print $1}')
                      PARSE_ERRORS=$(jq -cn --argjson errors "${PARSE_ERRORS}" --arg key "${key}" --arg digest "${digest}" \
                        '$errors + {($key): $digest}')
                    fi
                  fi
                done
                printf '%s\n' "${PARSE_ERRORS}" > "${ODIR}/configmap-payload-parse-errors.json"
                if jq '
                  . as $config_map
                  | .data = ((.data // {}) | with_entries(
                      select(.key == "routing-config.json" or .key == "routing-overlay.json")
                      | . as $entry
                      | .value = (try ($entry.value | fromjson) catch {
                          parse_error: true,
                          sha256: (($parse_errors | fromjson)[$entry.key])
                        })
                    ))
                  | {
                    metadata: {
                      name: $config_map.metadata.name,
                      namespace: $config_map.metadata.namespace,
                      resourceVersion: $config_map.metadata.resourceVersion,
                      creationTimestamp: $config_map.metadata.creationTimestamp,
                      managedFields: [($config_map.metadata.managedFields // [])[] | {manager, operation, time, subresource}],
                      annotations: (($config_map.metadata.annotations // {}) | with_entries(select(.key | startswith("grid.praxis.fast/overlay-"))))
                    },
                    data: .data
                  }
                ' --arg parse_errors "${PARSE_ERRORS}" "${RAW_CM}" > "${PARSE_SAFE_CM}"; then
                if redact_overlay_json "${PARSE_SAFE_CM}" "${ODIR}/configmap-redacted.json"; then
                for key in routing-config.json routing-overlay.json; do
                  if jq -e --arg key "${key}" '.data[$key] != null' "${ODIR}/configmap-redacted.json" > /dev/null; then
                    digest=$(jq -S -c --arg key "${key}" '.data[$key]' "${ODIR}/configmap-redacted.json" | sha256sum | awk '{print $1}')
                    printf '%s  %s\n' "${digest}" "${key}" >> "${ODIR}/configmap-payload-sha256.txt"
                  fi
                done
                else
                  rm -f "${ODIR}/configmap-redacted.json"
                  echo "ConfigMap redaction failed; raw payload was not retained" > "${ODIR}/configmap-snapshot-error.txt"
                fi
                else
                  echo "ConfigMap parse-safe snapshot failed; raw payload was not retained" > "${ODIR}/configmap-snapshot-error.txt"
                fi
              else
                echo "ConfigMap JSON was unavailable or invalid" > "${ODIR}/configmap-snapshot-error.txt"
              fi
              rm -f "${RAW_CM}"
              rm -f "${PARSE_SAFE_CM}"

              kubectl --context "${CTX}" -n grid-system get pods -o json \
                | jq '[.items[]? | select(.metadata.name | startswith("edge-gateway-")) | {
                    metadata: {name: .metadata.name, uid: .metadata.uid, creationTimestamp: .metadata.creationTimestamp},
                    nodeName: .spec.nodeName,
                    overlayVolume: [(.spec.volumes // [])[] | select(.name == "overlay") | {name, configMap: .configMap.name}],
                    overlayMounts: [(.spec.containers // [])[] | {container: .name, mounts: [(.volumeMounts // [])[] | select(.name == "overlay") | {mountPath, subPath, readOnly}]}],
                    containerStatuses: [(.status.containerStatuses // [])[] | {name, imageID, restartCount, state, lastState}]
                  }]' > "${ODIR}/gateway-pods.json" 2> "${ODIR}/gateway-pods-error.txt" || true

              mapfile -t GATEWAY_PODS < <(
                kubectl --context "${CTX}" -n grid-system get pods \
                  -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \
                  | awk '/^edge-gateway-/'
              )
              : > "${ODIR}/mounted-file-observations.jsonl"
              for pod in "${GATEWAY_PODS[@]}"; do
                gateway_container=$(kubectl --context "${CTX}" -n grid-system get pod "${pod}" -o json 2>/dev/null \
                  | jq -r '[.spec.containers[]? | select(any(.volumeMounts[]?; .name == "overlay" and .mountPath == "/etc/praxis/routing")) | .name] | if length == 1 then .[0] else empty end' 2>/dev/null \
                  || true)
                if [[ -z "${gateway_container}" ]]; then
                  echo "Could not uniquely identify the gateway container mounting /etc/praxis/routing" \
                    > "${ODIR}/${pod}-overlay-container-selection-error.txt"
                  kubectl --context "${CTX}" logs -n grid-system "${pod}" --all-containers=true --tail=2000 \
                    > "${ODIR}/${pod}-gateway-current.log" 2>&1 || true
                  kubectl --context "${CTX}" logs -n grid-system "${pod}" --all-containers=true --previous --tail=2000 \
                    > "${ODIR}/${pod}-gateway-previous.log" 2>&1 || true
                  continue
                fi
                for file in routing-overlay.json routing-config.json; do
                  mount_path="/etc/praxis/routing/${file}"
                  RAW_MOUNT="${RUNNER_TEMP}/${cluster}-${pod}-${file}"
                  FILE_TARGET_FILE="${ODIR}/${pod}-${file}-file-symlink.txt"
                  DATA_TARGET_FILE="${ODIR}/${pod}-${file}-data-symlink.txt"
                  projection_read_started_at=$(date -u +'%Y-%m-%dT%H:%M:%S.%NZ')
                  if kubectl --context "${CTX}" exec -n grid-system "${pod}" -c "${gateway_container}" -- \
                    readlink "${mount_path}" > "${FILE_TARGET_FILE}" 2> "${ODIR}/${pod}-${file}-file-symlink-error.txt"; then
                    file_symlink_status="success"
                  else
                    file_symlink_status="failed"
                  fi
                  if kubectl --context "${CTX}" exec -n grid-system "${pod}" -c "${gateway_container}" -- \
                    readlink /etc/praxis/routing/..data > "${DATA_TARGET_FILE}" 2> "${ODIR}/${pod}-${file}-data-symlink-error.txt"; then
                    data_symlink_status="success"
                  else
                    data_symlink_status="failed"
                  fi
                  projection_read_completed_at=$(date -u +'%Y-%m-%dT%H:%M:%S.%NZ')
                  read_started_at=$(date -u +'%Y-%m-%dT%H:%M:%S.%NZ')
                  if kubectl --context "${CTX}" exec -n grid-system "${pod}" -c "${gateway_container}" -- cat "${mount_path}" \
                    > "${RAW_MOUNT}" 2> "${ODIR}/${pod}-${file}-read-error.txt"; then
                    mounted_read_status="success"
                  else
                    mounted_read_status="failed"
                  fi
                  read_completed_at=$(date -u +'%Y-%m-%dT%H:%M:%S.%NZ')
                  file_target=$(cat "${FILE_TARGET_FILE}" 2>/dev/null || true)
                  data_target=$(cat "${DATA_TARGET_FILE}" 2>/dev/null || true)
                  jq -cn \
                    --arg cluster "${cluster}" --arg pod "${pod}" --arg file "${file}" --arg path "${mount_path}" \
                    --arg projection_read_started_at "${projection_read_started_at}" \
                    --arg projection_read_completed_at "${projection_read_completed_at}" \
                    --arg file_symlink_status "${file_symlink_status}" --arg data_symlink_status "${data_symlink_status}" \
                    --arg file_target "${file_target}" --arg data_target "${data_target}" \
                    --arg read_started_at "${read_started_at}" --arg read_completed_at "${read_completed_at}" \
                    --arg mounted_read_status "${mounted_read_status}" \
                    '{cluster: $cluster, pod: $pod, file: $file, path: $path,
                      projection_read_started_at: $projection_read_started_at,
                      projection_read_completed_at: $projection_read_completed_at,
                      file_symlink_status: $file_symlink_status, data_symlink_status: $data_symlink_status,
                      file_symlink_target: $file_target, data_symlink_target: $data_target,
                      read_started_at: $read_started_at, read_completed_at: $read_completed_at,
                      mounted_read_status: $mounted_read_status}' \
                    >> "${ODIR}/mounted-file-observations.jsonl"
                  if jq -e 'type == "object"' "${RAW_MOUNT}" > /dev/null 2>&1; then
                    if redact_overlay_json "${RAW_MOUNT}" "${ODIR}/${pod}-${file}-redacted.json"; then
                      digest=$(jq -S -c . "${ODIR}/${pod}-${file}-redacted.json" | sha256sum | awk '{print $1}')
                      printf '%s  %s/%s\n' "${digest}" "${pod}" "${file}" >> "${ODIR}/mounted-file-sha256.txt"
                    else
                      echo "Mounted file redaction failed; raw payload was not retained" > "${ODIR}/${pod}-${file}-redaction-error.txt"
                    fi
                  elif [[ "${mounted_read_status}" != "success" ]]; then
                    echo "Mounted-file read failed; see read error if present" > "${ODIR}/${pod}-${file}-snapshot-error.txt"
                  else
                    echo "Mounted file was unavailable or invalid JSON; see read error if present" > "${ODIR}/${pod}-${file}-snapshot-error.txt"
                  fi
                  rm -f "${RAW_MOUNT}"
                done
                kubectl --context "${CTX}" logs -n grid-system "${pod}" -c "${gateway_container}" --tail=2000 \
                  > "${ODIR}/${pod}-gateway-current.log" 2>&1 || true
                kubectl --context "${CTX}" logs -n grid-system "${pod}" -c "${gateway_container}" --previous --tail=2000 \
                  > "${ODIR}/${pod}-gateway-previous.log" 2>&1 || true
              done
              kubectl --context "${CTX}" logs -n grid-system deployment/grid-operator -c operator --tail=2000 \
                > "${ODIR}/grid-operator-current.log" 2>&1 || true
              kubectl --context "${CTX}" logs -n grid-system deployment/grid-operator -c operator --previous --tail=2000 \
                > "${ODIR}/grid-operator-previous.log" 2>&1 || true
            fi

            echo "::endgroup::"
          done

          echo "::group::Docker state"
          docker images > "${DIAG}/docker-images.txt" 2>&1 || true
          docker ps -a > "${DIAG}/docker-containers.txt" 2>&1 || true
          docker network ls > "${DIAG}/docker-networks-before-cleanup.txt" 2>&1 || true
          echo "::endgroup::"

          if [[ -n "${DEMO_DIR}" && -d "${DEMO_DIR}/.forge" ]]; then
            cp -r "${DEMO_DIR}/.forge" "${DIAG}/forge-output" 2>/dev/null || true
            find "${DIAG}/forge-output" \
              \( -name "*.key" -o -name "*.pem" -o -name "*.crt" \
                 -o -name "*.cer" -o -name "*.p12" -o -name "*.pfx" \
                 -o -name "*.jks" -o -name "*secret*" \
                 -o -name "*credential*" -o -name "*token*" \) \
              -type f -exec rm -f {} + 2>/dev/null || true
          fi

          if [[ -n "${DEMO_DIR}" && -d "${DEMO_DIR}/.forge/evidence" ]]; then
            cp -r "${DEMO_DIR}/.forge/evidence" \
              "${EVIDENCE_DIR}/forge" 2>/dev/null || true
          fi

          echo "Diagnostics collection complete"

      - name: Remove sensitive diagnostic content
        if: always()
        run: |
          set -euo pipefail

          # File-name filtering above does not catch PEM or credential material
          # embedded in logs, YAML, JSON, or ConfigMap output. Remove any file
          # containing recognizable private keys, certificates, or bearer tokens
          # before the artifact upload. Do not print matching content.
          for root in "${DIAGNOSTICS_DIR}" "${EVIDENCE_DIR}"; do
            [[ -d "${root}" ]] || continue
            while IFS= read -r -d '' file; do
              if grep -aEq -- \
                '-----BEGIN ([A-Z0-9 ]+ )?(PRIVATE KEY|CERTIFICATE)-----|Authorization:[[:space:]]*Bearer[[:space:]]+[A-Za-z0-9._~+/=-]{16,}|(client[_-]?secret|private[_-]?key|access[_-]?token)[[:space:]]*:[[:space:]]*[^[:space:]}]{16,}' \
                "${file}"; then
                echo "::warning::Removing sensitive diagnostic file: ${file#"${GITHUB_WORKSPACE}"/}"
                rm -f -- "${file}"
              fi
            done < <(find "${root}" -type f -print0)
          done

          echo "Sensitive diagnostic content scan complete"

      # ----------------------------------------------------------------------
      # Cleanup
      # ----------------------------------------------------------------------

      - name: Delete run-owned Kind resources
        id: cleanup
        if: always()
        continue-on-error: true
        run: |
          set -euo pipefail
          CLEANUP_RESULT="success"

          GRID_REPO="${GITHUB_WORKSPACE}/grid"
          STATE_FILE="${GRID_REPO}/.forge/state.json"
          OWNED_CLUSTERS=()
          OWNED_NETWORKS=()
          NETWORK_OWNER=""
          RUN_PREFIX=""
          RUN_CLUSTER=""
          RUN_NETWORK=""
          RESOLVED_CONFIG=""
          OPERATOR_ROUTING_STATE_DIR=""

          resolve_run_forge_state() {
            local topology_dir="$1"
            local state_dir_file="${EVIDENCE_DIR}/forge-state-dir.txt"
            local forge_state_dir=""

            if [[ ! -f "${state_dir_file}" ]]; then
              echo "::warning::Run-scoped Forge state path is absent; preserving its network"
              CLEANUP_RESULT="partial"
              STATE_FILE=""
              return 0
            fi
            IFS= read -r forge_state_dir < "${state_dir_file}"
            local expected_parent="${GRID_REPO}/tests/e2e/topologies/${topology_dir}/.forge"
            local expected_name_regex=""
            case "${topology_dir}" in
              grid-glb-demo)
                expected_name_regex='^\.forge\.resolved-[0-9]{8}T[0-9]{6}Z-[0-9]+$'
                ;;
              grid-combined-site)
                expected_name_regex='^\.forge\.resolved-[0-9]+-[0-9]+$'
                ;;
              *)
                echo "::warning::Unknown topology '${topology_dir}'; preserving its network"
                CLEANUP_RESULT="partial"
                STATE_FILE=""
                return 0
                ;;
            esac
            if [[ "${forge_state_dir%/*}" != "${expected_parent}" ]] \
              || ! [[ "${forge_state_dir##*/}" =~ ${expected_name_regex} ]]; then
              echo "::warning::Run-scoped Forge state path failed topology/run validation; preserving its network"
              CLEANUP_RESULT="partial"
              STATE_FILE=""
              return 0
            fi
            STATE_FILE="${forge_state_dir}/state.json"
          }

          case "${QUALIFICATION}" in
            glb)
              OWNED_CLUSTERS=(
                grid-glb-east-provider
                grid-glb-west-provider
                grid-glb-east-edge
                grid-glb-west-edge
                grid-glb-gtm-emulator
              )
              OWNED_NETWORKS=(grid-glb-demo-net)
              NETWORK_OWNER="grid-glb-demo"
              resolve_run_forge_state "grid-glb-demo"
              ;;
            static-weighted)
              RESOLVED_CONFIG="${EVIDENCE_DIR}/resolved-forge.yaml"
              STATE_FILE="${EVIDENCE_DIR}/forge-state/state.json"
              if [[ ! -f "${RESOLVED_CONFIG}" ]]; then
                echo "::warning::Static-weighted resolved Forge config is absent; refusing broad cleanup"
                CLEANUP_RESULT="partial"
              else
                RUN_PREFIX=$(sed -n 's/^    clusterPrefix:[[:space:]]*//p' "${RESOLVED_CONFIG}" | head -n 1 | tr -d '"')
                NETWORK_OWNER=$(sed -n 's/^  name:[[:space:]]*//p' "${RESOLVED_CONFIG}" | head -n 1 | tr -d '"')
                if ! [[ "${RUN_PREFIX}" =~ ^grid258-[0-9]+-[0-9]+$ ]] || [[ "${NETWORK_OWNER}" != "${RUN_PREFIX}" ]]; then
                  echo "::warning::Static-weighted Forge ownership could not be validated; refusing broad cleanup"
                  CLEANUP_RESULT="partial"
                elif [[ -f "${STATE_FILE}" ]] && ! jq -e --arg prefix "${RUN_PREFIX}" '
                  .network.name == ($prefix + "-net")
                  and ([.clusters[].kindName] - [
                    $prefix + "-provider-a",
                    $prefix + "-provider-b",
                    $prefix + "-provider-c"
                  ] | length == 0)
                ' "${STATE_FILE}" >/dev/null; then
                  echo "::warning::Static-weighted Forge state does not match resolved run ownership; refusing cleanup"
                  CLEANUP_RESULT="partial"
                else
                  OWNED_CLUSTERS=(
                    "${RUN_PREFIX}-provider-a"
                    "${RUN_PREFIX}-provider-b"
                    "${RUN_PREFIX}-provider-c"
                  )
                  OWNED_NETWORKS=("${RUN_PREFIX}-net")
                fi
              fi
              ;;
            combined-site)
              OWNED_CLUSTERS=(
                grid-combined-west
                grid-combined-central
                grid-combined-east
              )
              OWNED_NETWORKS=(grid-combined-site-net)
              NETWORK_OWNER="grid-combined-site"
              resolve_run_forge_state "grid-combined-site"
              ;;
            llmd-pool-metrics)
              BASE_KIND="${DIAGNOSTICS_DIR}/llmd-kind-before.txt"
              BASE_NETWORKS="${DIAGNOSTICS_DIR}/llmd-networks-before.txt"
              if [[ ! -f "${BASE_KIND}" || ! -f "${BASE_NETWORKS}" ]]; then
                echo "::warning::llm-d pre-run inventory is absent; refusing broad cleanup"
                CLEANUP_RESULT="partial"
              fi

              mapfile -t LLMD_STATE_FILES < <(find "${EVIDENCE_DIR}" -type f -path '*/forge-state/state.json' -print 2>/dev/null | sort)
              if [[ "${#LLMD_STATE_FILES[@]}" -ne 1 ]]; then
                echo "::warning::Expected one run-scoped llm-d Forge state file, found ${#LLMD_STATE_FILES[@]}"
                CLEANUP_RESULT="partial"
              else
                STATE_FILE="${LLMD_STATE_FILES[0]}"
                if ! POOL_A_CLUSTER=$(jq -er '.clusters[] | select(.name == "pool-a") | .kindName' "${STATE_FILE}"); then
                  echo "::warning::Unable to parse pool-a cluster from llm-d Forge state; refusing cleanup"
                  CLEANUP_RESULT="partial"
                  STATE_FILE=""
                else
                  RUN_PREFIX="${POOL_A_CLUSTER%-pool-a}"
                  if ! [[ "${RUN_PREFIX}" =~ ^grid-llmd-pm-[0-9]{8}t[0-9]{6}z-[0-9]+$ ]] \
                    || ! jq -e --arg prefix "${RUN_PREFIX}" '
                      .network.name == ($prefix + "-net")
                      and ([.clusters[].kindName] | sort == [($prefix + "-pool-a"), ($prefix + "-pool-b")])
                    ' "${STATE_FILE}" > /dev/null; then
                    echo "::warning::llm-d Forge state does not match a unique two-pool run prefix; refusing cleanup"
                    CLEANUP_RESULT="partial"
                    STATE_FILE=""
                  else
                    OWNED_CLUSTERS=("${RUN_PREFIX}-pool-a" "${RUN_PREFIX}-pool-b")
                    OWNED_NETWORKS=("${RUN_PREFIX}-net")
                    NETWORK_OWNER="${RUN_PREFIX}"
                    if ! jq -e '.networkCreatedByForge == true or .network.phase == "gone"' "${STATE_FILE}" > /dev/null; then
                      echo "::warning::llm-d Forge state does not show a run-created or already-removed network"
                      CLEANUP_RESULT="partial"
                      OWNED_NETWORKS=()
                    fi
                  fi
                fi
              fi
              ;;
            operator-routing)
              for _ in operator-routing-cleanup-identity; do
                EVIDENCE_FILE="${EVIDENCE_DIR}/evidence.json"
                if [[ ! -f "${EVIDENCE_FILE}" ]]; then
                  if kind get clusters 2>/dev/null | grep -Eq '^grid-operator-routing-[0-9]+-[0-9]+-local$' \
                    || docker network ls --format '{{.Name}}' | grep -Eq '^grid-operator-routing-[0-9]+-[0-9]+-net$'; then
                    echo "::warning::Operator-routing evidence is absent while a matching resource exists; preserving it"
                    CLEANUP_RESULT="partial"
                  fi
                  break
                fi
                if ! RUN_CLUSTER=$(jq -er '.cluster_name' "${EVIDENCE_FILE}") \
                  || ! RESOLVED_CONFIG=$(jq -er '.observations.forge_working_config' "${EVIDENCE_FILE}") \
                  || ! OPERATOR_ROUTING_STATE_DIR=$(jq -er '.observations.forge_state_dir' "${EVIDENCE_FILE}") \
                  || ! RUN_NETWORK=$(jq -er '.observations.forge_network_name' "${EVIDENCE_FILE}"); then
                  echo "::warning::Operator-routing evidence lacks exact cleanup identities; preserving resources"
                  CLEANUP_RESULT="partial"
                  break
                fi
                RUN_SUFFIX="${RUN_CLUSTER#grid-operator-routing-}"
                RUN_SUFFIX="${RUN_SUFFIX%-local}"
                RUN_PREFIX="grid-operator-routing-${RUN_SUFFIX}"
                if ! [[ "${RUN_SUFFIX}" =~ ^[0-9]+-[0-9]+$ ]] \
                  || [[ "${RUN_CLUSTER}" != "${RUN_PREFIX}-local" ]] \
                  || [[ "${RUN_NETWORK}" != "${RUN_PREFIX}-net" ]]; then
                  echo "::warning::Operator-routing run identity is malformed; refusing cleanup"
                  CLEANUP_RESULT="partial"
                  break
                fi
                if [[ "${RESOLVED_CONFIG}" != /* ]]; then
                  RESOLVED_CONFIG="${GRID_REPO}/${RESOLVED_CONFIG}"
                fi
                EXPECTED_TOPOLOGY_DIR="${GRID_REPO}/tests/e2e/topologies/grid-operator-routing"
                EXPECTED_CONFIG="${EXPECTED_TOPOLOGY_DIR}/.forge.resolved-${RUN_SUFFIX}.yaml"
                EXPECTED_STATE_DIR="${EXPECTED_TOPOLOGY_DIR}/.forge/.forge.resolved-${RUN_SUFFIX}"
                if [[ "$(realpath -m "${RESOLVED_CONFIG}")" != "$(realpath -m "${EXPECTED_CONFIG}")" ]] \
                  || [[ "$(realpath -m "${OPERATOR_ROUTING_STATE_DIR}")" != "$(realpath -m "${EXPECTED_STATE_DIR}")" ]]; then
                  echo "::warning::Operator-routing cleanup paths do not match the exact run identity; refusing cleanup"
                  CLEANUP_RESULT="partial"
                  break
                fi
                STATE_FILE="${OPERATOR_ROUTING_STATE_DIR}/state.json"
                if [[ -f "${STATE_FILE}" ]]; then
                  if ! jq -e --arg cluster "${RUN_CLUSTER}" --arg network "${RUN_NETWORK}" '
                    (.clusters | length <= 1)
                    and ([.clusters[]? | select(.name != "local" or .kindName != $cluster or .context != ("kind-" + $cluster))] | length == 0)
                    and (.network == null or .network.name == $network)
                  ' "${STATE_FILE}" > /dev/null; then
                    echo "::warning::Operator-routing Forge state does not match its unique cluster/network identity; refusing cleanup"
                    CLEANUP_RESULT="partial"
                    STATE_FILE=""
                    break
                  fi
                  OWNED_CLUSTERS=("${RUN_CLUSTER}")
                  if jq -e '.networkCreatedByForge == true' "${STATE_FILE}" > /dev/null \
                    && jq -e --arg network "${RUN_NETWORK}" '.network.name == $network' "${STATE_FILE}" > /dev/null; then
                    OWNED_NETWORKS=("${RUN_NETWORK}")
                    NETWORK_OWNER="${RUN_PREFIX}"
                  elif docker network inspect "${RUN_NETWORK}" > /dev/null 2>&1; then
                    echo "::warning::Operator-routing network exists but Forge state does not prove ownership; preserving it"
                    CLEANUP_RESULT="partial"
                  fi
                elif kind get clusters 2>/dev/null | grep -Fxq "${RUN_CLUSTER}" \
                  || docker network inspect "${RUN_NETWORK}" > /dev/null 2>&1; then
                  echo "::warning::Operator-routing resources exist without matching Forge state; preserving them"
                  CLEANUP_RESULT="partial"
                fi
              done
              ;;
            *)
              echo "::warning::Unknown qualification '${QUALIFICATION}'; refusing cleanup"
              CLEANUP_RESULT="partial"
              ;;
          esac

          for cluster in "${OWNED_CLUSTERS[@]}"; do
            if ! kind get clusters 2>/dev/null | grep -Fxq "${cluster}"; then
              continue
            fi
            if [[ "${QUALIFICATION}" == "llmd-pool-metrics" ]] \
              && [[ -f "${DIAGNOSTICS_DIR}/llmd-kind-before.txt" ]] \
              && grep -Fxq "${cluster}" "${DIAGNOSTICS_DIR}/llmd-kind-before.txt"; then
              echo "::warning::Preserving pre-existing Kind cluster '${cluster}'"
              CLEANUP_RESULT="partial"
              continue
            fi
            echo "Deleting run-owned cluster: ${cluster}"
            if ! kind delete cluster --name "${cluster}"; then
              echo "::warning::Failed to delete run-owned Kind cluster '${cluster}'"
              CLEANUP_RESULT="partial"
            fi
          done

          for net in "${OWNED_NETWORKS[@]}"; do
            if ! docker network inspect "${net}" >/dev/null 2>&1; then
              continue
            fi
            if [[ "${QUALIFICATION}" == "llmd-pool-metrics" ]] \
              && [[ -f "${DIAGNOSTICS_DIR}/llmd-networks-before.txt" ]] \
              && awk -v name="${net}" '$1 == name { found=1 } END { exit !found }' "${DIAGNOSTICS_DIR}/llmd-networks-before.txt"; then
              echo "::warning::Preserving pre-existing Docker network '${net}'"
              CLEANUP_RESULT="partial"
              continue
            fi
            LABELS=$(docker network inspect --format '{{.Labels}}' "${net}" 2>/dev/null || true)
            if [[ "${LABELS}" != *"forge.managed:true"* || "${LABELS}" != *"forge.environment:${NETWORK_OWNER}"* ]]; then
              echo "::warning::Preserving network '${net}': Forge ownership labels do not match ${NETWORK_OWNER}"
              CLEANUP_RESULT="partial"
              continue
            fi
            if [[ ! -f "${STATE_FILE}" ]] || ! jq -e '.networkCreatedByForge == true' "${STATE_FILE}" >/dev/null 2>&1; then
              echo "::warning::Preserving network '${net}': Forge state says it was pre-existing"
              CLEANUP_RESULT="partial"
              continue
            fi
            if ! docker network rm "${net}" >/dev/null; then
              echo "::warning::Failed to delete run-owned Docker network '${net}'"
              CLEANUP_RESULT="partial"
            fi
          done

          if [[ "${QUALIFICATION}" == "operator-routing" && -n "${RUN_CLUSTER}" \
            && "${CLEANUP_RESULT}" == "success" ]]; then
            if kind get clusters 2>/dev/null | grep -Fxq "${RUN_CLUSTER}"; then
              echo "::warning::Run-owned operator-routing cluster remains after cleanup: ${RUN_CLUSTER}"
              CLEANUP_RESULT="partial"
            elif docker network inspect "${RUN_NETWORK}" > /dev/null 2>&1; then
              echo "::warning::Run-owned operator-routing network remains after cleanup: ${RUN_NETWORK}"
              CLEANUP_RESULT="partial"
            else
              if [[ -d "${OPERATOR_ROUTING_STATE_DIR}" ]] \
                && ! rm -rf -- "${OPERATOR_ROUTING_STATE_DIR}"; then
                echo "::warning::Could not remove run-owned operator-routing Forge state"
                CLEANUP_RESULT="partial"
              fi
              if [[ -f "${RESOLVED_CONFIG}" ]] && ! rm -f -- "${RESOLVED_CONFIG}"; then
                echo "::warning::Could not remove run-owned resolved Forge config"
                CLEANUP_RESULT="partial"
              fi
            fi
          fi

          if [[ "${QUALIFICATION}" == "llmd-pool-metrics" ]]; then
            kind get clusters | sort > "${DIAGNOSTICS_DIR}/llmd-kind-after-cleanup.txt" 2>&1 || true
            docker network ls --format '{{.Name}} {{.ID}}' | sort > "${DIAGNOSTICS_DIR}/llmd-networks-after-cleanup.txt"
            docker ps -a --format '{{.ID}} {{.Names}}' | sort > "${DIAGNOSTICS_DIR}/llmd-containers-after-cleanup.txt"
            for cluster in "${OWNED_CLUSTERS[@]}"; do
              if grep -Fxq "${cluster}" "${DIAGNOSTICS_DIR}/llmd-kind-after-cleanup.txt"; then
                echo "::warning::Run-owned llm-d Kind cluster remains after cleanup: ${cluster}"
                CLEANUP_RESULT="partial"
              fi
            done
            for net in "${OWNED_NETWORKS[@]}"; do
              if awk -v name="${net}" '$1 == name { found=1 } END { exit !found }' "${DIAGNOSTICS_DIR}/llmd-networks-after-cleanup.txt"; then
                echo "::warning::Run-owned llm-d Docker network remains after cleanup: ${net}"
                CLEANUP_RESULT="partial"
              fi
            done
            if [[ -n "${RUN_PREFIX}" && -f "${DIAGNOSTICS_DIR}/llmd-containers-before.txt" ]]; then
              while read -r container_id container_name; do
                [[ "${container_name}" == "${RUN_PREFIX}-"* ]] || continue
                echo "::warning::Run-owned llm-d Docker container remains after cleanup: ${container_name} (${container_id})"
                CLEANUP_RESULT="partial"
              done < <(comm -13 "${DIAGNOSTICS_DIR}/llmd-containers-before.txt" "${DIAGNOSTICS_DIR}/llmd-containers-after-cleanup.txt")
            fi
          fi

          for cluster in "${OWNED_CLUSTERS[@]}"; do
            rm -rf "/tmp/kind-${cluster}" 2>/dev/null || true
          done

          echo "result=${CLEANUP_RESULT}" >> "${GITHUB_OUTPUT}"
          echo "Cleanup: ${CLEANUP_RESULT}"

      - name: Record post-cleanup inventory
        if: always()
        run: |
          mkdir -p "${DIAGNOSTICS_DIR}"
          kind get clusters > "${DIAGNOSTICS_DIR}/kind-clusters-after-cleanup.txt" 2>&1 || true
          docker ps -a > "${DIAGNOSTICS_DIR}/docker-containers-after-cleanup.txt" 2>&1 || true
          docker network ls > "${DIAGNOSTICS_DIR}/docker-networks-after-cleanup.txt" 2>&1 || true

      # ----------------------------------------------------------------------
      # Artifact upload
      # ----------------------------------------------------------------------

      - name: Upload artifacts
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: ${{ steps.tags.outputs.artifact_prefix || format('grid-{0}-{1}', needs.resolve-source.outputs.source_kind, needs.resolve-source.outputs.tested_sha) }}-operator-${{ needs.resolve-source.outputs.operator_image_mode }}-${{ env.QUALIFICATION }}-${{ env.MODE }}-${{ env.MEMBERSHIP }}
          path: |
            ${{ env.EVIDENCE_DIR }}/
            ${{ env.DIAGNOSTICS_DIR }}/
            ${{ github.workspace }}/run-${{ env.QUALIFICATION }}.log
            ${{ github.workspace }}/image-manifest.txt
            ${{ github.workspace }}/source-metadata.json
            ${{ github.workspace }}/versions.txt
          retention-days: 14
          if-no-files-found: warn

      # ----------------------------------------------------------------------
      # Job summary
      # ----------------------------------------------------------------------

      - name: Write job summary
        if: always()
        env:
          TAG: ${{ steps.tags.outputs.tag || 'unknown' }}
          OPERATOR_IMAGE: ${{ steps.tags.outputs.operator_image || 'unknown' }}
          OPERATOR_IMAGE_MODE: ${{ needs.resolve-source.outputs.operator_image_mode }}
          ARTIFACT_PREFIX: ${{ steps.tags.outputs.artifact_prefix || format('grid-{0}-{1}', needs.resolve-source.outputs.source_kind, needs.resolve-source.outputs.tested_sha) }}
          QUALIFICATION_RESULT: ${{ steps.qualification.outcome }}
          CLEANUP_RESULT: ${{ steps.cleanup.outputs.result || 'unknown' }}
          RESOLVED_DIGEST: ${{ steps.gateway.outputs.digest || 'unknown' }}
          PR_TITLE: ${{ needs.resolve-source.outputs.title }}
          PR_URL: ${{ needs.resolve-source.outputs.url }}
          PR_AUTHOR: ${{ needs.resolve-source.outputs.author }}
          PR_DRAFT: ${{ needs.resolve-source.outputs.draft }}
        run: |
          case "${QUALIFICATION}" in
            glb)
              QUALIFICATION_DISPLAY="GLB"
              TOPOLOGY="tests/e2e/topologies/grid-glb-demo"
              SIMULATOR="${SIM_IMAGE_TAG} (pinned from ${SIM_IMAGE_REF})"
              ;;
            static-weighted)
              QUALIFICATION_DISPLAY="Static weighted provider selection"
              TOPOLOGY="tests/e2e/topologies/grid-static-weighted"
              SIMULATOR="${STATIC_WEIGHTED_SIM_IMAGE_TAG} (pinned from ${STATIC_WEIGHTED_SIM_IMAGE_REF})"
              ;;
            combined-site)
              QUALIFICATION_DISPLAY="Combined-site lifecycle"
              TOPOLOGY="tests/e2e/topologies/grid-combined-site"
              SIMULATOR="${COMBINED_SITE_SIM_IMAGE_TAG} (default tag pinned to ${COMBINED_SITE_SIM_IMAGE_REF})"
              ;;
            llmd-pool-metrics)
              QUALIFICATION_DISPLAY="llm-d pool-metrics and signal lifecycle"
              TOPOLOGY="tests/e2e/topologies/grid-llmd-pool-metrics"
              SIMULATOR="${LLMD_SIM_IMAGE_TAG} (pinned from ${LLMD_SIM_IMAGE_REF})"
              ;;
            operator-routing)
              QUALIFICATION_DISPLAY="Operator-derived local routing and passive-health recovery"
              TOPOLOGY="tests/e2e/topologies/grid-operator-routing"
              SIMULATOR="not used; source-built mock-provider fixture"
              ;;
            *)
              QUALIFICATION_DISPLAY="Unknown (${QUALIFICATION})"
              TOPOLOGY="unknown"
              SIMULATOR="unknown"
              ;;
          esac

          case "${QUALIFICATION_RESULT}" in
            success) QUALIFICATION_DISPLAY_RESULT="success" ;;
            failure) QUALIFICATION_DISPLAY_RESULT="failure" ;;
            *) QUALIFICATION_DISPLAY_RESULT="${QUALIFICATION_RESULT}" ;;
          esac

          {
            echo "## Triggered Integration Test: ${QUALIFICATION_DISPLAY}"
            echo ""
            echo "| Field | Value |"
            echo "|-------|-------|"
            echo "| **Source** | ${SOURCE_LABEL} |"
            echo "| **Tested SHA** | \`${TESTED_SHA}\` |"
            echo "| **Qualification** | ${QUALIFICATION_DISPLAY} |"

            if [[ "${SOURCE_KIND}" == "pr" ]]; then
              echo "| **PR Title** | ${PR_TITLE} |"
              echo "| **PR URL** | ${PR_URL} |"
              echo "| **PR Author** | ${PR_AUTHOR} |"
              echo "| **Draft** | ${PR_DRAFT} |"
              echo "| **Head SHA** | \`${HEAD_SHA}\` |"
              echo "| **Base SHA** | \`${BASE_SHA}\` |"
              echo "| **Source verification** | PR merge verified; not base/main |"
            elif [[ "${SOURCE_KIND}" == "branch" ]]; then
              echo "| **Branch** | \`${BRANCH_NAME}\` |"
              echo "| **Source verification** | Branch SHA verified |"
            else
              echo "| **Source verification** | Main SHA verified |"
            fi

            echo "| **Topology** | \`${TOPOLOGY}\` at tested Grid SHA |"
            echo "| **Mode** | \`${MODE}\` |"
            echo "| **Operator image mode** | \`${OPERATOR_IMAGE_MODE}\` |"
            if [[ "${QUALIFICATION}" == "combined-site" && "${OPERATOR_IMAGE_MODE}" == "standard" ]]; then
              echo "| **Operator runtime scope** | Standard non-FIPS image; this run does not qualify FIPS startup or a FIPS-mode host |"
            elif [[ "${OPERATOR_IMAGE_MODE}" == "fips" ]]; then
              echo "| **Operator runtime scope** | FIPS startup was required on a FIPS-enabled runner; see the qualification result for whether it passed |"
            fi
            echo "| **Membership** | \`${MEMBERSHIP}\` |"
            echo "| **Qualification result** | ${QUALIFICATION_DISPLAY_RESULT} |"
            echo "| **Cleanup** | ${CLEANUP_RESULT} |"
            if [[ "${QUALIFICATION}" == "llmd-pool-metrics" ]]; then
              XTASK_EXIT="unknown"
              if [[ -f "${EVIDENCE_DIR}/xtask-exit-status.txt" ]]; then
                XTASK_EXIT=$(cat "${EVIDENCE_DIR}/xtask-exit-status.txt")
              fi
              echo "| **xtask exit status** | ${XTASK_EXIT} |"
            elif [[ "${QUALIFICATION}" == "operator-routing" ]]; then
              XTASK_EXIT="unknown"
              if [[ -f "${EVIDENCE_DIR}/xtask-exit-status.txt" ]]; then
                XTASK_EXIT=$(cat "${EVIDENCE_DIR}/xtask-exit-status.txt")
              fi
              echo "| **xtask exit status** | ${XTASK_EXIT} |"
            fi
            echo ""
            echo "### Images"
            echo ""
            echo "| Image | Reference |"
            echo "|-------|-----------|"
            echo "| Grid Operator | \`${OPERATOR_IMAGE}\` |"
            if [[ "${QUALIFICATION}" == "operator-routing" ]]; then
              echo "| Grid Gateway | \`grid-gateway:${TAG}\` (built from tested Grid SHA) |"
              echo "| Mock providers | \`grid-mock-providers:${TAG}\` (built from tested Grid SHA) |"
              echo "| Qualification client | \`${PROBE_IMAGE_REF}\` (pinned digest) |"
            else
              echo "| Gateway (pinned) | \`${GATEWAY_IMAGE_REF}\` (\`${RESOLVED_DIGEST}\`) |"
              echo "| Simulator | \`${SIMULATOR}\` |"
            fi
            if [[ "${QUALIFICATION}" == "llmd-pool-metrics" ]]; then
              echo "| llm-d EPP | \`${LLMD_EPP_IMAGE_TAG}\` (pinned from \`${LLMD_EPP_IMAGE_REF}\`) |"
              if [[ "${MODE}" == "full" ]]; then
                echo "| Grid Gateway | \`grid-gateway:${TAG}\` (built from tested Grid SHA) |"
              fi
            fi
            echo ""
            echo "### Artifact"
            echo ""
            echo "\`${ARTIFACT_PREFIX}-operator-${OPERATOR_IMAGE_MODE}-${QUALIFICATION}-${MODE}-${MEMBERSHIP}\`"
          } >> "${GITHUB_STEP_SUMMARY}"