name: Reproduce LiteLLM source with Presidio

on:
  workflow_dispatch:
    inputs:
      source_ref:
        description: LiteLLM tag, branch or commit; blank tests this repository's commit
        required: false
        type: string
      source_repository:
        description: Repository containing the selected LiteLLM source
        required: false
        type: string
  pull_request:
    paths:
      - .github/workflows/litellm-source-reproduction.yml

permissions:
  contents: read

jobs:
  reproduce:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      # Names this run on the results wall. Change them if your fork measures something else.
      WALL_PRODUCT: LiteLLM
      WALL_CONFIGURATION: Presidio pre_call guardrail with output_parse_pii
      WALL_LICENSE: MIT outside enterprise/
      WALL_PROJECT_URL: https://github.com/BerriAI/litellm
      SOURCE_REPOSITORY: ${{ inputs.source_repository || (github.repository == 'ninadphalak/LLM-Shield-Proxy' && 'BerriAI/litellm' || github.repository) }}
      SOURCE_SELECTOR: ${{ inputs.source_ref || (github.repository == 'ninadphalak/LLM-Shield-Proxy' && 'v1.99.0' || github.sha) }}
      SOURCE_IMAGE: litellm-source-reproduction:${{ github.run_id }}
      RUNTIME_IMAGE: ghcr.io/berriai/litellm@sha256:570a872d2fde8f1bc4a147634810941103c14697270f0f2918c5aa7d8201cac5
      POSTGRES_IMAGE: postgres@sha256:a3b7f434b2dc57ce85a67e171163eb8ab1a1ebcb39d27484661f26b1dfbe30d6
      ANALYZER_IMAGE: mcr.microsoft.com/presidio-analyzer@sha256:286e3fa7f3a7426e775e8564fe1870f1ba8f999d3ab8bbb8cc46a44355d9d6e9
      ANONYMIZER_IMAGE: mcr.microsoft.com/presidio-anonymizer@sha256:a10a12a2a613d13cf29d3ad3641e3258444dd8c90403dd644a0a114c472c2483
      CONFORMANCE_TARGET_API_KEY: sk-v2-profile-local
      V2_GATEWAY_TOKEN: sk-v2-profile-local

    steps:
      - name: Require a ref for another repository
        id: ref_check
        if: inputs.source_repository != '' && inputs.source_ref == ''
        run: |
          echo 'Choose an explicit tag or commit when testing another repository.'
          exit 2

      - name: Check out selected LiteLLM source
        id: checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          repository: ${{ env.SOURCE_REPOSITORY }}
          ref: ${{ env.SOURCE_SELECTOR }}
          path: proxy-source
          persist-credentials: false

      - name: Check out the pinned response instrument
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          repository: ninadphalak/LLM-Shield-Proxy
          ref: 74d744a72adb053ff4f4025bdd676bba0e5be03d # benchmark-v0.2.1
          path: benchmark-instrument
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
        with:
          python-version: '3.12'

      - name: Build source image over pinned runtime dependencies
        id: build
        run: |
          set -euo pipefail
          python -m pip install 'pii-leak-benchmark[validate]==0.2.1'
          out="$RUNNER_TEMP/litellm-source-reproduction"
          mkdir -p "$out"
          git -C proxy-source rev-parse HEAD > "$out/source-commit.txt"
          python -m pip freeze > "$out/response-harness-packages.txt"
          python --version > "$out/python-version.txt" 2>&1
          uname -a > "$out/runner.txt"
          docker version --format '{{.Server.Version}}' > "$out/docker-version.txt"
          for attempt in 1 2 3 4; do
            if docker pull "$RUNTIME_IMAGE"; then break; fi
            if [ "$attempt" = 4 ]; then exit 2; fi
            sleep $((2 ** attempt))
          done
          docker build --file - --tag "$SOURCE_IMAGE" proxy-source <<'DOCKERFILE'
          ARG RUNTIME_IMAGE=ghcr.io/berriai/litellm@sha256:570a872d2fde8f1bc4a147634810941103c14697270f0f2918c5aa7d8201cac5
          FROM ${RUNTIME_IMAGE}
          RUN mv /app/.venv/lib/python3.13/site-packages/litellm /app/litellm-runtime-base
          COPY litellm/ /app/.venv/lib/python3.13/site-packages/litellm/
          RUN cp /app/litellm-runtime-base/rust_bridge/_native*.so /app/.venv/lib/python3.13/site-packages/litellm/rust_bridge/
          # A fork's newer source can import a package the pinned image predates (main added
          # httpx2 after v1.99.0). Install what the selected source declares for the proxy;
          # anything the image already satisfies is left exactly as pinned.
          COPY pyproject.toml /tmp/source-pyproject.toml
          RUN /app/.venv/bin/python -c "import tomllib; p = tomllib.load(open('/tmp/source-pyproject.toml', 'rb'))['project']; open('/tmp/source-requirements.txt', 'w').write(chr(10).join(p.get('dependencies', []) + p.get('optional-dependencies', {}).get('proxy', [])) + chr(10))" \
              && /app/.venv/bin/python -m ensurepip --upgrade > /dev/null \
              && /app/.venv/bin/python -m pip install --disable-pip-version-check --no-cache-dir -r /tmp/source-requirements.txt \
              && /app/.venv/bin/python -m pip freeze > /app/source-build-packages.txt \
              && /app/.venv/bin/python -c "import litellm.proxy.proxy_server"
          DOCKERFILE
          docker image inspect "$SOURCE_IMAGE" --format '{{.Id}}' > "$out/image-id.txt"
          docker run --rm --entrypoint cat "$SOURCE_IMAGE" /app/source-build-packages.txt > "$out/proxy-packages.txt"
          python - <<'PY'
          import json
          import os
          from pathlib import Path
          root = Path(os.environ['RUNNER_TEMP']) / 'litellm-source-reproduction'
          identity = {
              'schema': 'pii-leak-benchmark/source-build/v1',
              'source_repository': os.environ['SOURCE_REPOSITORY'],
              'source_selector': os.environ['SOURCE_SELECTOR'],
              'source_commit': (root / 'source-commit.txt').read_text(encoding='utf-8').strip(),
              'product': os.environ['WALL_PRODUCT'],
              'configuration': os.environ['WALL_CONFIGURATION'],
              'license': os.environ['WALL_LICENSE'],
              'project_url': os.environ['WALL_PROJECT_URL'],
              'runtime_image': os.environ['RUNTIME_IMAGE'],
              'build_method': ('selected Python source over the v1.99.0 runtime image and native bridge, '
                               'plus any proxy dependency the source declares that the image lacks'),
          }
          (root / 'source-identity.json').write_text(
              json.dumps(identity, indent=2, sort_keys=True) + '\n', encoding='utf-8'
          )
          PY

      - name: Start pinned local dependencies
        # Every docker call below is bounded, and so is the step: one run hung silently for
        # 17 minutes inside a single unbounded call and would otherwise have held the job.
        timeout-minutes: 12
        run: |
          set -euo pipefail
          for image in "$POSTGRES_IMAGE" "$ANALYZER_IMAGE" "$ANONYMIZER_IMAGE"; do
            for attempt in 1 2 3 4; do
              if docker pull "$image"; then break; fi
              if [ "$attempt" = 4 ]; then exit 2; fi
              sleep $((2 ** attempt))
            done
          done
          # The Presidio images run gunicorn 25, whose master starts a control-socket thread
          # and then forks the worker. Intermittently the worker froze straight after fork,
          # before logging "Booting worker", and never answered /health: a lock held by that
          # thread at fork time. Caught on a hung runner: master logged "Control socket
          # listening", the worker process existed and never logged a line, and the image's
          # own in-container health checks piled up unanswered. --no-control-socket removes
          # the thread. Host networking on job-chosen ports keeps port publishing out of it.
          read -r pg_port analyzer_port anonymizer_port < <(python -c '
          import socket
          sockets = [socket.socket() for _ in range(3)]
          for sock in sockets:
              sock.bind(("127.0.0.1", 0))
          print(" ".join(str(sock.getsockname()[1]) for sock in sockets))
          for sock in sockets:
              sock.close()')
          pg=$(timeout 120 docker run -d --network host \
            -e POSTGRES_PASSWORD=litellm -e POSTGRES_USER=litellm -e POSTGRES_DB=litellm \
            "$POSTGRES_IMAGE" -p "$pg_port" -c listen_addresses=127.0.0.1)
          analyzer=$(timeout 120 docker run -d --network host -e PORT="$analyzer_port" \
            -e GUNICORN_CMD_ARGS='--no-control-socket --timeout 300' "$ANALYZER_IMAGE")
          anonymizer=$(timeout 120 docker run -d --network host -e PORT="$anonymizer_port" \
            -e GUNICORN_CMD_ARGS='--no-control-socket --timeout 300' "$ANONYMIZER_IMAGE")
          echo "PG_CONTAINER=$pg" >> "$GITHUB_ENV"
          echo "ANALYZER_CONTAINER=$analyzer" >> "$GITHUB_ENV"
          echo "ANONYMIZER_CONTAINER=$anonymizer" >> "$GITHUB_ENV"
          echo "DATABASE_URL=postgresql://litellm:litellm@127.0.0.1:$pg_port/litellm" >> "$GITHUB_ENV"
          echo "ANALYZER_PORT=$analyzer_port" >> "$GITHUB_ENV"
          echo "ANONYMIZER_PORT=$anonymizer_port" >> "$GITHUB_ENV"
          deadline=$((SECONDS + 360))
          attempt=0
          while [ "$SECONDS" -lt "$deadline" ]; do
            attempt=$((attempt + 1))
            if timeout 10 docker exec "$pg" pg_isready -h 127.0.0.1 -p "$pg_port" -U litellm -d litellm > /dev/null 2>&1 \
              && curl --max-time 5 -sf -o /dev/null "http://127.0.0.1:$analyzer_port/health" \
              && curl --max-time 5 -sf -o /dev/null "http://127.0.0.1:$anonymizer_port/health"; then
              echo "Local dependencies ready after $attempt checks."
              exit 0
            fi
            if [ $((attempt % 15)) = 0 ]; then echo "Still waiting for local dependencies ($attempt checks)."; fi
            sleep 2
          done
          echo 'A local dependency did not become ready within six minutes.'
          timeout 10 docker exec "$pg" pg_isready -h 127.0.0.1 -p "$pg_port" -U litellm -d litellm || true
          for port in "$analyzer_port" "$anonymizer_port"; do
            curl --max-time 5 --silent --show-error -o /dev/null -w "port=$port status=%{http_code}\n" \
              "http://127.0.0.1:$port/health" || true
          done
          free -h || true
          for id in "$pg" "$analyzer" "$anonymizer"; do
            timeout 10 docker inspect "$id" --format 'container={{.Name}} state={{.State.Status}} exit={{.State.ExitCode}} oom={{.State.OOMKilled}}' || true
            timeout 10 docker logs --tail 60 "$id" 2>&1 || true
          done
          exit 2

      - name: Write exact operator and response configurations
        run: |
          set -euo pipefail
          python - <<'PY'
          import os
          import socket
          from pathlib import Path

          root = Path(os.environ['RUNNER_TEMP']) / 'litellm-source-reproduction'
          with socket.socket() as listener:
              listener.bind(('127.0.0.1', 0))
              gateway_port = listener.getsockname()[1]
          with open(os.environ['GITHUB_ENV'], 'a', encoding='utf-8') as env:
              env.write(f'LITELLM_PORT={gateway_port}\n')
          for name, capture_port in (('operator', 8765), ('response', 8799)):
              config = f'''model_list:
            - model_name: capture
              litellm_params:
                model: openai/capture
                api_base: http://127.0.0.1:{capture_port}/v1
                api_key: sk-not-used-by-the-capture
          guardrails:
            - guardrail_name: presidio-mask
              litellm_params:
                guardrail: presidio
                mode: pre_call
                output_parse_pii: true
                presidio_analyzer_api_base: http://127.0.0.1:{os.environ['ANALYZER_PORT']}
                presidio_anonymizer_api_base: http://127.0.0.1:{os.environ['ANONYMIZER_PORT']}
                default_on: true
          litellm_settings:
            drop_params: true
          general_settings:
            master_key: sk-v2-profile-local
          '''
              (root / f'{name}-config.yaml').write_text(config, encoding='utf-8')
          PY

      - name: Measure request-path behavior
        id: operator
        continue-on-error: true
        uses: ninadphalak/LLM-Shield-Proxy@29a9932b085e5b063b8c366c3e6bfca2c4af4b06 # operator action; the recipe writes the submission
        with:
          target-base-url: http://127.0.0.1:${{ env.LITELLM_PORT }}/v1
          start-command: >-
            docker run --rm --network host --name litellm-operator-$GITHUB_RUN_ID
            -e DATABASE_URL=$DATABASE_URL -e LITELLM_MASTER_KEY=sk-v2-profile-local
            -v "$RUNNER_TEMP/litellm-source-reproduction/operator-config.yaml:/app/config.yaml:ro"
            "$SOURCE_IMAGE" --config /app/config.yaml --port "$LITELLM_PORT"
          target-model: capture
          target-version: ${{ env.SOURCE_SELECTOR }}
          duty: restore
          artifact-name: ''
          submission-section: 'false'

      - name: Stage the operator result
        if: always() && steps.operator.outputs.report-directory != ''
        env:
          OPERATOR_REPORT_DIRECTORY: ${{ steps.operator.outputs.report-directory }}
        run: |
          set -euo pipefail
          out="$RUNNER_TEMP/litellm-source-reproduction"
          for name in current.json summary.md; do
            if [ -f "$OPERATOR_REPORT_DIRECTORY/$name" ]; then
              cp "$OPERATOR_REPORT_DIRECTORY/$name" "$out/$name"
            fi
          done

      - name: Measure the 32-case midpoint response profile
        id: response
        continue-on-error: true
        working-directory: benchmark-instrument
        run: |
          set -euo pipefail
          name="litellm-response-$GITHUB_RUN_ID"
          docker run --rm --network host --name "$name" \
            -e DATABASE_URL="$DATABASE_URL" -e LITELLM_MASTER_KEY=sk-v2-profile-local \
            -v "$RUNNER_TEMP/litellm-source-reproduction/response-config.yaml:/app/config.yaml:ro" \
            "$SOURCE_IMAGE" --config /app/config.yaml --port "$LITELLM_PORT" \
            > "$RUNNER_TEMP/litellm-source-proxy.log" 2>&1 &
          docker_pid=$!
          trap 'docker rm -f "$name" > /dev/null 2>&1 || true; wait "$docker_pid" 2>/dev/null || true' EXIT
          for _ in $(seq 1 90); do
            if python -c 'import os, socket; socket.create_connection(("127.0.0.1", int(os.environ["LITELLM_PORT"])), 1).close()' 2>/dev/null; then break; fi
            sleep 2
          done
          if ! python -c 'import os, socket; socket.create_connection(("127.0.0.1", int(os.environ["LITELLM_PORT"])), 3).close()'; then
            echo 'LiteLLM did not open its local port before fixture generation.'
            docker inspect "$name" --format 'state={{.State.Status}} exit={{.State.ExitCode}} oom={{.State.OOMKilled}}' || true
            sed -n '1,80p' "$RUNNER_TEMP/litellm-source-proxy.log" || true
            exit 2
          fi
          python -m pii_leak_benchmark.v2_emitter \
            --validate --oracle midpoint --seed a1b2c3d4e5f60001 \
            --only litellm-presidio \
            --gateway-url "http://127.0.0.1:$LITELLM_PORT/v1/chat/completions" \
            --upstream-port 8799 --model capture \
            --out "$RUNNER_TEMP/litellm-source-reproduction" \
            > "$RUNNER_TEMP/litellm-source-instrument.log" 2>&1
          test -s "$RUNNER_TEMP/litellm-source-reproduction/litellm-presidio.json"
          mv "$RUNNER_TEMP/litellm-source-reproduction/litellm-presidio.json" \
             "$RUNNER_TEMP/litellm-source-reproduction/litellm-source.json"

      - name: Verify publishable evidence
        id: verify
        if: always()
        continue-on-error: true
        env:
          BENCHMARK_PYTHON: ${{ steps.operator.outputs.python-path }}
        run: |
          set -euo pipefail
          export RESPONSE_PYTHON="$(command -v python)"
          out="$RUNNER_TEMP/litellm-source-reproduction"
          "$BENCHMARK_PYTHON" -m pip freeze > "$out/operator-packages.txt"
          "$BENCHMARK_PYTHON" - <<'PY'
          import json
          import os
          import subprocess
          from base64 import b64encode, urlsafe_b64encode
          from html import escape
          from pathlib import Path
          from urllib.parse import quote, quote_plus
          from pii_leak_benchmark.operator_profile import seeded_fixture

          root = Path(os.environ['RUNNER_TEMP']) / 'litellm-source-reproduction'
          reports = ('current.json', 'summary.md', 'litellm-source.json', 'source-identity.json')
          provenance = ('source-commit.txt', 'image-id.txt', 'operator-config.yaml',
                        'response-config.yaml', 'response-harness-packages.txt',
                        'operator-packages.txt', 'python-version.txt', 'runner.txt', 'docker-version.txt')
          missing = [name for name in reports + provenance
                     if not (root / name).is_file() or not (root / name).stat().st_size]
          if missing:
              raise SystemExit('Incomplete evidence: missing or empty ' + ', '.join(missing))
          operator = json.loads((root / 'current.json').read_text(encoding='utf-8'))
          contract = operator.get('contract') or {}
          if (operator.get('schema') != 'pii-leak-benchmark/operator-run/v1'
                  or operator.get('verdict') not in ('CLEAN', 'LEAK')
                  or contract.get('profile') not in ('pii-v1', 'pii-secrets-v1')
                  or not isinstance(contract.get('seed'), str)):
              raise SystemExit('Incomplete evidence: operator profile was not measured.')
          fixture, marker = seeded_fixture(contract['seed'], contract['profile'] == 'pii-secrets-v1')
          response = subprocess.run(
              [os.environ['RESPONSE_PYTHON'], '-c',
               'import json; from pii_leak_benchmark.v2_emitter import build_segments; '
               's = build_segments("a1b2c3d4e5f60001"); '
               'print(json.dumps(list(s.echo.values()) + list(s.injection.values())))'],
              capture_output=True, check=True, text=True,
          )
          values = tuple(fixture.values()) + (marker,) + tuple(json.loads(response.stdout))
          needles = set(values)
          for value in values:
              encoded = value.encode('utf-8')
              needles.update((quote(value, safe=''), quote_plus(value), escape(value, quote=True),
                              b64encode(encoded).decode(), urlsafe_b64encode(encoded).decode(),
                              encoded.hex(), encoded.hex().upper()))

          def strings(node):
              if isinstance(node, str):
                  yield node
              elif isinstance(node, dict):
                  for key, item in node.items():
                      yield from strings(key)
                      yield from strings(item)
              elif isinstance(node, list):
                  for item in node:
                      yield from strings(item)

          for name in reports:
              content = (root / name).read_text(encoding='utf-8')
              fields = (content,) if name.endswith('.md') else strings(json.loads(content))
              if any(needle in field for field in fields for needle in needles):
                  raise SystemExit('Unsafe evidence: a generated value reached a report.')
          report = json.loads((root / 'litellm-source.json').read_text(encoding='utf-8'))
          if (report.get('schema') != 'llm-shield.streaming-privacy-http-profile/v2.0.0'
                  or (report.get('fixture') or {}).get('values_published') is not False
                  or (report.get('corpus') or {}).get('values_published') is not False
                  or (report.get('metrics') or {}).get('cases_scored') != 32
                  or ((report.get('capture') or {}).get('self_probe') or {}).get('recorded') is not True):
              raise SystemExit('Incomplete evidence: response profile contract failed.')
          (root / 'verification.txt').write_text(
              'Required reports complete; generated fixture values absent.\n', encoding='utf-8'
          )
          PY

      - name: Upload verified source evidence
        id: upload
        if: always() && steps.verify.outcome == 'success'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: source-reproduction
          path: |
            ${{ runner.temp }}/litellm-source-reproduction/current.json
            ${{ runner.temp }}/litellm-source-reproduction/summary.md
            ${{ runner.temp }}/litellm-source-reproduction/litellm-source.json
            ${{ runner.temp }}/litellm-source-reproduction/source-identity.json
            ${{ runner.temp }}/litellm-source-reproduction/source-commit.txt
            ${{ runner.temp }}/litellm-source-reproduction/image-id.txt
            ${{ runner.temp }}/litellm-source-reproduction/proxy-packages.txt
            ${{ runner.temp }}/litellm-source-reproduction/operator-config.yaml
            ${{ runner.temp }}/litellm-source-reproduction/response-config.yaml
            ${{ runner.temp }}/litellm-source-reproduction/response-harness-packages.txt
            ${{ runner.temp }}/litellm-source-reproduction/operator-packages.txt
            ${{ runner.temp }}/litellm-source-reproduction/python-version.txt
            ${{ runner.temp }}/litellm-source-reproduction/runner.txt
            ${{ runner.temp }}/litellm-source-reproduction/docker-version.txt
            ${{ runner.temp }}/litellm-source-reproduction/verification.txt
          if-no-files-found: error
          retention-days: 90

      - name: Stop local dependencies
        if: always()
        run: |
          for id in "$PG_CONTAINER" "$ANALYZER_CONTAINER" "$ANONYMIZER_CONTAINER"; do
            if [ -n "$id" ]; then docker rm -f "$id" > /dev/null 2>&1 || true; fi
          done

      - name: Report the result and how to submit it
        id: result
        if: always()
        continue-on-error: true
        env:
          REPORT_DIR: ${{ runner.temp }}/litellm-source-reproduction
          VERIFY_OUTCOME: ${{ steps.verify.outcome }}
          RESPONSE_OUTCOME: ${{ steps.response.outcome }}
          UPLOAD_OUTCOME: ${{ steps.upload.outcome }}
          REF_CHECK_OUTCOME: ${{ steps.ref_check.outcome }}
          CHECKOUT_OUTCOME: ${{ steps.checkout.outcome }}
          BUILD_OUTCOME: ${{ steps.build.outcome }}
          SOURCE_REF: ${{ inputs.source_ref }}
          STEP_OUTCOMES: request=${{ steps.operator.outcome }}, response=${{ steps.response.outcome }}, verification=${{ steps.verify.outcome }}, upload=${{ steps.upload.outcome }}
        run: |
          python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
          import json
          import os
          from pathlib import Path
          from urllib.parse import urlencode

          root = Path(os.environ['REPORT_DIR'])
          run_url = '{}/{}/actions/runs/{}'.format(
              os.environ['GITHUB_SERVER_URL'], os.environ['GITHUB_REPOSITORY'], os.environ['GITHUB_RUN_ID'])
          product = os.environ['WALL_PRODUCT']
          status = 'incomplete'
          # A verdict needs a complete run: verified and uploaded evidence, a passing response
          # step, and at least one applicable case in every condition of every non-control
          # response report. Anything less is incomplete, whatever it did observe, because the
          # summary calls a measured result complete and offers it to the wall.
          if (os.environ['VERIFY_OUTCOME'] == 'success' and (root / 'verification.txt').is_file()
                  and (root / 'current.json').is_file()):
              operator = json.loads((root / 'current.json').read_text(encoding='utf-8'))
              leaked = sorted(name for name, state in (operator.get('entities') or {}).items() if state == 'leak')
              table, response_leak, measured_reports, unmeasured = [], False, 0, False
              for path in sorted(root.glob('*.json')):
                  try:
                      metrics = json.loads(path.read_text(encoding='utf-8')).get('metrics') or {}
                  except (ValueError, AttributeError):
                      continue
                  rates = metrics.get('leak_rate') if isinstance(metrics, dict) else None
                  if not isinstance(rates, dict):
                      continue
                  counts = metrics.get('cases_by_condition') or {}
                  # The Shield recipe's redaction-off arm is a control: it is meant to leak.
                  control = path.stem.endswith('-off')
                  cells = []
                  for key in ('single_chunk', 'adversarial'):
                      rate, total = rates.get(key), counts.get(key)
                      known = isinstance(rate, (int, float)) and isinstance(total, int) and total > 0
                      cells.append(f'{round(rate * total)} of {total} leaked' if known else 'not measured')
                      response_leak = response_leak or (known and not control and rate > 0)
                      unmeasured = unmeasured or (not known and not control)
                  scored, inconclusive = metrics.get('cases_scored'), metrics.get('cases_inconclusive', 0)
                  applicable = scored - inconclusive if isinstance(scored, int) and isinstance(inconclusive, int) else 0
                  if not control:
                      measured_reports += 1
                      unmeasured = unmeasured or applicable <= 0
                  label = path.stem + (' (control, redaction off)' if control else '')
                  table.append(f"| {label} | {cells[0]} | {cells[1]} | {metrics.get('cases_inconclusive', 0)} |")
              complete = (os.environ['RESPONSE_OUTCOME'] == 'success' and os.environ['UPLOAD_OUTCOME'] == 'success'
                          and measured_reports > 0 and not unmeasured
                          and operator.get('verdict') in ('CLEAN', 'LEAK'))
              status = 'incomplete' if not complete else 'leak' if leaked or response_leak else 'clean'

          reason = ''
          if status == 'incomplete':
              # Name the first stage that failed and what to do about it, so an incomplete run
              # tells its operator more than "open the logs".
              # The ref is operator input: collapse whitespace so a newline in it cannot add
              # lines to GITHUB_OUTPUT (such as status=clean) or to the error annotation.
              source_ref = ' '.join(os.environ.get('SOURCE_REF', '').split())
              where = f'`{source_ref}`' if source_ref else 'the latest commit of the branch you ran'
              new_run = 'Start a new run with **Run workflow**; **Re-run jobs** reuses the same inputs.'
              rerun = ('This is usually the runner or a download, not the proxy: click **Re-run jobs**, '
                       'then **Re-run all jobs**. If it fails the same way twice, '
                       '[open an issue](https://github.com/ninadphalak/LLM-Shield-Proxy/issues) '
                       'with this run\'s link.')
              stages = (
                  ('REF_CHECK_OUTCOME', 'A source repository was named without a tag, branch or commit.',
                   'Type the tag, branch or commit to build in the first box. ' + new_run),
                  ('CHECKOUT_OUTCOME', f'The source to build, {where}, could not be fetched.',
                   'Check its spelling. A fork copies only its default branch, so a release tag may exist '
                   'only in the original repository. Type your fork\'s default branch, such as `main`, or a '
                   'commit your fork has. ' + new_run),
                  ('BUILD_OUTCOME', f'The source to build, {where}, did not build.',
                   'Open the failed step, marked with a red cross in the job\'s step list, for the error. '
                   'A download or network error usually passes on a rerun; a compile error means that '
                   'commit does not build, so pick another. ' + rerun),
                  ('STARTUP_OUTCOME', 'The proxy was built but did not start.',
                   'The step "Check proxy startup before fixtures" prints its log. If the log points at '
                   'the code, that commit does not start; pick another. ' + rerun),
                  ('RESPONSE_OUTCOME', 'The response-path measurement did not finish.', rerun),
                  ('VERIFY_OUTCOME', 'The evidence did not pass verification.', rerun),
                  ('UPLOAD_OUTCOME', 'The verified evidence could not be uploaded.', rerun),
              )
              failed = next((stage for stage in stages if os.environ.get(stage[0]) == 'failure'), None)
              reason, todo = (failed[1], failed[2]) if failed else (
                  'No step failed outright, but the evidence was not complete enough to count.', rerun)
              print('## Result: INCOMPLETE, do not submit')
              print()
              print('This run did not produce a verified measurement, so nothing was uploaded and the '
                    'results wall has nothing to read. It says nothing about the gateway.')
              print()
              print(f'**What went wrong:** {reason}')
              print()
              print(f'**What to do:** {todo}')
              print()
              print(f"Step outcomes: {os.environ['STEP_OUTCOMES']}.")
          else:
              print('## Result: MEASURED LEAK' if status == 'leak' else '## Result: MEASURED CLEAN')
              print()
              print('This run is a complete, verified measurement. '
                    + ('The job is red because the gateway leaked, not because the run broke. '
                       'A leak is a result worth publishing.' if status == 'leak'
                       else 'No leak in any case this run measured.'))
              print()
              print('Request path: ' + ('the provider received raw ' + ', '.join(leaked) + '.' if leaked
                                        else 'no raw value reached the provider.'))
              print()
              print('| Response report | Value in one chunk | Value split across two chunks | Inconclusive |')
              print('| :--- | :--- | :--- | ---: |')
              print('\n'.join(table))
              print()
              # One issue is one row. A run that measured several configurations at once sets
              # SUBMISSION_HOLD_REASON, and its summary says so in place of the submission link.
              hold = ' '.join(os.environ.get('SUBMISSION_HOLD_REASON', '').split())
              if hold:
                  print('### Not offered to the results wall')
                  print()
                  print(hold)
              else:
                  identity = json.loads((root / 'source-identity.json').read_text(encoding='utf-8'))
                  commit, selector = identity['source_commit'], identity['source_selector']
                  where = f'commit {commit[:12]}' if selector in ('', commit) else f'{selector} (commit {commit[:12]})'
                  sections = {
                      'Gateway': product,
                      'Version and configuration': f"{where}, {os.environ['WALL_CONFIGURATION']}",
                      'Project link': os.environ['WALL_PROJECT_URL'],
                      'CI run link': run_url,
                      'How it reads the stream': 'not stated',
                      'License': os.environ['WALL_LICENSE'],
                      'Outcome': status.upper(),
                  }
                  body = '\n\n'.join(f'### {heading}\n\n{value}' for heading, value in sections.items()) + '\n'
                  query = urlencode({'title': f'Result: {product}', 'labels': 'conformance-result', 'body': body})
                  print('### Put this run on the results wall')
                  print()
                  print(f'**[Submit this run to the results wall](https://github.com/ninadphalak/LLM-Shield-Proxy/issues/new?{query})**: '
                        'opens a prefilled issue. Press **Create**; nothing needs typing.')
                  print()
                  print('Or, from any terminal signed in to the GitHub CLI:')
                  print()
                  print('```')
                  print(f'gh issue create --repo ninadphalak/LLM-Shield-Proxy --title "Result: {product}" --body "{run_url}"')
                  print('```')
                  print()
                  print('Within a few minutes a bot answers on that issue. It either links your published row '
                        'or says exactly what stopped it. The numbers on the wall are read from this run\'s '
                        'artifact, not from the issue.')
          with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as output:
              output.write(f'status={status}\n')
              output.write(f'reason={reason}\n')
          PY

      - name: Fail unless the measurement is complete and clean
        if: always() && steps.result.outputs.status != 'clean'
        env:
          RESULT: ${{ steps.result.outputs.status }}
          REASON: ${{ steps.result.outputs.reason }}
        run: |
          if [ "${RESULT:-incomplete}" = incomplete ]; then
            echo "::error title=Incomplete, do not submit::${REASON:-The run did not finish.} The run's summary says what to do next."
          else
            echo "Result: $RESULT. This is a complete measurement: the red mark means a value leaked, not that the run broke. The job summary has the submission link."
          fi
          exit 1