name: E2E

# Runs the real tools on a small slice of the public GIAB HG002 genome, so a
# step that is broken in real use cannot pass CI behind a stub or a --version
# call. The jobs, see docs/testing.md (prs-memory: docs/25-prs.md#memory):
#
#   build-fixture  builds the data set and publishes it as release assets
#                  under the tag in tests/fixtures/VERSION. Runs on pull
#                  requests (a dispatch-only job could not run before this
#                  file is on main), on a push to main and on dispatch; it
#                  does nothing when the release already exists and was
#                  built by the same script.
#   e2e-plan, e2e-base, e2e-shards, e2e
#                  run every case in tests/e2e/ on that data set exactly once,
#                  split over parallel legs: e2e-plan splits the cases
#                  (scripts/ci/e2e-shards.sh), the leg "e2e base" runs the
#                  base cases and hands what they wrote to the legs
#                  "e2e shard N", which run the rest. The shard holding them
#                  then runs the parity checks: the bash single steps and the
#                  Nextflow pipeline, both from the fixture's FASTQ, must
#                  agree, and so must the SV consensus and the Y haplogroup
#                  of the hardening run (scripts/ci/parity-diff.sh). The job
#                  e2e is the required check: it merges every leg's results
#                  into one table and is red unless every case passed once.
#                  Runs on pull requests that touch the pipeline, on main
#                  after every merge that does, monthly, on dispatch. Every
#                  pull request starts this workflow, so main can require the
#                  e2e check: e2e-scope decides, and on a pull request that
#                  touches none of the pipeline paths the legs are skipped
#                  and e2e passes without running anything.
#   settle-doubts  answers questions a code review could not answer without
#                  running the tools. Not a gate: it prints a table.
#   prs-memory     on dispatch only: the memory of PRS_PREPARE and pgsc_calc's
#                  FORMAT_SCOREFILES on the default score list
#                  (tests/test_prs_memory.sh --measure), under the requests of
#                  conf/base.config, and killed under what the labels gave
#                  before them.
#
# Public repo: GitHub-hosted runners are free here, and the e2e job runs on
# pull requests, so nothing in this file may use a self-hosted runner.
on:
  push:
    # Main only. Main does not require a branch to be up to date, so two
    # pull requests merged close together are first tested together here.
    # A push to another branch is tested by its pull request: a push run
    # there would put a skipped e2e job, which counts as passed, on the same
    # commit as the pull request's required e2e check.
    branches: [main]
    # The e2e scope, the same paths e2e-scope checks on a pull request
    # (E2E_SCOPE_DIRS and E2E_SCOPE_FILES below; the e2e job fails when the
    # two lists differ), so a docs-only merge starts no run.
    paths:
      - 'scripts/**'
      - 'modules/**'
      - 'workflows/**'
      - 'bin/**'
      - 'conf/**'
      - 'assets/**'
      - 'tests/e2e/**'
      - 'tests/fixtures/VERSION'
      - 'main.nf'
      - 'nextflow.config'
      - 'versions.env'
      - '.github/workflows/e2e.yml'
  pull_request:
    branches: [main]
  schedule:
    # Monthly: catches upstream images, databases or URLs that changed under us.
    - cron: '0 5 2 * *'
  workflow_dispatch:
    inputs:
      job:
        description: 'Which job to run'
        type: choice
        options: [e2e, build-fixture, settle-doubts, prs-memory]
        default: e2e
      rebuild:
        description: 'build-fixture: replace the assets of an existing release'
        type: boolean
        default: false
      save_images:
        description: 'e2e: save the image and data caches from this branch too (only main saves by default)'
        type: boolean
        default: false
      trace:
        description: 'e2e: one serial shard that measures each case and what it reads, and prints a new scripts/ci/e2e-case-times.tsv'
        type: boolean
        default: false

permissions:
  contents: read

# The paths the e2e cases exercise: a directory covers everything under it.
env:
  E2E_SCOPE_DIRS: scripts modules workflows bin conf assets tests/e2e
  E2E_SCOPE_FILES: tests/fixtures/VERSION main.nf nextflow.config versions.env .github/workflows/e2e.yml
  # The shard legs besides the base leg. The cases took 230 minutes in one
  # job on 2026-11-02 (run 41390434697).
  E2E_SHARDS: 6

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
  # Never cancel a run on main half way; pull request runs may be
  # replaced (build-fixture publishes MANIFEST.txt last, so a cancelled
  # upload is rebuilt by the next run). A push run waits for the running
  # one, and GitHub keeps one waiting run per group: a newer merge replaces
  # the waiting one, so the next run on main tests every merge since the
  # last one.
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  build-fixture:
    if: >-
      github.event_name == 'push' || github.event_name == 'pull_request' ||
      (github.event_name == 'workflow_dispatch' && inputs.job == 'build-fixture')
    runs-on: ubuntu-latest
    # VEP --database answers at Ensembl's pace: 20 to 60 minutes so far.
    timeout-minutes: 150
    permissions:
      contents: write
    steps:
      # On a pull request, the head commit rather than the merge commit: the
      # release notes name the commit whose script built the data.
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          persist-credentials: false

      # A release that exists and was built by this exact script is left alone.
      # One built by a different script means the fixture changed without a new
      # version, which would change the data under every open pull request.
      - name: Decide whether to build
        id: need
        env:
          GH_TOKEN: ${{ github.token }}
          REBUILD: ${{ inputs.rebuild }}
        run: |
          tag=$(tr -d '[:space:]' < tests/fixtures/VERSION)
          want=$(sha256sum scripts/ci/build-fixture.sh | awk '{print $1}')
          echo "tag=${tag}" >> "$GITHUB_OUTPUT"
          if ! gh release view "$tag" -R "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
            echo "Release ${tag} does not exist yet: building it."
            echo "build=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          have=""
          if gh release download "$tag" -R "$GITHUB_REPOSITORY" -p MANIFEST.txt -D "${RUNNER_TEMP}/manifest" 2>/dev/null; then
            have=$(awk '/^build_script_sha256:/ {print $2}' "${RUNNER_TEMP}/manifest/MANIFEST.txt")
          fi
          if [ "$REBUILD" = "true" ] || [ -z "$have" ]; then
            echo "Rebuilding ${tag} (requested, or its MANIFEST.txt is missing)."
            echo "build=true" >> "$GITHUB_OUTPUT"
          elif [ "$have" = "$want" ]; then
            echo "Release ${tag} was built by this build script already: nothing to do."
            echo "build=false" >> "$GITHUB_OUTPUT"
          else
            echo "::error::scripts/ci/build-fixture.sh changed but tests/fixtures/VERSION is still ${tag}. Bump VERSION so the new data gets a new release."
            exit 1
          fi

      # A pull request from a fork gets a read-only token and could never
      # publish the release: say so now rather than after a two-hour build.
      - name: Refuse a fixture build from a fork
        if: >-
          steps.need.outputs.build == 'true' &&
          github.event_name == 'pull_request' &&
          github.event.pull_request.head.repo.full_name != github.repository
        run: |
          echo "::error::A pull request from a fork cannot publish the fixture release. A maintainer builds it with workflow_dispatch, job build-fixture."
          exit 1

      - name: Install bgzip and tabix
        if: steps.need.outputs.build == 'true'
        run: sudo apt-get update -q && sudo apt-get install -y -q tabix

      - name: Build the fixture
        if: steps.need.outputs.build == 'true'
        run: scripts/ci/build-fixture.sh "${RUNNER_TEMP}/fixture"

      - name: Publish the release assets
        if: steps.need.outputs.build == 'true'
        env:
          GH_TOKEN: ${{ github.token }}
          TAG: ${{ steps.need.outputs.tag }}
          # On a pull request GITHUB_SHA is the temporary merge commit, which
          # is on no branch; the notes name the pull request's head instead.
          BUILD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
        run: |
          cd "${RUNNER_TEMP}/fixture"
          # MANIFEST.txt goes up last: a release without it is rebuilt by the
          # next run, so an upload cut half way (a newer push cancels a pull
          # request run) is never taken for a finished one.
          files=()
          for f in *; do [ "$f" = MANIFEST.txt ] || files+=("$f"); done
          if gh release view "$TAG" -R "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
            # A rebuild in place: drop the old MANIFEST.txt first, or a cut
            # upload would leave it beside half-replaced data.
            # A failed listing stops the step; it must not read as "no manifest".
            assets=$(gh release view "$TAG" -R "$GITHUB_REPOSITORY" --json assets --jq '.assets[].name')
            if grep -qx MANIFEST.txt <<< "$assets"; then
              gh release delete-asset "$TAG" MANIFEST.txt -R "$GITHUB_REPOSITORY" -y
            fi
            gh release upload "$TAG" -R "$GITHUB_REPOSITORY" --clobber "${files[@]}"
          else
            # A prerelease that is never "latest": the project's real releases
            # stay the ones people see. The tag does not match release.yml's v*.
            # No --target: the job token may not create a tag on a commit that
            # changes a workflow file, so the tag goes on the default branch;
            # the notes name the commit that built the data.
            gh release create "$TAG" "${files[@]}" -R "$GITHUB_REPOSITORY" \
              --title "Test fixture ${TAG}" \
              --notes "Test data for the E2E workflow, not a pipeline release. A slice of the public GIAB HG002 genome, built by scripts/ci/build-fixture.sh at ${BUILD_SHA}; see MANIFEST.txt and docs/testing.md." \
              --prerelease --latest=false
          fi
          gh release upload "$TAG" -R "$GITHUB_REPOSITORY" --clobber MANIFEST.txt
          {
            echo "### Fixture ${TAG}"
            echo '```'
            cat MANIFEST.txt
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"

  # The pull request paths the e2e cases exercise. Outside a pull request the
  # answer is always run=true: on a push the paths filter above already
  # asked the same question. A failed or missing answer runs e2e anyway (the
  # test below is != 'false'), so a broken scope job can never skip the gate.
  e2e-scope:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      run: ${{ steps.scope.outputs.run }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        if: github.event_name == 'pull_request'
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Does this pull request touch the pipeline
        id: scope
        env:
          EVENT: ${{ github.event_name }}
          BASE: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        # Three dots: what the branch changed since it left main, as a paths
        # filter sees it, whatever main did meanwhile.
        run: |
          if [ "$EVENT" != "pull_request" ]; then echo "run=true" >> "$GITHUB_OUTPUT"; exit 0; fi
          git diff --name-only "${BASE}...${HEAD_SHA}" > "${RUNNER_TEMP}/changed.txt"
          cat "${RUNNER_TEMP}/changed.txt"
          dirs=$(tr ' ' '|' <<< "$E2E_SCOPE_DIRS")
          files=$(tr ' ' '|' <<< "$E2E_SCOPE_FILES" | sed 's/\./\\./g')
          if grep -qE "^(${dirs})/|^(${files})\$" "${RUNNER_TEMP}/changed.txt"; then
            echo "run=true" >> "$GITHUB_OUTPUT"
          else
            echo "Nothing the e2e cases exercise changed: e2e is skipped."
            echo "run=false" >> "$GITHUB_OUTPUT"
          fi

  # Splits the cases over the parallel jobs below (scripts/ci/e2e-shards.sh,
  # with the measured times and needs of scripts/ci/e2e-case-times.tsv), and
  # runs the checks that need no fixture. Runs when e2e runs; a failed or
  # missing scope answer runs it too.
  e2e-plan:
    needs: e2e-scope
    if: >-
      ${{ !cancelled() && needs.e2e-scope.outputs.run != 'false' && (
        github.event_name == 'pull_request' ||
        github.event_name == 'push' ||
        github.event_name == 'schedule' ||
        (github.event_name == 'workflow_dispatch' && inputs.job == 'e2e')) }}
    runs-on: ubuntu-latest
    timeout-minutes: 10
    outputs:
      jobs: ${{ steps.plan.outputs.jobs }}
      shards: ${{ steps.plan.outputs.shards }}
      trace: ${{ steps.plan.outputs.trace }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      # The push paths filter is the scope a second time, because a filter
      # cannot read env. A path in one list and not the other would start e2e
      # on a pull request and never on main after the merge, or the reverse.
      - name: The push paths are the e2e scope
        run: |
          read -ra dirs <<< "$E2E_SCOPE_DIRS"
          read -ra files <<< "$E2E_SCOPE_FILES"
          want=$( { printf '%s/**\n' "${dirs[@]}"; printf '%s\n' "${files[@]}"; } | sort)
          have=$(awk '/^  push:/ {p = 1; next} p && /^  [a-z_]+:/ {exit}
                      p && /^      - / {sub(/^      - /, ""); gsub(/\047/, ""); print}' .github/workflows/e2e.yml | sort)
          if [ "$want" != "$have" ]; then
            echo "::error::The push paths of e2e.yml differ from E2E_SCOPE_DIRS and E2E_SCOPE_FILES (< scope, > push paths):"
            diff <(echo "$want") <(echo "$have") || true
            exit 1
          fi
          echo "Push paths match the e2e scope: $(wc -l <<< "$have") entries."

      # Seconds, no network: a planted registry 500 must be tried again, and a
      # missing tag or a rate limit must not.
      - name: Pull retry self-test
        run: scripts/ci/e2e-run.sh --self-test-pull

      # The base job plus E2E_SHARDS shard jobs; a traced run puts every other
      # case in one shard, in name order, as one serial run would. The two
      # parity checks below read the outputs of two cases each, so each pair
      # must share a shard.
      - name: Split the cases
        id: plan
        env:
          TRACE: ${{ inputs.trace && 'true' || 'false' }}
        run: |
          set -o pipefail
          n=$E2E_SHARDS
          [ "$TRACE" = true ] && n=1
          scripts/ci/e2e-shards.sh plan "$n" > "${RUNNER_TEMP}/plan.tsv"
          { scripts/ci/e2e-shards.sh base; cut -f2 "${RUNNER_TEMP}/plan.tsv"; } > "${RUNNER_TEMP}/all.txt"
          scripts/ci/e2e-shards.sh cover "${RUNNER_TEMP}/all.txt"
          for pair in "nextflow-from-fastq-1-bash nextflow-from-fastq-2-nextflow" \
                      "nextflow-hardening-1-manta nextflow-hardening-2-nf"; do
            read -r a b <<< "$pair"
            sa=$(awk -F'\t' -v c="$a" '$2 == c {print $1}' "${RUNNER_TEMP}/plan.tsv")
            sb=$(awk -F'\t' -v c="$b" '$2 == c {print $1}' "${RUNNER_TEMP}/plan.tsv")
            if [ -z "$sa" ] || [ "$sa" != "$sb" ]; then
              echo "::error::${a} (shard ${sa:-none}) and ${b} (shard ${sb:-none}) must share a shard: the parity check reads both. Give ${b} the need ${a} in scripts/ci/e2e-case-times.tsv."
              exit 1
            fi
          done
          jobs=$(seq 1 "$n" | jq -R . | jq -sc .)
          { echo "jobs=${jobs}"; echo "shards=${n}"; echo "trace=${TRACE}"; } >> "$GITHUB_OUTPUT"
          {
            echo "### E2E split: base and ${n} shards"
            echo
            echo "| Job | Cases | Measured minutes |"
            echo "|---|---|---|"
            awk -F'\t' 'FILENAME == ARGV[1] { if ($0 !~ /^#/) t[$1] = $3; next }
                        FILENAME == ARGV[2] { n["base"]++; s["base"] += ($1 in t) ? t[$1] : 300; next }
                        { n[$1]++; s[$1] += ($2 in t) ? t[$2] : 300 }
                        END { printf "| base | %d | %.0f |\n", n["base"], s["base"] / 60
                              for (k = 1; k in n; k++) printf "| shard %d | %d | %.0f |\n", k, n[k], s[k] / 60 }' \
              scripts/ci/e2e-case-times.tsv <(scripts/ci/e2e-shards.sh base) "${RUNNER_TEMP}/plan.tsv"
          } | tee -a "$GITHUB_STEP_SUMMARY"

  # The legs. The base leg runs the base cases and uploads what they wrote as
  # the artifact e2e-base; the shard legs start once it passed, unpack that
  # artifact at the same paths and run their own cases. A shard does not take
  # a runner before the base leg is done: in a busy queue, shards that start
  # first and wait would hold runners the base legs of every run need. Both
  # jobs run the same steps (the YAML anchor leg-steps); matrix.job tells
  # them apart. Only the base leg saves the image cache. The checks are named
  # "e2e base" and "e2e shard N"; the required check is the job e2e below,
  # which sums them up.
  e2e-base:
    needs: [e2e-scope, e2e-plan]
    if: ${{ !cancelled() && needs.e2e-plan.result == 'success' }}
    name: ${{ matrix.job == 'base' && 'e2e base' || format('e2e shard {0}', matrix.job) }}
    runs-on: ubuntu-latest
    # The base leg waits up to 75 minutes for a fixture a pull request bumps
    # (scripts/ci/e2e-run.sh).
    timeout-minutes: 240
    permissions:
      contents: read
      actions: read
    strategy:
      matrix:
        job: [base]
    env:
      JOB: ${{ matrix.job }}
    steps: &leg-steps
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      # The same cases take about a third longer on some runners than on
      # others; the CPU line lets a timing claim name the hardware.
      - name: Runner CPU
        run: |
          cpu="$(lscpu | awk -F': *' '/^Model name/ && !n++ {print $2}'), $(nproc) CPUs"
          echo "$cpu"
          echo "Runner: ${cpu}" >> "$GITHUB_STEP_SUMMARY"

      # Before anything is pulled or restored: the legs ran out of disk on
      # 2026-11-04 with only part of the preinstalled software removed.
      - uses: ./.github/actions/free-disk

      # Images go to /var/lib/docker on /; the work area goes to whichever of
      # / and /mnt has more room, so the two do not compete for one disk. The
      # base outputs unpack only under the work area they were packed in, so
      # a shard leg takes the base leg's choice when its artifact arrives.
      - name: Pick the work area
        run: |
          work="${RUNNER_TEMP}/e2e-work"
          if [ "$(df --output=avail -k /mnt | tail -1)" -gt "$(df --output=avail -k / | tail -1)" ]; then
            work=/mnt/e2e-work
            sudo mkdir -p "$work" && sudo chown "$(id -u):$(id -g)" "$work"
          fi
          echo "E2E_WORK=${work}" >> "$GITHUB_ENV"
          # One fixed path: actions/cache counts the path in an entry's
          # version, so a path that followed the work area would miss on a
          # runner that picked the other disk. e2e-run.sh hard-links the
          # cached downloads into the work area, or copies them when the
          # work area is on /mnt.
          mkdir -p "${RUNNER_TEMP}/e2e-data"
          echo "E2E_DATA_CACHE=${RUNNER_TEMP}/e2e-data" >> "$GITHUB_ENV"

      - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6.1.0
        with:
          distribution: 'temurin'
          java-version: '17'

      - uses: nf-core/setup-nextflow@893c28b667aedeba26e37f296d260ccc5bc4d914 # v3.0.1
        with:
          version: "26.10.5"

      # Docker 29 on the ubuntu-26.04 image keeps images in containerd's
      # store, which holds each layer twice: the compressed blob in its
      # content store and the unpacked snapshot. DeepVariant's, GATK's and
      # VNtyper's images took 24.8 GB of disk there and 17.6 GB in the
      # classic overlay2 store, on 26.04 and on 24.04 alike (probe run
      # 41957796291 on the geiresx fork). With every case image on it, a
      # 72 GB runner (48 GB free) ran out in the pre-pull (run 41938436657,
      # shards 2 and 5). Runs before any image is restored or pulled; the
      # images already in the containerd store are pruned first, as overlay2
      # would not see them.
      - name: Docker image store
        run: |
          if docker info -f '{{json .DriverStatus}}' | grep -q io.containerd.snapshotter; then
            docker image prune -a -f > /dev/null
            f=/etc/docker/daemon.json
            { [ -s "$f" ] && sudo cat "$f" || echo '{}'; } \
              | jq '.features["containerd-snapshotter"] = false' | sudo tee "${f}.new" > /dev/null
            sudo mv "${f}.new" "$f"
            sudo systemctl restart docker
          fi
          driver=$(docker info -f '{{.Driver}}')
          echo "Docker $(docker version -f '{{.Server.Version}}'), storage driver ${driver}" | tee -a "$GITHUB_STEP_SUMMARY"
          [ "$driver" = overlay2 ] || { echo "::error::Docker's storage driver is ${driver}, not overlay2."; exit 1; }
          df -h /

      # Images are cached in four entries, one per group of
      # `scripts/ci/e2e-run.sh --image-groups`, each keyed on its own image
      # list: a version bump misses one group's key, and that group restores
      # the newest older entry of the group (restore-keys) and pulls only
      # what changed. Only runs on main save (see the save step), and caches
      # saved on main are the ones every pull request can read.
      - name: Image cache keys
        id: keys
        run: |
          grep -qx 'IMAGE_GROUPS=4' scripts/ci/e2e-run.sh \
            || { echo "::error::IMAGE_GROUPS in scripts/ci/e2e-run.sh is not 4: change the cache steps of e2e.yml with it."; exit 1; }
          scripts/ci/e2e-run.sh --image-groups > "${RUNNER_TEMP}/image-groups.txt"
          for g in 0 1 2 3; do
            # Only refs that can be saved: a digest bump never changes an entry.
            h=$(awk -v g="$g" '$1 == g && $2 !~ /@/ {print $2}' "${RUNNER_TEMP}/image-groups.txt" | sort | sha256sum | cut -c1-16)
            # "ovl": saved from the overlay2 store (step "Docker image store").
            # 7 of the 46 archives saved from containerd's store did not load
            # into overlay2 (run 41960912641), and an exact key hit never
            # saves again, so the old entries are left behind.
            echo "g${g}=e2e-images-ovl-g${g}-${h}" | tee -a "$GITHUB_OUTPUT"
          done

      - name: Restore cached images, group 0
        id: images0
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ runner.temp }}/images/g0
          key: ${{ steps.keys.outputs.g0 }}
          restore-keys: e2e-images-ovl-g0-

      - name: Restore cached images, group 1
        id: images1
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ runner.temp }}/images/g1
          key: ${{ steps.keys.outputs.g1 }}
          restore-keys: e2e-images-ovl-g1-

      - name: Restore cached images, group 2
        id: images2
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ runner.temp }}/images/g2
          key: ${{ steps.keys.outputs.g2 }}
          restore-keys: e2e-images-ovl-g2-

      - name: Restore cached images, group 3
        id: images3
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ runner.temp }}/images/g3
          key: ${{ steps.keys.outputs.g3 }}
          restore-keys: e2e-images-ovl-g3-

      # One image per file; the four groups load side by side. An image that
      # does not load is pulled by e2e-run.sh like any missing image.
      - name: Load cached images
        env:
          HITS: ${{ steps.images0.outputs.cache-hit || 'false' }} ${{ steps.images1.outputs.cache-hit || 'false' }} ${{ steps.images2.outputs.cache-hit || 'false' }} ${{ steps.images3.outputs.cache-hit || 'false' }}
        run: |
          # The archives that loaded, so the summary calls only them restored.
          loaded="${RUNNER_TEMP}/images-loaded.txt"
          : > "$loaded"
          for d in "${RUNNER_TEMP}"/images/g*; do
            [ -d "$d" ] || continue
            (
              for f in "$d"/*.tar.zst; do
                [ -f "$f" ] || continue
                if zstd -dc "$f" | docker load -q > /dev/null; then echo "${f##*/}" >> "$loaded"
                else echo "::warning::${f##*/} did not load; e2e-run.sh pulls it"; fi
                # Loaded, the archive only takes disk.
                rm -f "$f"
              done
            ) &
          done
          wait
          rm -rf "${RUNNER_TEMP}/images"
          echo "Image cache: $(wc -l < "$loaded") images loaded in ${SECONDS} s; exact key hit per group 0-3: ${HITS}" | tee -a "$GITHUB_STEP_SUMMARY"

      # This leg's cases, from the same split as the plan job (same commit,
      # same table), and whether one of the parity checks reads them.
      - name: This leg's cases
        id: cases-list
        env:
          SHARDS: ${{ needs.e2e-plan.outputs.shards }}
        run: |
          set -o pipefail
          list="${RUNNER_TEMP}/cases.txt"
          if [ "$JOB" = base ]; then scripts/ci/e2e-shards.sh base > "$list"
          else scripts/ci/e2e-shards.sh plan "$SHARDS" | awk -F'\t' -v k="$JOB" '$1 == k {print $2}' > "$list"; fi
          [ -s "$list" ] || { echo "::error::The split gave job ${JOB} no case."; exit 1; }
          echo "$(wc -l < "$list") cases: $(paste -sd' ' "$list")"
          grep -qx nextflow-from-fastq-2-nextflow "$list" && echo "parity_fastq=true" >> "$GITHUB_OUTPUT"
          grep -qx nextflow-hardening-2-nf "$list" && echo "parity_hardening=true" >> "$GITHUB_OUTPUT"
          grep -qx mocha-1 "$list" && echo "data_mocha=true" >> "$GITHUB_OUTPUT"
          grep -qx archaic-ancestry-1 "$list" && echo "data_archaic=true" >> "$GITHUB_OUTPUT"
          grep -qx bash-step-08-hla "$list" && echo "data_hla=true" >> "$GITHUB_OUTPUT"
          grep -qx prs-scores-and-run-options-1-strand-sex "$list" && echo "data_pgs=true" >> "$GITHUB_OUTPUT"
          echo "list=${list}" >> "$GITHUB_OUTPUT"

      # Before the base outputs: the base leg pulled the same images.
      - name: Pull the images the cases use
        if: matrix.job != 'base'
        run: scripts/ci/e2e-run.sh --pull-images

      # The base leg uploads e2e-base only once its cases all passed, and this
      # leg starts only after it passed. A rerun of failed legs finds the
      # artifact of the earlier attempt. A broken transfer is tried again.
      - name: Fetch the base leg's outputs
        if: matrix.job != 'base'
        env:
          GH_TOKEN: ${{ github.token }}
          RUN_ID: ${{ github.run_id }}
        run: |
          id=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/artifacts?name=e2e-base" --jq '.artifacts[0].id // empty')
          [ -n "$id" ] || { echo "::error::The base leg passed but this run has no e2e-base artifact."; exit 1; }
          for try in 1 2 3 4 5; do
            t0=$(date +%s)
            # curl, not gh api: gh took 5 minutes for the 3 GB of run 41455829120.
            if curl -sSfL --retry 3 -H "Authorization: Bearer ${GH_TOKEN}" -o "${RUNNER_TEMP}/e2e-base.zip" \
                 "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/artifacts/${id}/zip" \
               && unzip -o -q "${RUNNER_TEMP}/e2e-base.zip" -d "${RUNNER_TEMP}/e2e-base"; then
              rm -f "${RUNNER_TEMP}/e2e-base.zip"
              read -r work repo < "${RUNNER_TEMP}/e2e-base/base.tar.zst.from"
              if [ "$repo" != "$GITHUB_WORKSPACE" ]; then
                echo "::error::The base leg ran in the checkout ${repo}, this leg in ${GITHUB_WORKSPACE}."
                exit 1
              fi
              sudo mkdir -p "$work" && sudo chown "$(id -u):$(id -g)" "$work"
              echo "E2E_WORK=${work}" >> "$GITHUB_ENV"
              echo "Base outputs in $(( $(date +%s) - t0 )) s to fetch and unzip: $(du -h "${RUNNER_TEMP}/e2e-base/base.tar.zst" | cut -f1), work area ${work}"
              exit 0
            fi
            echo "::warning::Fetching or unpacking e2e-base failed (try ${try} of 5)."
            rm -rf "${RUNNER_TEMP}/e2e-base.zip" "${RUNNER_TEMP}/e2e-base"
            sleep 30
          done
          echo "::error::Could not fetch e2e-base in 5 tries."
          exit 1

      # The two largest downloads of the cases, from EBI's 1000 Genomes
      # server, which is slow and stalls at times: MoChA's chr20 panel and
      # its manifest (mocha-1, 0.64 GB), keyed on the manifest's pin and the
      # chromosome (the whole raw chr20 panel, which no fixture changes), and the 1000
      # Genomes sites VCF (archaic-ancestry-1, 1.36 GB), keyed on its md5.
      # e2e-run.sh hands them to the two cases ("Downloads kept between
      # runs" there) and downloads what a miss leaves out. Saved like the
      # images: on main, or a run by hand with save_images, on a miss. Only
      # the leg whose cases include the case restores and saves its entry.
      # Two small ones the same way, keyed by e2e-run.sh --data-key: hla.dat
      # of two IPD-IMGT/HLA releases and GENCODE's genes and exons
      # (bash-step-08-hla, which then runs with the network blocked), and the
      # four real PGS Catalog score files of prs-scores-and-run-options-1.
      - name: Data cache keys
        id: data-keys
        run: |
          . versions.env
          echo "mocha=e2e-data-mocha-${MOCHA_PANEL_MANIFEST_SHA256:0:16}-chr20" | tee -a "$GITHUB_OUTPUT"
          echo "archaic=e2e-data-1kg-sites-${ARCHAIC_1KG_SITES_MD5}" | tee -a "$GITHUB_OUTPUT"
          echo "hla=$(scripts/ci/e2e-run.sh --data-key hla)" | tee -a "$GITHUB_OUTPUT"
          echo "pgs=$(scripts/ci/e2e-run.sh --data-key pgs)" | tee -a "$GITHUB_OUTPUT"

      - name: Restore cached data, MoChA panel
        id: data-mocha
        if: steps.cases-list.outputs.data_mocha == 'true'
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ env.E2E_DATA_CACHE }}/mocha
          key: ${{ steps.data-keys.outputs.mocha }}

      - name: Restore cached data, 1000 Genomes sites
        id: data-archaic
        if: steps.cases-list.outputs.data_archaic == 'true'
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ env.E2E_DATA_CACHE }}/archaic
          key: ${{ steps.data-keys.outputs.archaic }}

      - name: Restore cached data, IPD-IMGT/HLA and GENCODE
        id: data-hla
        if: steps.cases-list.outputs.data_hla == 'true'
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ env.E2E_DATA_CACHE }}/hla
          key: ${{ steps.data-keys.outputs.hla }}

      - name: Restore cached data, PGS Catalog scores
        id: data-pgs
        if: steps.cases-list.outputs.data_pgs == 'true'
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ env.E2E_DATA_CACHE }}/pgs
          key: ${{ steps.data-keys.outputs.pgs }}

      # The PGS Catalog's synthetic HAPNEST panel (prs-2-ancestry and
      # prs-scores-and-run-options-2, 268 MB), from EBI's FTP server, keyed
      # on the sha256 tests/e2e/lib.sh pins. The cases take it from
      # E2E_DATA_CACHE/hapnest, check it and then run setup.sh with EBI
      # unreachable; on a miss the first one downloads it and puts it there
      # (setup_ancestry_panel in lib.sh). Saved like the entries above: on
      # main, or a run by hand with save_images, on a miss.
      - name: Data cache, HAPNEST panel
        id: hapnest
        env:
          LIST: ${{ steps.cases-list.outputs.list }}
          SAVE: ${{ (github.ref == 'refs/heads/main' || inputs.save_images) && 'true' || 'false' }}
        run: |
          grep -qxE 'prs-2-ancestry|prs-scores-and-run-options-2-panel-chrx' "$LIST" || exit 0
          sha=$(sed -n 's/^HAPNEST_SHA256=//p' tests/e2e/lib.sh)
          [ "${#sha}" -eq 64 ] || { echo "::error::No HAPNEST_SHA256= line in tests/e2e/lib.sh"; exit 1; }
          { echo "run=true"; echo "key=e2e-data-hapnest-${sha:0:16}"; } | tee -a "$GITHUB_OUTPUT"
          echo "E2E_HAPNEST_KEEP=${SAVE}" >> "$GITHUB_ENV"

      - name: Restore cached data, HAPNEST panel
        id: data-hapnest
        if: steps.hapnest.outputs.run == 'true'
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ env.E2E_DATA_CACHE }}/hapnest
          key: ${{ steps.hapnest.outputs.key }}

      - name: Run the e2e cases
        id: cases
        env:
          GH_TOKEN: ${{ github.token }}
          E2E_CASES: ${{ steps.cases-list.outputs.list }}
          E2E_JOB: ${{ matrix.job == 'base' && 'base' || format('shard {0}', matrix.job) }}
          E2E_TRACE: ${{ needs.e2e-plan.outputs.trace }}
          # The free disk a leg needs before its first case (e2e-run.sh).
          E2E_MIN_FREE_GB: 10
          E2E_BASE_OUT: ${{ matrix.job == 'base' && format('{0}/base.tar.zst', runner.temp) || '' }}
          E2E_BASE_IN: ${{ matrix.job != 'base' && format('{0}/e2e-base/base.tar.zst', runner.temp) || '' }}
          # Keep the downloads to the end of the job only when a save step uses them.
          E2E_DATA_KEEP: ${{ (github.ref == 'refs/heads/main' || inputs.save_images) && ((steps.cases-list.outputs.data_mocha == 'true' && steps.data-mocha.outputs.cache-hit != 'true') || (steps.cases-list.outputs.data_archaic == 'true' && steps.data-archaic.outputs.cache-hit != 'true') || (steps.cases-list.outputs.data_hla == 'true' && steps.data-hla.outputs.cache-hit != 'true') || (steps.cases-list.outputs.data_pgs == 'true' && steps.data-pgs.outputs.cache-hit != 'true')) && 'true' || 'false' }}
          HIT_MOCHA: ${{ steps.data-mocha.outputs.cache-hit || 'false' }}
          HIT_ARCHAIC: ${{ steps.data-archaic.outputs.cache-hit || 'false' }}
          HIT_HLA: ${{ steps.data-hla.outputs.cache-hit || 'false' }}
          HIT_PGS: ${{ steps.data-pgs.outputs.cache-hit || 'false' }}
          DATA_ANY: ${{ steps.cases-list.outputs.data_mocha || steps.cases-list.outputs.data_archaic || steps.cases-list.outputs.data_hla || steps.cases-list.outputs.data_pgs || 'false' }}
        run: |
          if [ "$DATA_ANY" = true ]; then
            echo "Data cache (${JOB}): exact key hit, MoChA panel ${HIT_MOCHA}, 1000 Genomes sites ${HIT_ARCHAIC}, IPD-IMGT/HLA and GENCODE ${HIT_HLA}, PGS scores ${HIT_PGS}" | tee -a "$GITHUB_STEP_SUMMARY"
          fi
          scripts/ci/e2e-run.sh

      # A miss saves what e2e-run.sh downloaded, checked against its pin
      # (data_fetch there), when every file of the entry is in place.
      - name: Data cache, what to save
        id: data-save
        if: >-
          always() && steps.cases.outcome != 'skipped' &&
          (github.ref == 'refs/heads/main' || inputs.save_images)
        env:
          HIT_MOCHA: ${{ steps.data-mocha.outputs.cache-hit }}
          HIT_ARCHAIC: ${{ steps.data-archaic.outputs.cache-hit }}
          HIT_HLA: ${{ steps.data-hla.outputs.cache-hit }}
          HIT_PGS: ${{ steps.data-pgs.outputs.cache-hit }}
        run: |
          d=$E2E_DATA_CACHE
          if [ "$HIT_MOCHA" != true ] && [ -s "${d}/mocha/manifest.txt" ] \
             && [ "$(find "${d}/mocha/raw" -type f ! -name '*.part' 2> /dev/null | wc -l)" -eq 2 ]; then
            echo "mocha=true" >> "$GITHUB_OUTPUT"
          fi
          if [ "$HIT_ARCHAIC" != true ] && [ -s "${d}/archaic/1kg_sites.vcf.gz" ]; then
            echo "archaic=true" >> "$GITHUB_OUTPUT"
          fi
          # Whole and checked (data_ok in e2e-run.sh), or not saved.
          for n in hla pgs; do
            hit=HIT_${n^^}
            if [ "${!hit}" != true ] && [ -d "${d}/${n}" ] && scripts/ci/e2e-run.sh --data-check "$n"; then
              echo "${n}=true" >> "$GITHUB_OUTPUT"
            fi
          done
          du -sh "${d}"/* 2> /dev/null || echo "Data cache: nothing downloaded"

      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        if: always() && steps.data-save.outputs.mocha == 'true'
        with:
          path: ${{ env.E2E_DATA_CACHE }}/mocha
          key: ${{ steps.data-keys.outputs.mocha }}

      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        if: always() && steps.data-save.outputs.archaic == 'true'
        with:
          path: ${{ env.E2E_DATA_CACHE }}/archaic
          key: ${{ steps.data-keys.outputs.archaic }}

      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        if: always() && steps.data-save.outputs.hla == 'true'
        with:
          path: ${{ env.E2E_DATA_CACHE }}/hla
          key: ${{ steps.data-keys.outputs.hla }}

      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        if: always() && steps.data-save.outputs.pgs == 'true'
        with:
          path: ${{ env.E2E_DATA_CACHE }}/pgs
          key: ${{ steps.data-keys.outputs.pgs }}

      # The case leaves the panel there only once its sha256 matched.
      - name: Data cache, HAPNEST panel, what to save
        id: hapnest-save
        if: >-
          always() && steps.hapnest.outputs.run == 'true' && env.E2E_HAPNEST_KEEP == 'true' &&
          steps.data-hapnest.outputs.cache-hit != 'true'
        run: |
          if [ -s "${E2E_DATA_CACHE}/hapnest/GRCh38_HAPNEST_reference.tar.zst" ]; then
            echo "save=true" >> "$GITHUB_OUTPUT"
          fi
          du -sh "${E2E_DATA_CACHE}/hapnest" 2> /dev/null || echo "Data cache: no HAPNEST panel to save"

      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        if: always() && steps.hapnest-save.outputs.save == 'true'
        with:
          path: ${{ env.E2E_DATA_CACHE }}/hapnest
          key: ${{ steps.hapnest.outputs.key }}

      - name: Upload the base outputs
        if: matrix.job == 'base'
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: e2e-base
          path: |
            ${{ runner.temp }}/base.tar.zst
            ${{ runner.temp }}/base.tar.zst.from
          compression-level: 0
          # Three days, so "Re-run failed jobs" the next day still finds it.
          retention-days: 3
          overwrite: true

      # The cases nextflow-from-fastq-1-bash and -2-nextflow ran the bash steps
      # and the Nextflow pipeline on the same reads as sample HG002P, in this
      # shard. Runs after a failed case too, so a run shows every difference at
      # once. Each passing check writes a line the job e2e counts: a parity
      # check that ran in no shard turns e2e red.
      - name: Parity self-test (a planted difference in each item must be caught)
        if: ${{ !cancelled() && steps.cases.outcome != 'skipped' && steps.cases-list.outputs.parity_fastq == 'true' }}
        run: scripts/ci/parity-diff.sh --self-test "${E2E_WORK}/genome/HG002P" HG002P

      - name: Parity of the bash scripts and the Nextflow pipeline
        if: ${{ !cancelled() && steps.cases.outcome != 'skipped' && steps.cases-list.outputs.parity_fastq == 'true' }}
        run: |
          scripts/ci/parity-diff.sh "${E2E_WORK}/genome/HG002P" "${E2E_WORK}/genome/nf-fastq/HG002P" HG002P
          printf 'fastq\tpass\n' >> "${E2E_WORK}/report/parity.tsv"

      # The case nextflow-hardening-2-nf ran the bash steps 22 (SURVIVOR) and 37
      # (Yleaf) beside the pipeline's SURVIVOR and Y haplogroup processes, on
      # the calls of steps 04 and 19 and on the same BAM.
      - name: Parity self-test, SV consensus and Y haplogroup
        if: ${{ !cancelled() && steps.cases.outcome != 'skipped' && steps.cases-list.outputs.parity_hardening == 'true' }}
        env:
          PARITY_ITEMS: sv_merged y_haplogroup
        run: scripts/ci/parity-diff.sh --self-test "${E2E_WORK}/genome-hardening/HG002" HG002

      - name: Parity of the SV consensus and the Y haplogroup
        if: ${{ !cancelled() && steps.cases.outcome != 'skipped' && steps.cases-list.outputs.parity_hardening == 'true' }}
        env:
          PARITY_ITEMS: sv_merged y_haplogroup
        run: |
          scripts/ci/parity-diff.sh "${E2E_WORK}/genome-hardening/HG002" "${E2E_WORK}/genome/nf-hardening/HG002" HG002
          printf 'hardening\tpass\n' >> "${E2E_WORK}/report/parity.tsv"

      - name: Upload this leg's results
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: e2e-report-${{ matrix.job }}
          path: ${{ env.E2E_WORK }}/report/*.*
          if-no-files-found: ignore
          retention-days: 7
          overwrite: true

      - name: Upload case logs
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: e2e-logs-${{ matrix.job }}
          path: ${{ env.E2E_WORK }}/logs/
          if-no-files-found: ignore
          overwrite: true

      # Base leg only. Prints each image's size and what became of it to the
      # job summary. Saves only on main (or when a run by hand asks for it with
      # save_images), only from a run whose base leg got through its cases
      # after pulling every image the cases use, and only
      # the groups whose exact key missed. A cache saved on a pull request is
      # readable by that pull request alone, and every entry counts against
      # the repository's 10 GB, so it would only push main's entries out.
      # Each group keeps its smallest images up to GROUP_BUDGET_MB of zstd
      # output; the rest are pulled on every run. Four groups of 1500 MB are
      # 5.9 GB, and while main replaces one group the old and the new entry
      # both count: 7.3 GB. With the five data entries above (about 2.3 GB:
      # MoChA 0.64, 1000 Genomes sites 1.36, HAPNEST 0.27, hla and GENCODE
      # 0.03, PGS a few KB) that is 9.6 GB, under the 10 GB limit. An image pinned by digest
      # is never saved: docker load gives it back without its digest, so
      # e2e-run.sh would pull it again anyway.
      - name: Image sizes, and save images for the next run
        id: save
        if: always() && matrix.job == 'base' && steps.keys.outcome == 'success'
        env:
          SAVE: ${{ (github.ref == 'refs/heads/main' || inputs.save_images) && 'true' || 'false' }}
          GROUP_BUDGET_MB: 1500
          HIT0: ${{ steps.images0.outputs.cache-hit }}
          HIT1: ${{ steps.images1.outputs.cache-hit }}
          HIT2: ${{ steps.images2.outputs.cache-hit }}
          HIT3: ${{ steps.images3.outputs.cache-hit }}
        run: |
          budget=$((GROUP_BUDGET_MB * 1024 * 1024))
          why=""
          if [ "$SAVE" != true ]; then
            why="not saved: only main, or a run by hand with save_images, saves"
          elif [ ! -f "${E2E_WORK}/all-cases-ran" ]; then
            why="not saved: the cases did not all run"
          else
            sudo rm -rf "${E2E_WORK}"
          fi
          table="${RUNNER_TEMP}/image-table.md"
          printf '### Image cache\n\n| Group | Image | Size | Cache |\n|---|---|---|---|\n' > "$table"
          for g in 0 1 2 3; do
            hit_var="HIT${g}"
            hit=${!hit_var}
            used=0 full=no
            # Smallest first, so the budget holds as many images as it can.
            while read -r size ref; do
              if [ "$size" = 0 ]; then shown="not here" state="-"
              else shown=$(numfmt --to=iec "$size"); fi
              if [ "$size" = 0 ]; then :
              elif [[ "$ref" == *@* ]]; then state="not saved: pinned by digest"
              elif grep -qxF "$(tr '/:' '__' <<< "$ref").tar.zst" "${RUNNER_TEMP}/images-loaded.txt" 2> /dev/null \
                   && { [ "$hit" = true ] || [ -n "$why" ]; }; then
                # Loaded from the cache and not saved again by this run.
                if [ "$hit" = true ]; then state="restored (exact key hit)"
                else state="restored (older entry); ${why}"; fi
              elif [ "$hit" = true ]; then state="pulled: not in the cache entry"
              elif [ -n "$why" ]; then state="$why"
              elif [ "$full" = yes ]; then state="not saved: over the group budget"
              else
                from=""
                grep -qxF "$(tr '/:' '__' <<< "$ref").tar.zst" "${RUNNER_TEMP}/images-loaded.txt" 2> /dev/null \
                  && from="restored (older entry), "
                f="${RUNNER_TEMP}/images/g${g}/$(tr '/:' '__' <<< "$ref").tar.zst"
                mkdir -p "${f%/*}"
                docker save "$ref" | zstd -T0 -12 -q -o "$f"
                fsize=$(stat -c %s "$f")
                if [ $((used + fsize)) -gt "$budget" ]; then
                  rm -f "$f"
                  full=yes state="not saved: over the group budget"
                else
                  used=$((used + fsize)) state="${from}saved, $(numfmt --to=iec "$fsize")"
                fi
              fi
              echo "| ${g} | ${ref} | ${shown} | ${state} |" >> "$table"
            done < <(awk -v g="$g" '$1 == g {print $2}' "${RUNNER_TEMP}/image-groups.txt" | while read -r ref; do
                       # A missing image prints an empty line and fails: read the size first.
                       size=$(docker image inspect -f '{{.Size}}' "$ref" 2> /dev/null) || size=0
                       echo "${size:-0} ${ref}"
                     done | sort -n)
            if [ "$used" -gt 0 ]; then
              echo "save${g}=true" >> "$GITHUB_OUTPUT"
              echo "| ${g} | group ${g} saved | | $(numfmt --to=iec "$used") of $(numfmt --to=iec "$budget") |" >> "$table"
            fi
          done
          tee -a "$GITHUB_STEP_SUMMARY" < "$table"

      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        if: always() && steps.save.outputs.save0 == 'true'
        with:
          path: ${{ runner.temp }}/images/g0
          key: ${{ steps.keys.outputs.g0 }}

      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        if: always() && steps.save.outputs.save1 == 'true'
        with:
          path: ${{ runner.temp }}/images/g1
          key: ${{ steps.keys.outputs.g1 }}

      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        if: always() && steps.save.outputs.save2 == 'true'
        with:
          path: ${{ runner.temp }}/images/g2
          key: ${{ steps.keys.outputs.g2 }}

      - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        if: always() && steps.save.outputs.save3 == 'true'
        with:
          path: ${{ runner.temp }}/images/g3
          key: ${{ steps.keys.outputs.g3 }}

  e2e-shards:
    needs: [e2e-scope, e2e-plan, e2e-base]
    if: ${{ !cancelled() && needs.e2e-plan.result == 'success' && needs.e2e-base.result == 'success' }}
    name: ${{ matrix.job == 'base' && 'e2e base' || format('e2e shard {0}', matrix.job) }}
    runs-on: ubuntu-latest
    # A traced run is one serial leg of every case: 230 minutes on 2026-11-02
    # before the scans.
    timeout-minutes: ${{ needs.e2e-plan.outputs.trace == 'true' && 355 || 240 }}
    permissions:
      contents: read
      actions: read
    strategy:
      fail-fast: false
      matrix:
        job: ${{ fromJSON(needs.e2e-plan.outputs.jobs) }}
    env:
      JOB: ${{ matrix.job }}
    steps: *leg-steps

  # The required check. Green when e2e-scope finds nothing to run; otherwise
  # green only when the plan and every leg succeeded, every case of the split
  # reported exactly one pass (one table of every case goes to the summary)
  # and each parity check passed once. A leg that was skipped, cancelled or
  # cut by its time limit leaves cases without a result, which is red too.
  e2e:
    needs: [e2e-scope, e2e-plan, e2e-base, e2e-shards]
    if: >-
      ${{ always() && (
        github.event_name == 'pull_request' ||
        github.event_name == 'push' ||
        github.event_name == 'schedule' ||
        (github.event_name == 'workflow_dispatch' && inputs.job == 'e2e')) }}
    runs-on: ubuntu-latest
    timeout-minutes: 15
    permissions:
      contents: read
      actions: read
    steps:
      - name: Does anything need to run
        id: scope
        env:
          RUN: ${{ needs.e2e-scope.outputs.run }}
        run: |
          if [ "$RUN" = false ]; then
            echo "Nothing the e2e cases exercise changed: e2e passes without running." | tee -a "$GITHUB_STEP_SUMMARY"
            echo "skip=true" >> "$GITHUB_OUTPUT"
          fi

      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        if: steps.scope.outputs.skip != 'true'
        with:
          persist-credentials: false

      - name: Every case passed once
        if: steps.scope.outputs.skip != 'true'
        env:
          GH_TOKEN: ${{ github.token }}
          RUN_ID: ${{ github.run_id }}
          PLAN: ${{ needs.e2e-plan.result }}
          BASE: ${{ needs.e2e-base.result }}
          LEGS: ${{ needs.e2e-shards.result }}
          SHARDS: ${{ needs.e2e-plan.outputs.shards }}
          TRACE: ${{ needs.e2e-plan.outputs.trace }}
        run: |
          set -o pipefail
          rc=0
          if [ "$PLAN" != success ] || [ "$BASE" != success ] || [ "$LEGS" != success ]; then
            echo "::error::The plan job ended ${PLAN:-unknown}, the base leg ${BASE:-unknown} and the shard legs ${LEGS:-unknown}; e2e needs all three to succeed."
            rc=1
          fi
          dir="${RUNNER_TEMP}/reports"
          mkdir -p "$dir"
          gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/artifacts?per_page=100" \
            --jq '.artifacts[] | select(.name | startswith("e2e-report-")) | "\(.id) \(.name)"' |
            while read -r id name; do
              gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${id}/zip" > "${dir}/${name}.zip"
              unzip -o -q "${dir}/${name}.zip" -d "${dir}/${name}"
            done
          expected="${RUNNER_TEMP}/expected.txt"
          { scripts/ci/e2e-shards.sh base; scripts/ci/e2e-shards.sh plan "${SHARDS:-1}" | cut -f2; } > "$expected"
          shopt -s nullglob
          results=("$dir"/*/results.tsv)
          tag=$(tr -d '[:space:]' < tests/fixtures/VERSION)
          table="${RUNNER_TEMP}/table.md"
          scripts/ci/e2e-shards.sh merge "$expected" /dev/null "${results[@]}" > "$table" || rc=1
          {
            echo "### E2E on fixture ${tag}: $(tail -n 1 "$table")"
            echo
            sed '$d' "$table"
            echo "Logs are in the e2e-logs-<job> artifacts of this run."
            for f in "$dir"/*/extra.md; do [ -s "$f" ] && cat "$f"; done
          } >> "$GITHUB_STEP_SUMMARY"
          tail -n 1 "$table"
          for p in fastq hardening; do
            n=$(cat /dev/null "$dir"/*/parity.tsv | grep -cx "${p}"$'\t'"pass" || true)
            if [ "$n" -ne 1 ]; then
              echo "::error::The ${p} parity check passed ${n} times; it must pass once."
              rc=1
            fi
          done
          if [ "$TRACE" = true ]; then
            traces=("$dir"/*/trace.tsv)
            if scripts/ci/e2e-shards.sh times "${traces[@]}" > "${RUNNER_TEMP}/e2e-case-times.tsv"; then
              { echo; echo "### Traced scripts/ci/e2e-case-times.tsv"; echo; echo '```'; cat "${RUNNER_TEMP}/e2e-case-times.tsv"; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
            else
              rc=1
            fi
          fi
          exit "$rc"

      - name: Upload the traced table
        if: always() && needs.e2e-plan.outputs.trace == 'true' && steps.scope.outputs.skip != 'true'
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: e2e-case-times
          path: ${{ runner.temp }}/e2e-case-times.tsv
          if-no-files-found: ignore

  settle-doubts:
    needs: build-fixture
    if: >-
      always() &&
      (needs.build-fixture.result == 'success' || needs.build-fixture.result == 'skipped') &&
      (github.event_name == 'push' || github.event_name == 'pull_request' ||
       (github.event_name == 'workflow_dispatch' && inputs.job == 'settle-doubts'))
    runs-on: ubuntu-latest
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          fetch-depth: 0

      # On a push or a pull request this job only runs when its own script
      # changed: in the push, or in the pull request against main. A pull
      # request ignores github.event.before: on a second push that is the old
      # head, and the diff to the merge commit would also hold whatever main
      # changed since.
      - name: Decide whether to run
        id: need
        env:
          BEFORE: ${{ github.event.before }}
          EVENT: ${{ github.event_name }}
        run: |
          if [ "$EVENT" = "workflow_dispatch" ]; then echo "run=true" >> "$GITHUB_OUTPUT"; exit 0; fi
          [ "$EVENT" = "push" ] || BEFORE=""
          base="$BEFORE"
          if [ -z "$base" ] || [[ "$base" =~ ^0+$ ]] || ! git cat-file -e "${base}^{commit}" 2>/dev/null; then
            base=$(git merge-base origin/main HEAD)
          fi
          if git diff --name-only "$base" HEAD | grep -qx 'scripts/ci/settle-doubts.sh'; then
            echo "run=true" >> "$GITHUB_OUTPUT"
          else
            echo "scripts/ci/settle-doubts.sh did not change: nothing to do."
            echo "run=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Free disk space
        if: steps.need.outputs.run == 'true'
        run: |
          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup
          df -h /

      - name: Settle the doubts
        if: steps.need.outputs.run == 'true'
        env:
          GH_TOKEN: ${{ github.token }}
        run: scripts/ci/settle-doubts.sh "${RUNNER_TEMP}/settle"

  # Two legs, about 30 minutes each. requests: each task under its request in
  # conf/base.config, capped below the runner's 16 GB, with swap on top so
  # the 24 GB of PRS can be honoured; it must finish, with the headroom the
  # script checks. labels: the limits of the process labels before the
  # requests (PRS_PREPARE 2 GB, FORMAT_SCOREFILES 8 GB on 2 threads, as
  # pgsc_calc gave it), no swap; every task must be killed, which shows the
  # first leg can fail.
  prs-memory:
    if: github.event_name == 'workflow_dispatch' && inputs.job == 'prs-memory'
    runs-on: ubuntu-latest
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        include:
          - leg: requests
            swap_gb: 28
            format_limit_gb: ''
            prepare_limit_gb: ''
            threads: '1'
            expect: pass
          - leg: labels
            swap_gb: 0
            format_limit_gb: '8'
            prepare_limit_gb: '2'
            threads: '2'
            expect: killed
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Add swap
        if: matrix.swap_gb != 0
        env:
          SWAP_GB: ${{ matrix.swap_gb }}
        run: |
          sudo fallocate -l "${SWAP_GB}G" /mnt/swap-prs
          sudo chmod 600 /mnt/swap-prs
          sudo mkswap /mnt/swap-prs
          sudo swapon /mnt/swap-prs
          free -m

      - name: Measure (${{ matrix.leg }})
        env:
          SWAP_GB: ${{ matrix.swap_gb }}
          FORMAT_LIMIT_GB: ${{ matrix.format_limit_gb }}
          PREPARE_LIMIT_GB: ${{ matrix.prepare_limit_gb }}
          FORMAT_THREADS: ${{ matrix.threads }}
          EXPECT: ${{ matrix.expect }}
        run: |
          export MEASURE_SWAP_MB=$((SWAP_GB * 1024))
          bash tests/test_prs_memory.sh --measure "${RUNNER_TEMP}/prs-memory"