# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 name: "CI: Coverage" on: schedule: - cron: '0 0 * * *' # This runs the workflow every day at 12:00 AM UTC workflow_dispatch: {} env: PY_VER: "3.14" # Floor spec for setup-python: first 3.14 whose ssl module rejects the # compromised eMudhra keys (CVE-2026-115557). PY_VER stays minor-only for # env-vars/naming; PY_VER_FLOOR is used only as the setup-python spec. PY_VER_FLOOR: "~3.14.10" LOCAL_CTK: "1" GPU: "a100" DRIVER: "latest" jobs: coverage-vars: if: ${{ github.repository_owner == 'nvidia' }} runs-on: ubuntu-latest outputs: CUDA_VER: ${{ steps.get-vars.outputs.cuda_ver }} steps: - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Get CUDA version id: get-vars run: | cuda_ver=$(yq '.cuda.build.version' ci/versions.yml) echo "cuda_ver=$cuda_ver" >> $GITHUB_OUTPUT coverage-linux: name: Coverage (Linux) needs: [coverage-vars] runs-on: "linux-amd64-gpu-a100-latest-1" permissions: id-token: write contents: write defaults: run: shell: bash --noprofile --norc -xeuo pipefail {0} env: HOST_PLATFORM: "linux-64" ARCH: "x86_64" CUDA_VER: ${{ needs.coverage-vars.outputs.CUDA_VER }} # Our self-hosted runners require a container # TODO: use a different (nvidia?) container container: options: -u root --security-opt seccomp=unconfined --shm-size 16g image: ubuntu:24.04 env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} PIP_CACHE_DIR: "/tmp/pip-cache" steps: - name: Ensure GPU is working run: nvidia-smi # We have to install git before checking out the repo (so that we can # deploy the docs at the end). This means we can't use install_unix_deps # action so install the git package. - name: Install git run: | apt-get update apt-get install -y git - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Treeless clone: setuptools-scm needs the commit graph (`git describe`) # but not historical blobs. fetch-depth: 0 filter: blob:none # third_party/cccl provides the CCCL headers cuda.core compiles against. submodules: true - name: Fix workspace ownership run: | chown -R $(id -u):$(id -g) "$GITHUB_WORKSPACE" - name: Install dependencies uses: ./.github/actions/install_unix_deps continue-on-error: false with: dependencies: "tree rsync libsqlite3-0 g++ jq wget libgl1 libegl1 yq" dependent_exes: "tree rsync libsqlite3-0 g++ jq wget libgl1 libegl1 yq" - name: Setup proxy cache uses: nv-gha-runners/setup-proxy-cache@main continue-on-error: true - name: Set environment variables env: BUILD_CUDA_VER: ${{ env.CUDA_VER }} CUDA_VER: ${{ env.CUDA_VER }} HOST_PLATFORM: ${{ env.HOST_PLATFORM }} LOCAL_CTK: ${{ env.LOCAL_CTK }} PY_VER: ${{ env.PY_VER }} SHA: ${{ github.sha }} run: | ./ci/tools/env-vars test echo "CUDA_PYTHON_COVERAGE=1" >> $GITHUB_ENV - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PY_VER_FLOOR }} env: # we use self-hosted runners on which setup-python behaves weirdly... AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache" - name: Set up mini CTK if: ${{ env.LOCAL_CTK == '1' }} uses: ./.github/actions/fetch_ctk continue-on-error: false with: host-platform: ${{ env.HOST_PLATFORM }} cuda-version: ${{ env.CUDA_VER }} - name: Create venv run: | python -m venv .venv - name: Harden pip uses: ./.github/actions/harden-pip with: python: .venv/bin/python - name: Build and install cuda-pathfinder wheel run: | .venv/bin/pip wheel -v --no-deps ./cuda_pathfinder -w ./wheels/ .venv/bin/pip install -v ./wheels/cuda_pathfinder*.whl --group ./cuda_pathfinder/pyproject.toml:test - name: Constrain builds to the local cuda-pathfinder wheel run: | pathfinder_wheels=(wheels/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test -f "${pathfinder_wheels[0]}" mkdir -p wheel-constraints pathfinder_uri="file://$(realpath "${pathfinder_wheels[0]}")" printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}" | tee wheel-constraints/cuda-bindings.txt - name: Build and install cuda-bindings wheel run: | export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-bindings.txt" export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" .venv/bin/pip wheel -v --no-deps ./cuda_bindings_13 -w ./wheels/ .venv/bin/pip install -v ./wheels/cuda_bindings*.whl --group ./cuda_bindings_13/pyproject.toml:test - name: Constrain cuda-core to the local cuda-bindings wheel run: | CUDA_MAJOR="${CUDA_VER%%.*}" pathfinder_wheels=(wheels/cuda_pathfinder-*.whl) bindings_wheels=(wheels/cuda_bindings-"${CUDA_MAJOR}".*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test "${#bindings_wheels[@]}" -eq 1 test -f "${pathfinder_wheels[0]}" test -f "${bindings_wheels[0]}" mkdir -p wheel-constraints pathfinder_uri="file://$(realpath "${pathfinder_wheels[0]}")" bindings_uri="file://$(realpath "${bindings_wheels[0]}")" { printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}" printf 'cuda-bindings @ %s\n' "${bindings_uri}" } | tee wheel-constraints/cuda-core.txt - name: Build and install cuda-core run: | export PIP_BUILD_CONSTRAINT="$(pwd)/wheel-constraints/cuda-core.txt" export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" cd cuda_core ../.venv/bin/pip install -v . --group test - name: Install coverage tools run: | .venv/bin/pip install coverage pytest-cov Cython - name: Set cuda package install root run: | echo "INSTALL_ROOT=$(.venv/bin/python -c 'import cuda; import os; print(os.path.dirname(cuda.__path__[0]))')" >> $GITHUB_ENV echo "REPO_ROOT=$(pwd)" >> $GITHUB_ENV - name: Run cuda.pathfinder tests continue-on-error: true run: | cd $INSTALL_ROOT $REPO_ROOT/.venv/bin/pytest -v --cov=./cuda --cov-append --cov-context=test --cov-config=$REPO_ROOT/.coveragerc $REPO_ROOT/cuda_pathfinder/tests - name: Run cuda.bindings tests continue-on-error: true run: | cd $INSTALL_ROOT $REPO_ROOT/.venv/bin/pytest -v --cov=./cuda --cov-append --cov-context=test --cov-config=$REPO_ROOT/.coveragerc $REPO_ROOT/cuda_bindings_13/tests - name: Run cuda.core tests continue-on-error: true run: | cd $INSTALL_ROOT $REPO_ROOT/.venv/bin/pytest -v --cov=./cuda --cov-append --cov-context=test --cov-config=$REPO_ROOT/.coveragerc $REPO_ROOT/cuda_core/tests - name: Copy Linux coverage file to workspace run: | cp $INSTALL_ROOT/.coverage $REPO_ROOT/.coverage.linux echo "Copied .coverage.linux to $REPO_ROOT" ls -lh $REPO_ROOT/.coverage.linux - name: Upload Linux coverage data uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: coverage-data-linux path: .coverage.linux retention-days: 7 include-hidden-files: true if-no-files-found: error - name: Upload cuda source code for coverage mapping uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: cuda-source-linux path: ${{ env.INSTALL_ROOT }}/cuda/ retention-days: 7 if-no-files-found: error # Build Windows wheels on GitHub-hosted runner (has VS, no GPU) build-wheel-windows: name: Build Wheels (Windows) needs: [coverage-vars] runs-on: windows-2022 permissions: contents: read defaults: run: shell: bash --noprofile --norc -xeuo pipefail {0} env: HOST_PLATFORM: "win-64" CUDA_PYTHON_COVERAGE: "1" CUDA_VER: ${{ needs.coverage-vars.outputs.CUDA_VER }} steps: - name: Enable Git symlinks (Windows, must precede checkout) run: git config --global core.symlinks true - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Treeless clone: setuptools-scm needs the commit graph (`git describe`) # but not historical blobs. fetch-depth: 0 filter: blob:none # third_party/cccl provides the CCCL headers cuda.core compiles against. submodules: true - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PY_VER_FLOOR }} - name: Set up MSVC uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4 # v1.13.1 - name: Set up mini CTK uses: ./.github/actions/fetch_ctk continue-on-error: false with: host-platform: win-64 cuda-version: ${{ env.CUDA_VER }} - name: Create venv run: | python -m venv .venv - name: Harden pip uses: ./.github/actions/harden-pip with: python: .venv/Scripts/python - name: Build cuda.pathfinder wheel run: | .venv/Scripts/python -m pip install wheel setuptools Cython .venv/Scripts/pip wheel -v --no-deps ./cuda_pathfinder -w ./wheels/ - name: Constrain builds to the local cuda.pathfinder wheel run: | pathfinder_wheels=(wheels/cuda_pathfinder-*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test -f "${pathfinder_wheels[0]}" mkdir -p wheel-constraints pathfinder_uri="file:///$(cygpath -am "${pathfinder_wheels[0]}")" printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}" | tee wheel-constraints/cuda-bindings.txt - name: Build cuda.bindings wheel run: | export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-bindings.txt")" export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" cd cuda_bindings_13 ../.venv/Scripts/pip wheel -v --no-deps . -w ../wheels/ - name: Constrain cuda.core to the local cuda.bindings wheel run: | CUDA_MAJOR="${CUDA_VER%%.*}" pathfinder_wheels=(wheels/cuda_pathfinder-*.whl) bindings_wheels=(wheels/cuda_bindings-"${CUDA_MAJOR}".*.whl) test "${#pathfinder_wheels[@]}" -eq 1 test "${#bindings_wheels[@]}" -eq 1 test -f "${pathfinder_wheels[0]}" test -f "${bindings_wheels[0]}" mkdir -p wheel-constraints pathfinder_uri="file:///$(cygpath -am "${pathfinder_wheels[0]}")" bindings_uri="file:///$(cygpath -am "${bindings_wheels[0]}")" { printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}" printf 'cuda-bindings @ %s\n' "${bindings_uri}" } | tee wheel-constraints/cuda-core.txt - name: Build cuda.core wheel run: | export PIP_BUILD_CONSTRAINT="$(cygpath -w "$(pwd)/wheel-constraints/cuda-core.txt")" export PIP_CONSTRAINT="${PIP_BUILD_CONSTRAINT}" cd cuda_core ../.venv/Scripts/pip wheel -v --no-deps . -w ../wheels/ # Vendor the DLLs these wheels were built against, the way cibuildwheel # does for every other Windows build. --namespace-pkg is needed because # `cuda` is a namespace package. - name: Repair the Windows wheels run: | .venv/Scripts/pip install delvewheel mkdir -p wheels-repaired for whl in ./wheels/cuda_bindings-*.whl ./wheels/cuda_core-*.whl; do .venv/Scripts/delvewheel repair --namespace-pkg cuda \ --exclude "torch_cpu.dll;torch_python.dll" \ -w ./wheels-repaired "$whl" done mv -f ./wheels-repaired/*.whl ./wheels/ - name: List wheel artifacts run: | echo "=== Windows wheel artifacts ===" ls -lahR ./wheels/ - name: Upload Windows wheel artifacts uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: coverage-windows-wheels path: ./wheels/*.whl retention-days: 1 if-no-files-found: error # Run coverage tests on self-hosted GPU runner (no VS needed, installs pre-built wheels) coverage-windows: name: Coverage (Windows) needs: [coverage-vars, build-wheel-windows] runs-on: "windows-amd64-gpu-a100-latest-1" permissions: id-token: write contents: write env: HOST_PLATFORM: "win-64" ARCH: "amd64" CUDA_PYTHON_COVERAGE: "1" CUDA_VER: ${{ needs.coverage-vars.outputs.CUDA_VER }} defaults: run: shell: bash --noprofile --norc -xeuo pipefail {0} steps: - name: Enable Git symlinks (Windows, must precede checkout) run: git config --global core.symlinks true - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 1 - name: Setup proxy cache uses: nv-gha-runners/setup-proxy-cache@main continue-on-error: true # DRIVER above is 'latest' so install_gpu_driver.ps1 is intentionally # skipped (it errors on latest/earliest); configure_driver_mode.ps1 # still runs to put the pre-installed driver into TCC mode. - name: Configure driver mode shell: powershell env: DRIVER_MODE: "TCC" run: | ci/tools/configure_driver_mode.ps1 - name: Ensure GPU is working run: | nvidia-smi - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PY_VER_FLOOR }} - name: Set up mini CTK uses: ./.github/actions/fetch_ctk continue-on-error: false with: host-platform: win-64 cuda-version: ${{ env.CUDA_VER }} - name: Download Windows wheel artifacts uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: coverage-windows-wheels path: ./wheels/ - name: List downloaded wheel artifacts run: | echo "=== Downloaded wheel artifacts ===" ls -lahR ./wheels/ - name: Create venv run: | python -m venv .venv - name: Harden pip uses: ./.github/actions/harden-pip with: python: .venv/Scripts/python - name: Install wheels from build job run: | .venv/Scripts/pip install ./wheels/cuda_pathfinder*.whl echo "Installed cuda.pathfinder" .venv/Scripts/pip install ./wheels/cuda_bindings*.whl echo "Installed cuda.bindings" .venv/Scripts/pip install ./wheels/cuda_core*.whl echo "Installed cuda.core" - name: Install test dependencies and coverage tools run: | .venv/Scripts/pip install coverage pytest-cov Cython .venv/Scripts/pip install --group ./cuda_pathfinder/pyproject.toml:test .venv/Scripts/pip install --group ./cuda_bindings_13/pyproject.toml:test .venv/Scripts/pip install --group ./cuda_core/pyproject.toml:test - name: Get install root id: install-root run: | INSTALL_ROOT=$(.venv/Scripts/python -c 'import cuda; import os; print(os.path.dirname(cuda.__path__[0]))') echo "INSTALL_ROOT=$INSTALL_ROOT" >> $GITHUB_OUTPUT echo "Install root: $INSTALL_ROOT" - name: Run cuda.pathfinder tests continue-on-error: true run: | cd "${{ steps.install-root.outputs.INSTALL_ROOT }}" "$GITHUB_WORKSPACE/.venv/Scripts/pytest" -v --cov=./cuda --cov-append --cov-context=test --cov-config="$GITHUB_WORKSPACE/.coveragerc" "$GITHUB_WORKSPACE/cuda_pathfinder/tests" # Cython linetrace under coverage on Windows needs more stack than the # default 1 MB thread size. The helper runs pytest on an 8 MB thread. - name: Run cuda.bindings tests (with 8MB stack) continue-on-error: true run: | "$GITHUB_WORKSPACE/.venv/Scripts/python" "$GITHUB_WORKSPACE/ci/tools/run_pytest_with_stack.py" \ --cwd "${{ steps.install-root.outputs.INSTALL_ROOT }}" \ -v --cov=./cuda --cov-append --cov-context=test \ --cov-config="$GITHUB_WORKSPACE/.coveragerc" \ "$GITHUB_WORKSPACE/cuda_bindings_13/tests" - name: Run cuda.core tests (with 8MB stack) continue-on-error: true run: | "$GITHUB_WORKSPACE/.venv/Scripts/python" "$GITHUB_WORKSPACE/ci/tools/run_pytest_with_stack.py" \ --cwd "${{ steps.install-root.outputs.INSTALL_ROOT }}" \ -v --cov=./cuda --cov-append --cov-context=test \ --cov-config="$GITHUB_WORKSPACE/.coveragerc" \ "$GITHUB_WORKSPACE/cuda_core/tests" - name: Copy Windows coverage file to workspace run: | cp "${{ steps.install-root.outputs.INSTALL_ROOT }}/.coverage" "$GITHUB_WORKSPACE/.coverage.windows" echo "Copied .coverage.windows to $GITHUB_WORKSPACE" ls -lh "$GITHUB_WORKSPACE/.coverage.windows" - name: Upload Windows coverage data uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: coverage-data-windows path: .coverage.windows retention-days: 7 include-hidden-files: true if-no-files-found: error combine-and-deploy: name: Combine Coverage needs: [coverage-linux, coverage-windows] runs-on: ubuntu-latest if: ${{ always() && github.repository_owner == 'nvidia' }} permissions: id-token: write contents: write steps: - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PY_VER_FLOOR }} - name: Harden pip uses: ./.github/actions/harden-pip - name: Install coverage run: | # .coveragerc enables Cython.Coverage plugin; ensure it's available here too. pip install coverage[toml] Cython - name: Download Linux coverage data uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: coverage-data-linux path: ./ - name: Download cuda source code uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: cuda-source-linux path: ./cuda/ - name: Download Windows coverage data uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: coverage-data-windows path: ./ - name: Combine coverage data run: | echo "=== Working directory ===" pwd echo "" echo "=== All files in root ===" ls -la echo "" # Verify both Linux and Windows coverage files exist echo "=== Checking coverage files ===" if [ ! -f ".coverage.linux" ]; then echo "[ERROR] .coverage.linux not found!" exit 1 fi echo "[OK] Found .coverage.linux" if [ ! -f ".coverage.windows" ]; then echo "[ERROR] .coverage.windows not found!" exit 1 fi echo "[OK] Found .coverage.windows" echo "=== Available coverage files ===" ls -lh .coverage.* echo "=== Cuda source structure ===" find ./cuda -type d || { echo "[ERROR] No cuda source directory!" exit 1 } # Combine all .coverage.* files echo "" echo "=== Combining coverage data ===" coverage combine --rcfile=./.coveragerc --keep .coverage.* - name: Generate coverage reports run: | echo "" echo "=== Generating HTML, XML, and text reports ===" coverage html --rcfile=./.coveragerc coverage xml --rcfile=./.coveragerc -o coverage.xml coverage report --rcfile=./.coveragerc # Prepare for upload mkdir -p ./docs/coverage mv htmlcov/* ./docs/coverage/ mv coverage.xml ./docs/coverage/ echo "" echo "[SUCCESS] Coverage reports generated successfully" - name: Archive combined coverage results uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: coverage-combined path: docs/coverage/ retention-days: 7 include-hidden-files: true deploy: name: Deploy Coverage # GitHub discards prerequisite artifacts on a new run attempt. If only this # job fails, use "Re-run all jobs" so coverage recreates its payload. needs: combine-and-deploy if: ${{ needs.combine-and-deploy.result == 'success' && github.repository_owner == 'nvidia' }} runs-on: ubuntu-latest permissions: contents: write # Must match every job that writes to gh-pages. concurrency: group: cuda-python-gh-pages-publish queue: max cancel-in-progress: false steps: - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Download combined coverage results uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: coverage-combined path: docs/coverage/ - name: Deploy to gh-pages uses: JamesIves/github-pages-deploy-action@fa24774553152dd7873cd16ebd8d959b010c5445 # v4.9.0 with: git-config-name: cuda-python-bot git-config-email: cuda-python-bot@users.noreply.github.com folder: docs/ target-folder: docs/ commit-message: "Deploy combined coverage (Linux + Windows): ${{ github.sha }}" clean: false # Rebase onto concurrent docs deployments instead of force-pushing, # which the gh-pages ruleset rejects. force: false