name: End-to-end tests on: workflow_dispatch: pull_request: merge_group: permissions: contents: read env: CONTAINER_TOOL: docker # Retry a failed spec once before failing the job. Retries are reported as # flakes in the ginkgo summary, so this hides nothing; it only stops a single # infrastructure hiccup from failing an unrelated PR. E2E_FLAKE_ATTEMPTS: "2" # Run the non-Serial containers two at a time. The runner has 4 CPUs and the # suite spends most of its time waiting on the cluster, so a second process # overlaps the two long containers (hooks, core) rather than competing for CPU. E2E_PROCS: "2" concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} cancel-in-progress: true jobs: changes: if: github.repository_owner == 'jumpstarter-dev' runs-on: ubuntu-latest outputs: should_run: ${{ steps.filter.outputs.e2e }} e2e-matrix: ${{ steps.matrix.outputs.e2e-matrix }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4 id: filter with: base: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }} # Required for the '!...' exclusion patterns below to work; with the # default 'some' quantifier a negated pattern matches every file it # does NOT cover, so docs-only PRs would still trigger e2e. predicate-quantifier: some-with-excludes filters: | # Run e2e on any change except pure documentation/tooling: # the suite exercises the whole stack, so a positive path list # would keep missing new code directories (e.g. rust/). e2e: - '**' - '!docs/**' - '!**/*.md' - '!**/*.mdc' - '!.devcontainer/**' - name: Compute e2e architecture matrix id: matrix run: | # PRs: run only amd64 e2e tests to save CI time. # Merge queue and workflow_dispatch run both architectures. if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo 'e2e-matrix=[{"os": "ubuntu-24.04", "arch": "amd64"}]' >> "$GITHUB_OUTPUT" else echo 'e2e-matrix=[{"os": "ubuntu-24.04", "arch": "amd64"}, {"os": "ubuntu-24.04-arm", "arch": "arm64"}]' >> "$GITHUB_OUTPUT" fi # =========================================================================== # Build jobs: container images and Python wheels built in parallel # =========================================================================== build-controller-image: needs: changes if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' strategy: matrix: include: ${{ fromJson(needs.changes.outputs.e2e-matrix) }} runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Install Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: .go-version cache-dependency-path: "**/go.sum" - name: Cache controller image id: cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: /tmp/controller-image.tar # The tar also holds the telemetry image; the prefix changed when it was added. key: controller-telemetry-image-${{ matrix.arch }}-${{ hashFiles('.go-version', 'controller/Makefile', 'controller/Containerfile', 'controller/Containerfile.prebuilt', 'controller/Containerfile.telemetry.prebuilt', 'controller/go.mod', 'controller/go.sum', 'controller/cmd/**', 'controller/api/**', 'controller/internal/**') }} - name: Build controller image if: steps.cache.outputs.cache-hit != 'true' run: | make -C controller docker-build-ci docker save quay.io/jumpstarter-dev/jumpstarter-controller:latest \ quay.io/jumpstarter-dev/jumpstarter-telemetry:latest -o /tmp/controller-image.tar - name: Upload controller image uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: controller-image-${{ matrix.arch }} path: /tmp/controller-image.tar retention-days: 1 build-operator-image: needs: changes if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' strategy: matrix: include: ${{ fromJson(needs.changes.outputs.e2e-matrix) }} runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Install Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: .go-version cache-dependency-path: "**/go.sum" - name: Cache operator artifacts id: cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: | /tmp/operator-image.tar /tmp/operator-install.yaml key: operator-image-${{ matrix.arch }}-${{ hashFiles('.go-version', 'controller/Makefile', 'controller/Containerfile.operator', 'controller/Containerfile.prebuilt', 'controller/go.mod', 'controller/go.sum', 'controller/deploy/operator/**', 'controller/api/**', 'controller/internal/**') }} - name: Build operator image and installer manifest if: steps.cache.outputs.cache-hit != 'true' run: | make -C controller/deploy/operator docker-build-ci build-installer VERSION=latest docker save quay.io/jumpstarter-dev/jumpstarter-operator:latest -o /tmp/operator-image.tar cp controller/deploy/operator/dist/install.yaml /tmp/operator-install.yaml - name: Upload operator artifacts uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: operator-image-${{ matrix.arch }} path: | /tmp/operator-image.tar /tmp/operator-install.yaml retention-days: 1 build-exporterset-controller-image: needs: changes if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' strategy: matrix: include: ${{ fromJson(needs.changes.outputs.e2e-matrix) }} runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Install Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: .go-version cache-dependency-path: "**/go.sum" - name: Cache exporterset-controller image id: cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: /tmp/exporterset-controller-image.tar key: exporterset-controller-image-${{ matrix.arch }}-${{ hashFiles('.go-version', 'controller/Makefile', 'controller/Containerfile.exporter-set-controller', 'controller/Containerfile.prebuilt', 'controller/go.mod', 'controller/go.sum', 'controller/cmd/exporter-set-controller/**', 'controller/api/**', 'controller/internal/**') }} - name: Build exporterset-controller image if: steps.cache.outputs.cache-hit != 'true' run: | make -C controller docker-build-exporter-set-controller-ci docker save quay.io/jumpstarter-dev/jumpstarter-exporterset-controller:latest -o /tmp/exporterset-controller-image.tar - name: Upload exporterset-controller image uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: exporterset-controller-image-${{ matrix.arch }} path: /tmp/exporterset-controller-image.tar retention-days: 1 build-exporter-image: needs: changes if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' strategy: matrix: include: ${{ fromJson(needs.changes.outputs.e2e-matrix) }} runs-on: ${{ matrix.os }} timeout-minutes: 45 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Cache exporter image id: cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: /tmp/exporter-image.tar # Bust when exporter image inputs change: Containerfile, Python sources, # lockfiles, or embedded jumpstarter-exec (rust/). key: exporter-image-${{ matrix.arch }}-${{ hashFiles('python/Containerfile', 'python/packages/**/*.py', 'python/**/pyproject.toml', 'python/uv.lock', 'rust/jumpstarter-exec/**', 'controller/Makefile') }} - name: Build exporter image if: steps.cache.outputs.cache-hit != 'true' run: | make -C controller docker-build-exporter docker save quay.io/jumpstarter-dev/jumpstarter:latest -o /tmp/exporter-image.tar - name: Upload exporter image uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: exporter-image-${{ matrix.arch }} path: /tmp/exporter-image.tar retention-days: 1 build-qemu-runtime-image: needs: changes if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' strategy: matrix: include: ${{ fromJson(needs.changes.outputs.e2e-matrix) }} runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Cache qemu-runtime image id: cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: /tmp/qemu-runtime-image.tar # Bust when QEMU runtime inputs change. key: qemu-runtime-image-${{ matrix.arch }}-${{ hashFiles('controller/Makefile', 'controller/Containerfile.qemu-runtime') }} - name: Build qemu-runtime image if: steps.cache.outputs.cache-hit != 'true' run: | make -C controller docker-build-qemu-runtime docker save quay.io/jumpstarter-dev/virtual/qemu-runtime:latest -o /tmp/qemu-runtime-image.tar - name: Upload qemu-runtime image uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: qemu-runtime-image-${{ matrix.arch }} path: /tmp/qemu-runtime-image.tar retention-days: 1 build-python-wheels: needs: changes if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-26.04 timeout-minutes: 15 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - id: uv run: echo "version=$(cat .uv-version)" >> "$GITHUB_OUTPUT" - name: Install uv uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: version: ${{ steps.uv.outputs.version }} python-version-file: .py-version - name: Cache python wheels id: cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: python/dist key: python-wheels-${{ hashFiles('python/**/*.py', 'python/**/pyproject.toml', 'python/uv.lock') }} - name: Build python wheels if: steps.cache.outputs.cache-hit != 'true' working-directory: python run: make build - name: Upload python wheels uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: python-wheels path: python/dist/ retention-days: 1 # =========================================================================== # E2E test jobs: depend on pre-built artifacts # =========================================================================== e2e-tests: needs: [changes, build-controller-image, build-operator-image, build-exporterset-controller-image, build-exporter-image, build-qemu-runtime-image, build-python-wheels] strategy: matrix: include: ${{ fromJson(needs.changes.outputs.e2e-matrix) }} runs-on: ${{ matrix.os }} # Includes ExporterSet QEMU coverage (TCG), including flash/boot. # Job > Ginkgo suite timeout (60m in e2e/lib/common.sh) for setup/log upload. timeout-minutes: 70 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - id: uv run: echo "version=$(cat .uv-version)" >> "$GITHUB_OUTPUT" - name: Install uv uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: version: ${{ steps.uv.outputs.version }} python-version-file: .py-version - name: Install Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: .go-version cache-dependency-path: "**/go.sum" - name: Load e2e artifacts uses: ./.github/actions/load-e2e-artifacts with: arch: ${{ matrix.arch }} - name: Install nftables and dnsmasq for dut-network E2E run: | sudo apt-get update sudo apt-get install -y nftables dnsmasq-base - name: Load kernel modules for DUT network tests run: | for mod in veth bridge nf_nat nf_conntrack nft_masq nft_nat nft_chain_nat; do sudo modprobe "$mod" 2>/dev/null || true done - name: Read tool versions from Makefile uses: ./.github/actions/read-tool-versions - name: Cache controller tools uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: controller/bin key: controller-tools-${{ matrix.arch }}-kind${{ env.KIND_VERSION }}-kustomize${{ env.KUSTOMIZE_VERSION }}-grpcurl${{ env.GRPCURL_VERSION }} - name: Setup e2e test environment run: make e2e-setup env: CI: true SKIP_BUILD: "true" PREBUILT_WHEELS_DIR: /tmp/python-wheels OPERATOR_IMG: quay.io/jumpstarter-dev/jumpstarter-operator:latest - name: Run e2e tests run: make e2e-run env: CI: true - name: Collect and upload e2e logs if: always() uses: ./.github/actions/collect-e2e-logs with: artifact-name: e2e-logs-${{ matrix.arch }} # =========================================================================== # ExporterSet QEMU-SSH: off-cluster provisioner via SSH to localhost # =========================================================================== e2e-exporterset-qemu-ssh: needs: [changes, build-controller-image, build-operator-image, build-exporterset-controller-image, build-exporter-image, build-qemu-runtime-image, build-python-wheels] if: needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' strategy: matrix: include: ${{ fromJson(needs.changes.outputs.e2e-matrix) }} runs-on: ${{ matrix.os }} # qemu-ssh setup + TCG flash/boot; timeout > ginkgo's 60m. timeout-minutes: 75 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - id: uv run: echo "version=$(cat .uv-version)" >> "$GITHUB_OUTPUT" - name: Install uv uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: version: ${{ steps.uv.outputs.version }} python-version-file: .py-version - name: Install Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: .go-version cache-dependency-path: "**/go.sum" - name: Load e2e artifacts uses: ./.github/actions/load-e2e-artifacts with: arch: ${{ matrix.arch }} - name: Read tool versions from Makefile uses: ./.github/actions/read-tool-versions - name: Cache controller tools uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: controller/bin key: controller-tools-${{ matrix.arch }}-kind${{ env.KIND_VERSION }}-kustomize${{ env.KUSTOMIZE_VERSION }}-grpcurl${{ env.GRPCURL_VERSION }} - name: Setup e2e test environment run: make e2e-setup env: CI: true SKIP_BUILD: "true" PREBUILT_WHEELS_DIR: /tmp/python-wheels OPERATOR_IMG: quay.io/jumpstarter-dev/jumpstarter-operator:latest - name: Setup qemu-ssh host (SSH + rootful Podman on runner) run: bash e2e/scripts/setup-qemu-ssh-e2e.sh env: CI: true - name: Run ExporterSet QEMU-SSH e2e tests run: make e2e-exporterset-qemu-ssh env: CI: true - name: Cleanup qemu-ssh host resources if: always() run: bash e2e/scripts/setup-qemu-ssh-e2e.sh --cleanup - name: Collect and upload e2e logs if: always() uses: ./.github/actions/collect-e2e-logs with: artifact-name: e2e-logs-qemu-ssh-${{ matrix.arch }} # ============================================================================ # Compatibility tests: cross-version interop between controller and client/exporter # ============================================================================ e2e-compat-old-controller: needs: changes # Skip on PRs — compat tests run in merge queue and workflow_dispatch. if: >- github.event_name != 'pull_request' && (needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-26.04 timeout-minutes: 60 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - id: uv run: echo "version=$(cat .uv-version)" >> "$GITHUB_OUTPUT" - name: Install uv uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: version: ${{ steps.uv.outputs.version }} python-version-file: .py-version - name: Install Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: .go-version cache-dependency-path: "**/go.sum" - name: Read tool versions from Makefile uses: ./.github/actions/read-tool-versions - name: Cache controller tools uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: controller/bin key: controller-tools-amd64-kind${{ env.KIND_VERSION }}-kustomize${{ env.KUSTOMIZE_VERSION }}-grpcurl${{ env.GRPCURL_VERSION }} - name: Setup compat environment (old controller) run: make e2e-compat-setup COMPAT_SCENARIO=old-controller env: CI: true # renovate: datasource=github-releases depName=jumpstarter-dev/jumpstarter COMPAT_CONTROLLER_TAG: v0.9.0 - name: Run compat tests (old controller + new client/exporter) run: make e2e-compat-run COMPAT_TEST=old-controller env: CI: true - name: Collect and upload e2e logs if: always() uses: ./.github/actions/collect-e2e-logs with: artifact-name: e2e-logs-compat-old-controller e2e-compat-old-client: needs: [changes, build-controller-image, build-operator-image, build-exporterset-controller-image, build-exporter-image, build-qemu-runtime-image, build-python-wheels] # Skip on PRs — compat tests run in merge queue and workflow_dispatch. if: >- github.event_name != 'pull_request' && (needs.changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-26.04 timeout-minutes: 60 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - id: uv run: echo "version=$(cat .uv-version)" >> "$GITHUB_OUTPUT" - name: Install uv uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: version: ${{ steps.uv.outputs.version }} python-version-file: .py-version - name: Install Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: .go-version cache-dependency-path: "**/go.sum" - name: Load e2e artifacts uses: ./.github/actions/load-e2e-artifacts with: arch: amd64 - name: Read tool versions from Makefile uses: ./.github/actions/read-tool-versions - name: Cache controller tools uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: controller/bin key: controller-tools-amd64-kind${{ env.KIND_VERSION }}-kustomize${{ env.KUSTOMIZE_VERSION }}-grpcurl${{ env.GRPCURL_VERSION }} - name: Setup compat environment (old client) run: make e2e-compat-setup COMPAT_SCENARIO=old-client env: CI: true # renovate: datasource=pypi depName=jumpstarter COMPAT_CLIENT_VERSION: "0.9.0" SKIP_BUILD: "true" PREBUILT_WHEELS_DIR: /tmp/python-wheels OPERATOR_IMG: quay.io/jumpstarter-dev/jumpstarter-operator:latest - name: Run compat tests (new controller + old client/exporter) run: make e2e-compat-run COMPAT_TEST=old-client env: CI: true - name: Collect and upload e2e logs if: always() uses: ./.github/actions/collect-e2e-logs with: artifact-name: e2e-logs-compat-old-client # https://github.com/orgs/community/discussions/26822 e2e: runs-on: ubuntu-latest needs: [changes, e2e-tests, e2e-exporterset-qemu-ssh, e2e-compat-old-controller, e2e-compat-old-client] if: ${{ always() }} steps: - run: exit 1 # Fail on failures or cancellations. Also fail if e2e-tests or # e2e-exporterset-qemu-ssh was skipped when it should have run # (compat jobs are intentionally skipped on PRs). if: >- ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || (needs.e2e-tests.result == 'skipped' && needs.changes.outputs.should_run == 'true') || (needs.e2e-exporterset-qemu-ssh.result == 'skipped' && needs.changes.outputs.should_run == 'true') }}