# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # Manual PR runs bind the selected head or base SHA independently from reusable image publication. name: E2E / Main and Manual Suite run-name: "${{ inputs.checkout_sha != '' && format('E2E PR #{0} ({1})', inputs.pr_number, inputs.correlation_id) || inputs.correlation_id != '' && inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && !inputs.allow_jetson_dispatch && format('E2E full {0} ({1})', github.ref_name, inputs.correlation_id) || inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && !inputs.allow_jetson_dispatch && format('E2E full {0}', github.ref_name) || inputs.correlation_id != '' && format('E2E {0} ({1})', github.ref_name, inputs.correlation_id) || format('E2E {0}', github.ref_name) }}" on: push: branches: [main] workflow_dispatch: inputs: targets: description: "Optional comma-separated typed target IDs. Empty jobs and targets select all default registry E2E tests. Manual PR runs require a source branch in NVIDIA/NemoClaw." required: false default: "" type: string jobs: description: "Optional comma-separated E2E test IDs. Empty selectors choose the default suite. Set include_staging_brev_launchable for full Launchable E2E, or select staging-brev-launchable-identity for the explicit identity smoke run. Jetson dispatch and DGX Spark require their opt-in flags. Manual PR runs require a source branch in NVIDIA/NemoClaw." required: false default: "" type: string include_staging_brev_launchable: description: "Include staging Brev Launchable in a full E2E run when jobs and targets are empty." required: false default: false type: boolean inference_mode: description: "Inference adapter mode for compatible Vitest E2E jobs: mock, internal-nvidia, or public-nvidia." required: false default: "mock" type: choice options: - mock - internal-nvidia - public-nvidia gateway_runtime: description: "Compatibility input for a single gateway runtime. Prefer gateway_runtimes for parallel runtime coverage." required: false default: "docker" type: choice options: - docker - podman gateway_runtimes: description: "Comma-separated managed gateway runtimes to execute in parallel, for example docker,podman." required: false default: "" type: string allow_jetson_dispatch: description: "Set true for a manual jetson-nvmap-gpu run. Main pushes dispatch it automatically. The operator-owned dispatch backend must be available, and JETSON_DISPATCH_URL must contain its verified HTTPS origin. Refer to test/e2e/docs/jetson-dispatch.md." required: false default: false type: boolean allow_dgx_spark_runner_queue: description: "Allow an explicit llama-cpp-dgx-spark-qualification dispatch to queue the protected DGX Spark runner and approval environment." required: false default: false type: boolean pr_number: description: Optional current PR number for manual exact-revision E2E. required: false type: string default: "" post_to_slack: description: Post a selective-dispatch scorecard to the preview Slack route. required: false default: false type: boolean checkout_sha: description: Optional lowercase 40-character PR head or exact PR base SHA for manual E2E. required: false default: "" type: string checkout_repository: description: Source repository for manual PR E2E; must be NVIDIA/NemoClaw. required: false default: "" type: string base_sha: description: Optional lowercase PR base SHA for manual exact-revision E2E. required: false default: "" type: string workflow_sha: description: Optional trusted main workflow SHA for manual E2E. required: false default: "" type: string managed_image_revision: description: Optional ancestor revision whose successful immutable PR managed-image contracts should be reused. required: false default: "" type: string correlation_id: description: Run correlation ID (UUIDv4). required: false default: "" type: string permissions: checks: read contents: read pull-requests: read concurrency: group: e2e-${{ github.ref }}-${{ inputs.jobs == 'llama-cpp-dgx-spark-qualification' && format('spark-{0}', github.run_id) || (inputs.jobs == 'staging-brev-launchable' || inputs.jobs == 'staging-brev-launchable-identity' || inputs.include_staging_brev_launchable) && format('launchable-{0}', github.run_id) || inputs.checkout_sha != '' && format('pr-{0}', inputs.pr_number) || inputs.targets || 'supported' }}-${{ inputs.checkout_sha != '' && 'manual-pr' || inputs.jobs || 'all-jobs' }} cancel-in-progress: ${{ inputs.checkout_sha != '' && inputs.jobs != 'llama-cpp-dgx-spark-qualification' && !inputs.allow_jetson_dispatch && !contains(format(',{0},{1},', inputs.jobs, inputs.targets), ',dgx-station-express,') && !contains(format(',{0},', inputs.jobs), ',staging-brev-launchable,') && !contains(format(',{0},', inputs.jobs), ',staging-brev-launchable-identity,') && !inputs.include_staging_brev_launchable }} env: NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} NEMOCLAW_E2E_CORRELATION_ID: ${{ inputs.correlation_id }} NEMOCLAW_E2E_SHARD: default NEMOCLAW_GATEWAY_RUNTIMES: ${{ inputs.gateway_runtimes || inputs.gateway_runtime || 'docker' }} jobs: package-openshell-sdk: if: ${{ inputs.checkout_sha == '' || (inputs.jobs != 'native-runtime-qualification-producer' && inputs.jobs != 'llama-cpp-dgx-spark-qualification') || inputs.targets != '' }} runs-on: ubuntu-latest timeout-minutes: 5 permissions: contents: read packages: read outputs: artifact_name: ${{ steps.identity.outputs.artifact_name }} steps: - name: Check out trusted OpenShell SDK package verifier uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.workflow_sha }} persist-credentials: false sparse-checkout: | .github/actions/setup-reviewed-npm ci/reviewed-npm-audit.json scripts/audit-reviewed-npm-graph.mts scripts/lib/bundled-npm-package.mts scripts/lib/patch-bundled-npm-undici.mts scripts/lib/remediate-openclaw-runtime-dependencies.mts scripts/lib/reviewed-npm-identity.mts scripts/checks/package-openshell-sdk-for-pr.mts scripts/lib/npm-diagnostics.sh scripts/lib/openclaw-npm-remediation.mts scripts/lib/reviewed-npm-archive.mts scripts/lib/reviewed-npm-audit.mts scripts/lib/npm-audit-receipt.mts scripts/lib/repository-input-path.mts sparse-checkout-cone-mode: false - name: Set up Node for reviewed package download uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "24.22.1" registry-url: https://npm.pkg.github.com scope: "@nvidia" - name: Install reviewed npm uses: ./.github/actions/setup-reviewed-npm - id: package name: Download and verify reviewed OpenShell SDK packages env: NEMOCLAW_OPEN_SHELL_SDK_OUTPUT_DIRECTORY: ${{ runner.temp }}/openshell-sdk NEMOCLAW_OPEN_SHELL_SDK_INCLUDE_AVAILABLE_REPLACEMENT: "1" NODE_AUTH_TOKEN: ${{ github.token }} run: | set -euo pipefail artifact_path="$(node scripts/checks/package-openshell-sdk-for-pr.mts)" test -n "$artifact_path" printf 'artifact_path=%s\n' "$artifact_path" >> "$GITHUB_OUTPUT" - id: identity name: Record reviewed OpenShell SDK artifact identity env: RUN_ATTEMPT: ${{ github.run_attempt }} RUN_ID: ${{ github.run_id }} run: | set -euo pipefail artifact_name="openshell-sdk-e2e-${RUN_ID}-${RUN_ATTEMPT}" printf 'artifact_name=%s\n' "$artifact_name" >> "$GITHUB_OUTPUT" - name: Upload reviewed OpenShell SDK archive uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ steps.identity.outputs.artifact_name }} path: ${{ steps.package.outputs.artifact_path }} if-no-files-found: error retention-days: 1 base-image-publication: needs: generate-matrix if: ${{ needs.generate-matrix.outputs.managed_image_required != 'false' }} runs-on: ubuntu-latest timeout-minutes: 55 outputs: dcode_base_contract: ${{ steps.validate_dcode_base.outputs.contract }} dcode_base_ref: ${{ steps.validate_dcode_base.outputs.base_ref }} managed_image_receipt: ${{ steps.validate_managed_cohort.outputs.receipt }} managed_image_revision: ${{ steps.validate_managed_cohort.outputs.revision }} managed_image_catalog: ${{ steps.select_pr_source.outputs.catalog }} workload_source: managed-image permissions: actions: read contents: read pull-requests: read steps: - id: publication_mode name: Classify base-image publication requirement env: CHECKOUT_SHA: ${{ inputs.checkout_sha }} BASE_SHA: ${{ inputs.base_sha }} EVENT_NAME: ${{ github.event_name }} REF: ${{ github.ref }} REPOSITORY: ${{ github.repository }} WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash run: | set -euo pipefail case "${REPOSITORY}:${REF}:${EVENT_NAME}:${CHECKOUT_SHA:+controller}" in NVIDIA/NemoClaw:refs/heads/main:push:) expected_sha="$WORKFLOW_SHA" allow_non_head=0 select_nearest_successful=0 ;; NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:) expected_sha="$WORKFLOW_SHA" allow_non_head=0 select_nearest_successful=1 ;; NVIDIA/NemoClaw:refs/heads/*:workflow_dispatch:controller) [[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::manual PR publication selection requires an exact base SHA" >&2 exit 1 } expected_sha="$BASE_SHA" allow_non_head=1 select_nearest_successful=1 ;; *) echo "::error::base-image publication mode is not trusted" >&2 exit 1 ;; esac printf 'allow_non_head=%s\n' "${allow_non_head}" >> "${GITHUB_OUTPUT}" printf 'expected_sha=%s\n' "${expected_sha}" >> "${GITHUB_OUTPUT}" printf 'select_nearest_successful=%s\n' "${select_nearest_successful}" >> "${GITHUB_OUTPUT}" - name: Check out trusted E2E workflow uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.workflow_sha }} fetch-depth: 0 persist-credentials: false - name: Set up Node for publication verification uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24.22.1 - name: Install reviewed npm uses: ./.github/actions/setup-reviewed-npm - id: select_pr_source name: Resolve exact PR managed-image publication if: ${{ inputs.pr_number != '' }} env: BASE_SHA: ${{ inputs.base_sha }} CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository }} CANDIDATE_SHA: ${{ inputs.checkout_sha }} GITHUB_TOKEN: ${{ github.token }} MANAGED_IMAGE_SHA: ${{ inputs.managed_image_revision }} PR_NUMBER: ${{ inputs.pr_number }} shell: bash run: | set -euo pipefail catalog_path="${RUNNER_TEMP}/pr-managed-image-catalog.json" rm -f -- "$catalog_path" if [[ -n "$MANAGED_IMAGE_SHA" ]]; then [[ "$MANAGED_IMAGE_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::managed_image_revision must be a lowercase 40-character SHA" >&2 exit 1 } git merge-base --is-ancestor "$MANAGED_IMAGE_SHA" "$CANDIDATE_SHA" || { echo "::error::managed_image_revision must be an ancestor of checkout_sha" >&2 exit 1 } fi selection="$(node --no-warnings tools/e2e/pr-managed-image-publication.mts "$catalog_path")" case "$selection" in base-cohort) [[ ! -e "$catalog_path" && ! -L "$catalog_path" ]] || { echo "::error::base-cohort selection produced a candidate catalog" >&2 exit 1 } ;; candidate-catalog) [[ -f "$catalog_path" && ! -L "$catalog_path" && -s "$catalog_path" ]] || { echo "::error::exact PR managed-image catalog is invalid" >&2 exit 1 } catalog="$(jq -ce . "$catalog_path")" (( ${#catalog} <= 65536 )) || { echo "::error::exact PR managed-image catalog exceeds the output limit" >&2 exit 1 } printf 'catalog=%s\n' "$catalog" >>"$GITHUB_OUTPUT" echo "::notice::Jetson dispatch is unavailable because the exact PR managed-image catalog qualifies linux/amd64 only." ;; *) echo "::error::PR managed-image selection is invalid" >&2; exit 1 ;; esac printf 'selection=%s\n' "$selection" >>"$GITHUB_OUTPUT" - id: publication name: Select base and optional managed-image publication if: ${{ inputs.pr_number == '' || steps.select_pr_source.outputs.selection == 'base-cohort' || inputs.jobs != '' || inputs.targets == '' || contains(inputs.targets, 'managed-image-') }} env: EXPECTED_SHA: ${{ steps.publication_mode.outputs.expected_sha }} GITHUB_TOKEN: ${{ github.token }} PUBLICATION_HISTORY_ALLOW_NON_HEAD: ${{ steps.publication_mode.outputs.allow_non_head }} REQUIRE_MANAGED_IMAGE_PUBLICATION: ${{ steps.select_pr_source.outputs.selection == 'candidate-catalog' && '0' || '1' }} SELECT_NEAREST_SUCCESSFUL_PUBLICATION: ${{ steps.publication_mode.outputs.select_nearest_successful }} shell: bash run: | set -euo pipefail export GITHUB_REF=refs/heads/main export GITHUB_SHA="$EXPECTED_SHA" wait_seconds=3000 if [[ "$PUBLICATION_HISTORY_ALLOW_NON_HEAD" == "1" ]]; then wait_seconds=300 fi node --no-warnings tools/e2e/base-image-publication.mts --wait-seconds "$wait_seconds" --poll-seconds 30 - name: Download immutable Deep Agents Code base contract if: ${{ inputs.pr_number == '' || steps.select_pr_source.outputs.selection == 'base-cohort' || inputs.jobs != '' || inputs.targets == '' || contains(inputs.targets, 'managed-image-') }} env: GITHUB_TOKEN: ${{ github.token }} PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }} PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }} PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }} run: node --no-warnings tools/e2e/exact-artifact-download.mts "${RUNNER_TEMP}/dcode-base-contract" - id: validate_dcode_base name: Validate immutable Deep Agents Code base if: ${{ inputs.pr_number == '' || steps.select_pr_source.outputs.selection == 'base-cohort' || inputs.jobs != '' || inputs.targets == '' || contains(inputs.targets, 'managed-image-') }} env: PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }} PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }} PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }} run: node --no-warnings tools/e2e/dcode-base-image-contract.mts "${RUNNER_TEMP}/dcode-base-contract/contract.json" - id: download_managed_cohort name: Download immutable managed-image cohort contract if: ${{ inputs.pr_number == '' || steps.select_pr_source.outputs.selection == 'base-cohort' }} env: GITHUB_TOKEN: ${{ github.token }} PUBLICATION_ARTIFACT_KIND: managed-image-cohort PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }} PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }} PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }} run: node --no-warnings tools/e2e/exact-artifact-download.mts "${RUNNER_TEMP}/managed-image-cohort" - id: validate_managed_cohort name: Validate immutable managed-image cohort contract if: ${{ inputs.pr_number == '' || steps.select_pr_source.outputs.selection == 'base-cohort' }} env: PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }} PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }} PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }} run: node --no-warnings tools/e2e/managed-image-cohort-contract.mts "${RUNNER_TEMP}/managed-image-cohort/cohort.json" generate-matrix: needs: package-openshell-sdk if: ${{ !cancelled() && (needs.package-openshell-sdk.result == 'success' || (needs.package-openshell-sdk.result == 'skipped' && inputs.checkout_sha != '' && (inputs.jobs == 'native-runtime-qualification-producer' || inputs.jobs == 'llama-cpp-dgx-spark-qualification') && inputs.targets == '')) }} runs-on: ubuntu-latest timeout-minutes: 10 permissions: actions: read contents: read pull-requests: read outputs: cli_artifact_provenance: ${{ steps.record_cli_artifact.outputs.provenance }} workload_source: managed-image e2e_credentials_allowed: ${{ steps.e2e_credentials.outputs.allowed }} matrix: ${{ steps.matrix.outputs.matrix }} test_matrix: ${{ steps.matrix.outputs.test_matrix }} hermes_selected: ${{ steps.matrix.outputs.hermes_selected }} explicit_only_jobs: ${{ steps.matrix.outputs.explicit_only_jobs }} release_required_jobs: ${{ steps.matrix.outputs.release_required_jobs }} selected_jobs: ${{ steps.matrix.outputs.selected_jobs }} selected_workflow_jobs: ${{ steps.matrix.outputs.selected_workflow_jobs }} managed_image_required: ${{ steps.matrix.outputs.managed_image_required }} openshell_version: ${{ steps.openshell_version.outputs.version }} openshell_supervisor_image: ${{ steps.openshell_version.outputs.supervisor_image }} openshell_runtime_provenance: ${{ steps.openshell_version.outputs.runtime_provenance }} catalogue_standard_matrix: ${{ steps.matrix.outputs.catalogue_standard_matrix }} catalogue_nvidia_api_matrix: ${{ steps.matrix.outputs.catalogue_nvidia_api_matrix }} catalogue_nvidia_inference_matrix: ${{ steps.matrix.outputs.catalogue_nvidia_inference_matrix }} catalogue_hosted_inference_matrix: ${{ steps.matrix.outputs.catalogue_hosted_inference_matrix }} catalogue_github_read_matrix: ${{ steps.matrix.outputs.catalogue_github_read_matrix }} gateway_runtimes: ${{ steps.matrix.outputs.gateway_runtimes }} runtime_providers_by_job: ${{ steps.matrix.outputs.runtime_providers_by_job }} runner_routing: ${{ steps.runner_routing.outputs.runner_routing }} steps: - id: runner_routing name: Build trusted larger-runner routing env: CHECKOUT_SHA: ${{ inputs.checkout_sha }} LARGER_RUNNER_LABEL: ${{ vars.E2E_LARGER_RUNNER_LABEL }} REF: ${{ github.ref }} REPOSITORY: ${{ github.repository }} shell: bash run: | set -euo pipefail larger_runner="ubuntu-latest" if [[ "${REPOSITORY}" == "NVIDIA/NemoClaw" && "${REF}" == "refs/heads/main" && -z "${CHECKOUT_SHA}" && -n "${LARGER_RUNNER_LABEL}" ]]; then if [[ ! "${LARGER_RUNNER_LABEL}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]; then echo "::error::E2E_LARGER_RUNNER_LABEL must be a 1-64 character workflow label using letters, digits, dots, underscores, or hyphens" >&2 exit 1 fi larger_runner="${LARGER_RUNNER_LABEL}" fi runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" '{"channels-stop-start-hermes":$larger,"common-egress-agent":$larger,"hermes-discord":$larger,"hermes-e2e":$larger,"hermes-inference-switch":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"security-posture-hermes":$larger}')" printf 'runner_routing=%s\n' "${runner_routing}" >> "${GITHUB_OUTPUT}" - id: candidate_authorization name: Authenticate manual PR dispatch if: ${{ inputs.pr_number != '' || inputs.checkout_sha != '' || inputs.checkout_repository != '' || inputs.base_sha != '' || inputs.workflow_sha != '' }} env: ALLOW_JETSON_DISPATCH: ${{ inputs.allow_jetson_dispatch && 'true' || 'false' }} BASE_SHA: ${{ inputs.base_sha }} CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }} CHECKOUT_SHA: ${{ inputs.checkout_sha }} EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} GITHUB_TOKEN: ${{ github.token }} INCLUDE_LAUNCHABLE: ${{ inputs.include_staging_brev_launchable && 'true' || 'false' }} JOBS: ${{ inputs.jobs }} PR_NUMBER: ${{ inputs.pr_number }} TARGETS: ${{ inputs.targets }} WORKFLOW_EVENT: ${{ github.event_name }} WORKFLOW_REF: ${{ github.ref }} WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash run: | set -euo pipefail [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == refs/heads/* ]] || { echo "::error::Manual PR E2E must be dispatched from this repository branch" >&2 exit 1 } [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || { echo "::error::pr_number must be a positive integer" >&2; exit 1; } [[ "$CHECKOUT_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || { echo "::error::checkout_repository must be an owner/repository name" >&2; exit 1; } [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::checkout_sha must be a lowercase 40-character SHA" >&2; exit 1; } [[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::base_sha must be a lowercase 40-character SHA" >&2; exit 1; } [[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" ]] || { echo "::error::workflow_sha must match the trusted main workflow SHA" >&2; exit 1; } [[ -n "$GITHUB_TOKEN" ]] || { echo "::error::GitHub token is required for PR identity validation" >&2; exit 1; } pull_json="$(curl --fail --silent --show-error --proto '=https' \ --header "Authorization: Bearer ${GITHUB_TOKEN}" \ --header "Accept: application/vnd.github+json" \ --header "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must be open" >&2; exit 1; } [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]] || { echo "::error::pull request base repository must be NVIDIA/NemoClaw" >&2; exit 1; } [[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]] || { echo "::error::pull request base branch must be main" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha must match the PR base SHA" >&2; exit 1; } if [[ "$CHECKOUT_SHA" == "$BASE_SHA" ]]; then [[ "$ALLOW_JETSON_DISPATCH" != "true" ]] || { echo "::error::exact-base E2E cannot launch dedicated hardware dispatches" >&2; exit 1; } [[ ",${TARGETS}," != *",jetson-nvmap-gpu,"* && ",${JOBS}," != *",jetson-nvmap-gpu,"* ]] || { echo "::error::exact-base E2E cannot select dedicated hardware jobs" >&2; exit 1; } [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR base repository" >&2; exit 1; } else [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR source repository" >&2; exit 1; } [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the latest PR commit SHA" >&2; exit 1; } fi nvidia_owned=false if [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" && "$(jq -r '.head.repo.owner.login // ""' <<< "$pull_json")" == "NVIDIA" && "$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization" ]]; then nvidia_owned=true else echo "::error::Manual PR E2E requires a source branch in NVIDIA/NemoClaw. Review and adopt fork contributions onto a repository branch first." >&2 exit 1 fi if [[ ",${JOBS}," == *",staging-brev-launchable-identity,"* ]]; then echo "::error::Launchable identity smoke runs only against trusted main" >&2 exit 1 fi if [[ "$INCLUDE_LAUNCHABLE" == "true" || ",${JOBS}," == *",staging-brev-launchable,"* ]]; then [[ "$nvidia_owned" == "true" ]] || { echo "::error::Launchable PR E2E requires an NVIDIA-owned source repository" >&2; exit 1; } [[ "$CHECKOUT_REPOSITORY" == "NVIDIA/NemoClaw" ]] || { echo "::error::Launchable PR E2E requires a branch in NVIDIA/NemoClaw" >&2; exit 1; } fi printf 'nvidia_owned=%s\n' "$nvidia_owned" >> "$GITHUB_OUTPUT" - name: Record trusted E2E dispatch receipt if: ${{ github.event_name == 'workflow_dispatch' }} env: ACTOR: ${{ github.actor }} ALLOW_JETSON_DISPATCH: ${{ inputs.allow_jetson_dispatch && 'true' || 'false' }} ALLOW_DGX_SPARK_RUNNER_QUEUE: ${{ inputs.allow_dgx_spark_runner_queue && 'true' || 'false' }} ALLOW_JETSON_RUNNER_QUEUE: "false" BASE_SHA: ${{ inputs.checkout_sha != '' && inputs.base_sha || github.sha }} CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository || github.repository }} CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} DISPATCH_JOBS: ${{ inputs.jobs }} DISPATCH_RECEIPT_DIR: ${{ runner.temp }}/nemoclaw-e2e-dispatch DISPATCH_TARGETS: ${{ inputs.targets }} EVENT_NAME: ${{ github.event_name }} INCLUDE_STAGING_BREV_LAUNCHABLE: ${{ inputs.include_staging_brev_launchable && 'true' || 'false' }} PR_NUMBER: ${{ inputs.checkout_sha != '' && inputs.pr_number || '' }} REPOSITORY: ${{ github.repository }} RUN_ATTEMPT: ${{ github.run_attempt }} RUN_ID: ${{ github.run_id }} TRIGGERING_ACTOR: ${{ github.triggering_actor }} WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash run: | set -euo pipefail install -d -m 0700 "$DISPATCH_RECEIPT_DIR" jq -n \ --arg actor "$ACTOR" \ --arg baseSha "$BASE_SHA" \ --arg candidateRepository "$CANDIDATE_REPOSITORY" \ --arg candidateSha "$CANDIDATE_SHA" \ --arg eventName "$EVENT_NAME" \ --arg jobs "$DISPATCH_JOBS" \ --arg prNumber "$PR_NUMBER" \ --arg repository "$REPOSITORY" \ --arg targets "$DISPATCH_TARGETS" \ --arg triggeringActor "$TRIGGERING_ACTOR" \ --arg workflowRunId "$RUN_ID" \ --arg workflowSha "$WORKFLOW_SHA" \ --argjson allowJetsonDispatch "$ALLOW_JETSON_DISPATCH" \ --argjson allowDgxSparkRunnerQueue "$ALLOW_DGX_SPARK_RUNNER_QUEUE" \ --argjson allowJetsonRunnerQueue "$ALLOW_JETSON_RUNNER_QUEUE" \ --argjson includeStagingBrevLaunchable "$INCLUDE_STAGING_BREV_LAUNCHABLE" \ --argjson workflowRunAttempt "$RUN_ATTEMPT" \ '{ kind: "nemoclaw-e2e-dispatch-v2", actor: $actor, repository: $repository, prNumber: (if $prNumber == "" then null else ($prNumber | tonumber) end), candidateRepository: $candidateRepository, candidateSha: $candidateSha, baseSha: $baseSha, workflowSha: $workflowSha, workflowRunId: $workflowRunId, workflowRunAttempt: $workflowRunAttempt, eventName: $eventName, jobs: $jobs, targets: $targets, allowDgxSparkRunnerQueue: $allowDgxSparkRunnerQueue, allowJetsonDispatch: $allowJetsonDispatch, allowJetsonRunnerQueue: $allowJetsonRunnerQueue, includeStagingBrevLaunchable: $includeStagingBrevLaunchable, triggeringActor: $triggeringActor, emptySelectors: ($jobs == "" and $targets == "") }' >"$DISPATCH_RECEIPT_DIR/dispatch.json" - name: Upload trusted E2E dispatch receipt if: ${{ github.event_name == 'workflow_dispatch' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-dispatch-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/nemoclaw-e2e-dispatch/dispatch.json - name: Authorize Launchable E2E maintainer dispatch if: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.jobs == 'staging-brev-launchable-identity' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }} env: ACTOR: ${{ github.actor }} GITHUB_TOKEN: ${{ github.token }} TRIGGERING_ACTOR: ${{ github.triggering_actor }} shell: bash run: | set -euo pipefail read_collaborator_permission() { local maintainer="$1" local attempt curl_exit failure http_status permission_file permission_file="$(mktemp "${RUNNER_TEMP:-/tmp}/nemoclaw-collaborator-permission.XXXXXX")" for attempt in 1 2 3; do : >"$permission_file" if http_status="$(curl --silent --proto '=https' --connect-timeout 10 --max-time 30 \ --output "$permission_file" --write-out "%{http_code}" \ --header "Authorization: Bearer ${GITHUB_TOKEN}" \ --header "Accept: application/vnd.github+json" \ --header "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${maintainer}/permission" \ 2>/dev/null)"; then if [[ "$http_status" =~ ^2[0-9]{2}$ ]]; then if jq -e 'type == "object" and (.user.login | type == "string") and (.role_name | type == "string")' "$permission_file" >/dev/null 2>&1; then if (( attempt > 1 )); then echo "::notice::Collaborator permission read passed after retry on attempt ${attempt}/3" >&2 fi cat "$permission_file" rm -f "$permission_file" return 0 fi echo "::error::Collaborator permission read attempt ${attempt}/3 failed: malformed response" >&2 rm -f "$permission_file" return 1 fi if [[ "$http_status" =~ ^[0-9]{3}$ ]]; then failure="HTTP ${http_status}" case "$http_status" in 408 | 429 | 5??) ;; *) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: ${failure}" >&2; rm -f "$permission_file"; return 1 ;; esac else echo "::error::Collaborator permission read attempt ${attempt}/3 failed: invalid HTTP status" >&2 rm -f "$permission_file" return 1 fi else curl_exit=$? case "$curl_exit" in 5 | 6 | 7 | 16 | 18 | 28 | 35 | 52 | 55 | 56 | 92 | 95 | 96) failure="transport" ;; *) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: curl exit ${curl_exit}" >&2; rm -f "$permission_file"; return 1 ;; esac fi if (( attempt == 3 )); then echo "::error::Collaborator permission read exhausted after attempt ${attempt}/3: ${failure}" >&2 rm -f "$permission_file" return 1 fi echo "::warning::Collaborator permission read attempt ${attempt}/3 failed: ${failure}; retrying" >&2 sleep "$attempt" done } require_maintainer() { local maintainer="$1" if [[ ! "$maintainer" =~ ^[A-Za-z0-9-]{1,39}$ || "$maintainer" == -* || "$maintainer" == *- ]]; then echo "::error::Launchable E2E actor is invalid" >&2 exit 1 fi local permission_json permission_json="$(read_collaborator_permission "$maintainer")" if [[ "$(jq -r '.user.login // ""' <<< "$permission_json" | tr '[:upper:]' '[:lower:]')" != "$(tr '[:upper:]' '[:lower:]' <<< "$maintainer")" ]]; then echo "::error::Launchable E2E permission response did not match the actor" >&2 exit 1 fi case "$(jq -r '.role_name // ""' <<< "$permission_json")" in maintain | admin) ;; *) echo "::error::Launchable E2E requires a repository maintainer or administrator" >&2 exit 1 ;; esac } require_maintainer "$ACTOR" if [[ "$(printf '%s' "$TRIGGERING_ACTOR" | tr '[:upper:]' '[:lower:]')" != "$(printf '%s' "$ACTOR" | tr '[:upper:]' '[:lower:]')" ]]; then require_maintainer "$TRIGGERING_ACTOR" fi - name: Check out trusted E2E planner uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ github.repository }} ref: ${{ github.workflow_sha }} fetch-depth: 0 persist-credentials: false - name: Set up Node for trusted E2E planning uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24.22.1 - name: Install reviewed npm for trusted E2E planning uses: NVIDIA/NemoClaw/.github/actions/setup-reviewed-npm@98669f24d35f18e49b6b2769cd68709509ea24f2 - name: Install trusted E2E planner dependencies run: npm ci --ignore-scripts --no-audit --no-fund - id: matrix name: Generate E2E target matrix env: NEMOCLAW_E2E_INCLUDE_STAGING_BREV_LAUNCHABLE: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.include_staging_brev_launchable && 'true' || 'false' }} INFERENCE_MODE: ${{ inputs.inference_mode || 'mock' }} NEMOCLAW_GATEWAY_RUNTIMES: ${{ inputs.jobs == 'portable-hermes-finalization' && 'podman' || inputs.gateway_runtimes || inputs.gateway_runtime || 'docker' }} JOBS: ${{ inputs.jobs }} TARGETS: ${{ inputs.targets }} EVENT_NAME: ${{ github.event_name }} BEFORE_SHA: ${{ github.event.before }} CANDIDATE_SHA: ${{ github.sha }} NEMOCLAW_E2E_CREDENTIALS_ALLOWED: ${{ (inputs.checkout_sha == '' || steps.candidate_authorization.outputs.nvidia_owned == 'true') && 'true' || 'false' }} run: | set -euo pipefail if [[ "${JOBS:-}" == "llama-cpp-dgx-spark-qualification" && -z "${TARGETS:-}" ]]; then { echo 'matrix=[]' echo 'test_matrix=[]' echo 'catalogue_standard_matrix=[]' echo 'catalogue_nvidia_api_matrix=[]' echo 'catalogue_nvidia_inference_matrix=[]' echo 'catalogue_hosted_inference_matrix=[]' echo 'catalogue_github_read_matrix=[]' echo 'gateway_runtimes=["docker"]' echo 'runtime_providers_by_job={"llama-cpp-dgx-spark-qualification":["docker"]}' echo 'selected_jobs=["llama-cpp-dgx-spark-qualification"]' echo 'selected_workflow_jobs=["llama-cpp-dgx-spark-qualification"]' echo 'hermes_selected=false' echo 'explicit_only_jobs=llama-cpp-dgx-spark-qualification' echo 'release_required_jobs=[]' echo 'managed_image_required=false' } >>"$GITHUB_OUTPUT" exit 0 fi if [[ "${JOBS:-}" == "native-runtime-qualification-producer" && -z "${TARGETS:-}" ]]; then { echo 'matrix=[]' echo 'test_matrix=[]' echo 'catalogue_standard_matrix=[]' echo 'catalogue_nvidia_api_matrix=[]' echo 'catalogue_nvidia_inference_matrix=[]' echo 'catalogue_hosted_inference_matrix=[]' echo 'catalogue_github_read_matrix=[]' echo 'gateway_runtimes=["docker"]' echo 'runtime_providers_by_job={"native-runtime-qualification-producer":["docker"]}' echo 'selected_jobs=["native-runtime-qualification-producer"]' echo 'selected_workflow_jobs=["native-runtime-qualification-producer"]' echo 'hermes_selected=false' echo 'explicit_only_jobs=native-runtime-qualification-producer' echo 'release_required_jobs=[]' echo 'managed_image_required=false' } >>"$GITHUB_OUTPUT" exit 0 fi if [ "${EVENT_NAME:-}" = "push" ]; then [[ "${BEFORE_SHA}" =~ ^[a-f0-9]{40}$ && "${CANDIDATE_SHA}" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::Main push E2E requires exact before and candidate commit SHAs" >&2 exit 1 } CHANGED_FILES="$(git diff --name-only --diff-filter=ACMRD "${BEFORE_SHA}" "${CANDIDATE_SHA}")" export CHANGED_FILES fi npx --no-install tsx tools/e2e/workflow-plan.mts --ci-output - id: openshell_version name: Resolve reviewed candidate OpenShell version if: ${{ contains(fromJSON(steps.matrix.outputs.selected_jobs), 'openshell-gateway-auth-contract') || contains(fromJSON(steps.matrix.outputs.selected_jobs), 'external-gateway-health') || contains(fromJSON(steps.matrix.outputs.selected_jobs), 'mcp-bridge') || contains(fromJSON(steps.matrix.outputs.selected_jobs), 'openshell-credential-generation-window') }} env: MCP_RUNTIME_REQUIRED: ${{ contains(fromJSON(steps.matrix.outputs.selected_jobs), 'mcp-bridge') || contains(fromJSON(steps.matrix.outputs.selected_jobs), 'openshell-credential-generation-window') }} CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository || github.repository }} CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} UPGRADE_PR_NUMBER: ${{ inputs.pr_number }} UPGRADE_BASE_SHA: ${{ inputs.base_sha }} GITHUB_TOKEN: ${{ github.token }} run: node --no-warnings tools/e2e/candidate-openshell-version.mts # Stage the reviewed toolchains before candidate checkout or # candidate-controlled workspace preparation can execute on this runner. - name: Stage immutable native Podman E2E toolchains uses: NVIDIA/NemoClaw/.github/actions/stage-native-podman-e2e-toolchains@19eb13446b3eb5e41cca2e65b44c9b33c963a595 with: enabled: ${{ (contains(format(',{0},', inputs.gateway_runtimes || inputs.gateway_runtime || 'docker'), ',podman,') || contains(fromJSON(steps.matrix.outputs.selected_jobs), 'portable-hermes-finalization')) && 'true' || 'false' }} github-token: ${{ github.token }} - name: Check out E2E candidate uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: ${{ inputs.checkout_sha == '' || (inputs.jobs != 'native-runtime-qualification-producer' && inputs.jobs != 'llama-cpp-dgx-spark-qualification') || inputs.targets != '' }} with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - name: Validate manual PR checkout if: ${{ inputs.checkout_sha != '' && ((inputs.jobs != 'native-runtime-qualification-producer' && inputs.jobs != 'llama-cpp-dgx-spark-qualification') || inputs.targets != '') }} env: BASE_SHA: ${{ inputs.base_sha }} CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }} CHECKOUT_SHA: ${{ inputs.checkout_sha }} GITHUB_TOKEN: ${{ steps.candidate_authorization.outputs.nvidia_owned == 'true' && github.token || '' }} NVIDIA_OWNED: ${{ steps.candidate_authorization.outputs.nvidia_owned }} PR_NUMBER: ${{ inputs.pr_number }} shell: bash run: | set -euo pipefail [[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::checked-out commit does not match checkout_sha" >&2; exit 1; } auth_args=() if [[ "$NVIDIA_OWNED" == "true" ]]; then [[ -n "$GITHUB_TOKEN" ]] || { echo "::error::authenticated NVIDIA-owned checkout validation requires a GitHub token" >&2; exit 1; } auth_args=(--header "Authorization: Bearer ${GITHUB_TOKEN}") fi pull_json="$(curl --fail --silent --show-error --proto '=https' \ "${auth_args[@]}" \ --header "Accept: application/vnd.github+json" \ --header "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must still be open" >&2; exit 1; } [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]] || { echo "::error::pull request base repository changed before execution" >&2; exit 1; } [[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]] || { echo "::error::pull request base branch changed before execution" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha changed before execution" >&2; exit 1; } if [[ "$CHECKOUT_SHA" == "$BASE_SHA" ]]; then [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository changed before base execution" >&2; exit 1; } else [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository changed before head execution" >&2; exit 1; } [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha changed before head execution" >&2; exit 1; } fi if [[ "$NVIDIA_OWNED" == "true" ]]; then [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" && "$(jq -r '.head.repo.owner.login // ""' <<< "$pull_json")" == "NVIDIA" && "$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization" ]] || { echo "::error::PR source repository ownership changed before execution" >&2 exit 1 } fi - id: e2e_credentials name: Authorize E2E credentials if: ${{ inputs.checkout_sha != '' && ((inputs.jobs != 'native-runtime-qualification-producer' && inputs.jobs != 'llama-cpp-dgx-spark-qualification') || inputs.targets != '') }} env: CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }} CHECKOUT_SHA: ${{ inputs.checkout_sha }} EVENT_NAME: ${{ github.event_name }} EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} NVIDIA_OWNED: ${{ steps.candidate_authorization.outputs.nvidia_owned }} REF: ${{ github.ref }} WORKFLOW_REPOSITORY: ${{ github.repository }} WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash run: | set -euo pipefail credentials_allowed=false if [[ "$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw" && "$CHECKOUT_REPOSITORY" == "NVIDIA/NemoClaw" && "$NVIDIA_OWNED" == "true" && "$EVENT_NAME" == "workflow_dispatch" && "$REF" == refs/heads/* && "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" && "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]]; then credentials_allowed=true fi printf 'allowed=%s\n' "$credentials_allowed" >> "$GITHUB_OUTPUT" - name: Check out trusted compiled artifact action if: ${{ inputs.checkout_sha == '' || (inputs.jobs != 'native-runtime-qualification-producer' && inputs.jobs != 'llama-cpp-dgx-spark-qualification') || inputs.targets != '' }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ github.repository }} ref: ${{ github.workflow_sha }} path: .trusted-ci-actions persist-credentials: false sparse-checkout: | .github/actions/ci-compile-artifacts .github/actions/ci-install-dependencies.sh ci/reviewed-npm-audit.json scripts/check-dist-sourcemaps.mts scripts/audit-reviewed-npm-graph.mts scripts/lib/bundled-npm-package.mts scripts/lib/patch-bundled-npm-undici.mts scripts/lib/remediate-openclaw-runtime-dependencies.mts scripts/lib/reviewed-npm-identity.mts scripts/checks/prepare-ci-npm-install.mts scripts/lib/reviewed-ci-npm-install.mts scripts/lib/openclaw-npm-remediation.mts scripts/lib/reviewed-npm-archive.mts scripts/lib/reviewed-npm-cache.mts scripts/lib/reviewed-npm-audit.mts scripts/lib/npm-audit-receipt.mts scripts/lib/repository-input-path.mts sparse-checkout-cone-mode: false - name: Download reviewed OpenShell SDK archive if: ${{ inputs.checkout_sha == '' || (inputs.jobs != 'native-runtime-qualification-producer' && inputs.jobs != 'llama-cpp-dgx-spark-qualification') || inputs.targets != '' }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace if: ${{ inputs.checkout_sha == '' || (inputs.jobs != 'native-runtime-qualification-producer' && inputs.jobs != 'llama-cpp-dgx-spark-qualification') || inputs.targets != '' }} uses: ./.trusted-ci-actions/.github/actions/ci-compile-artifacts with: package-mode: artifact - id: package_cli_artifact name: Package exact-commit CLI if: ${{ inputs.checkout_sha == '' || (inputs.jobs != 'native-runtime-qualification-producer' && inputs.jobs != 'llama-cpp-dgx-spark-qualification') || inputs.targets != '' }} env: CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository || github.repository }} CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} RUN_ATTEMPT: ${{ github.run_attempt }} RUN_ID: ${{ github.run_id }} WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash run: scripts/e2e/package-cli-artifact.sh - id: upload_cli_artifact name: Publish content-addressed CLI artifact if: ${{ inputs.checkout_sha == '' || (inputs.jobs != 'native-runtime-qualification-producer' && inputs.jobs != 'llama-cpp-dgx-spark-qualification') || inputs.targets != '' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ steps.package_cli_artifact.outputs.artifact_name }} path: ${{ runner.temp }}/nemoclaw-cli-artifact/ if-no-files-found: error retention-days: 3 compression-level: 0 - id: record_cli_artifact name: Record CLI artifact provenance if: ${{ inputs.checkout_sha == '' || (inputs.jobs != 'native-runtime-qualification-producer' && inputs.jobs != 'llama-cpp-dgx-spark-qualification') || inputs.targets != '' }} env: ARTIFACT_DIGEST: ${{ steps.upload_cli_artifact.outputs.artifact-digest }} ARTIFACT_ID: ${{ steps.upload_cli_artifact.outputs.artifact-id }} ARTIFACT_NAME: ${{ steps.package_cli_artifact.outputs.artifact_name }} CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository || github.repository }} CANDIDATE_SHA: ${{ steps.package_cli_artifact.outputs.candidate_sha }} PAYLOAD_SHA256: ${{ steps.package_cli_artifact.outputs.payload_sha256 }} RUN_ATTEMPT: ${{ github.run_attempt }} RUN_ID: ${{ github.run_id }} WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash run: | set -euo pipefail [[ "$ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]] || { echo "::error::artifact upload did not return an immutable artifact ID"; exit 1; } [[ "$ARTIFACT_DIGEST" =~ ^[a-f0-9]{64}$ ]] || { echo "::error::artifact upload did not return a SHA-256 digest"; exit 1; } provenance="$(jq -cn \ --arg artifactDigest "$ARTIFACT_DIGEST" \ --arg artifactId "$ARTIFACT_ID" \ --arg artifactName "$ARTIFACT_NAME" \ --arg candidateRepository "$CANDIDATE_REPOSITORY" \ --arg candidateSha "$CANDIDATE_SHA" \ --arg payloadSha256 "$PAYLOAD_SHA256" \ --arg runAttempt "$RUN_ATTEMPT" \ --arg runId "$RUN_ID" \ --arg workflowSha "$WORKFLOW_SHA" \ '{ kind: "nemoclaw-e2e-cli-provenance-v1", artifactDigest: $artifactDigest, artifactId: $artifactId, artifactName: $artifactName, candidateRepository: $candidateRepository, candidateSha: $candidateSha, payloadSha256: $payloadSha256, workflowSha: $workflowSha, runId: $runId, runAttempt: $runAttempt }')" printf 'provenance=%s\n' "$provenance" >>"$GITHUB_OUTPUT" { echo "## Exact-commit CLI artifact" echo echo "- Candidate: \`${CANDIDATE_SHA}\`" echo "- Artifact: \`${ARTIFACT_NAME}\` (ID \`${ARTIFACT_ID}\`)" echo "- GitHub archive digest: \`${ARTIFACT_DIGEST}\`" echo "- Payload digest: \`${PAYLOAD_SHA256}\`" } >>"$GITHUB_STEP_SUMMARY" llama-cpp-dgx-spark-plan: name: Authenticate and compile the DGX Spark qualification plan needs: generate-matrix if: ${{ !cancelled() && needs.generate-matrix.result == 'success' && github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.jobs == 'llama-cpp-dgx-spark-qualification' && inputs.targets == '' }} runs-on: ubuntu-latest timeout-minutes: 10 permissions: contents: read pull-requests: read outputs: runner: ${{ steps.plan.outputs.runner }} environment: ${{ steps.plan.outputs.environment }} plan: ${{ steps.plan.outputs.plan }} plan_sha256: ${{ steps.plan.outputs.plan_sha256 }} model_host_path: ${{ steps.plan.outputs.model_host_path }} steps: - name: Check out trusted Spark plan compiler uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: NVIDIA/NemoClaw ref: ${{ github.workflow_sha }} persist-credentials: false - name: Check out inert Spark candidate data uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: NVIDIA/NemoClaw ref: ${{ inputs.checkout_sha }} path: .candidate-source fetch-depth: 0 persist-credentials: false - name: Authenticate Spark qualification dispatch env: ALLOW_QUEUE: ${{ inputs.allow_dgx_spark_runner_queue && 'true' || 'false' }} BASE_SHA: ${{ inputs.base_sha }} CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }} CHECKOUT_SHA: ${{ inputs.checkout_sha }} EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ inputs.pr_number }} WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash run: | set -euo pipefail [[ "$ALLOW_QUEUE" == true && "$CHECKOUT_REPOSITORY" == NVIDIA/NemoClaw ]] [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] [[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ && "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] [[ "$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" && "$CHECKOUT_SHA" != "$BASE_SHA" ]] [[ "$(git -C .candidate-source rev-parse HEAD)" == "$CHECKOUT_SHA" ]] git -C .candidate-source merge-base --is-ancestor "$BASE_SHA" "$CHECKOUT_SHA" pull="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" jq -e --arg head "$CHECKOUT_SHA" --arg base "$BASE_SHA" ' .state == "open" and .head.repo.full_name == "NVIDIA/NemoClaw" and .head.sha == $head and .base.repo.full_name == "NVIDIA/NemoClaw" and .base.ref == "main" and .base.sha == $base ' <<< "$pull" >/dev/null # Candidate executable build inputs must match the trusted image source. git -C .candidate-source diff --exit-code "$WORKFLOW_SHA" "$CHECKOUT_SHA" -- \ managed-inference/images/llama-cpp ':!managed-inference/images/llama-cpp/image.yaml' - name: Set up Node for Spark plan compilation uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "24.22.1" - name: Install trusted Spark compiler dependencies uses: NVIDIA/NemoClaw/.github/actions/setup-reviewed-npm@98669f24d35f18e49b6b2769cd68709509ea24f2 - name: Install trusted Spark plan dependencies run: npm ci --ignore-scripts --no-audit --no-fund - id: plan name: Export the digest-bound Spark plan run: node --no-warnings scripts/checks/export-llama-cpp-dgx-spark-qualification-plan.mts --source-root "$GITHUB_WORKSPACE/.candidate-source" llama-cpp-dgx-spark-qualification: name: Protected llama.cpp DGX Spark qualification needs: llama-cpp-dgx-spark-plan if: ${{ !cancelled() && needs.llama-cpp-dgx-spark-plan.result == 'success' }} runs-on: ${{ needs.llama-cpp-dgx-spark-plan.outputs.runner }} environment: ${{ needs.llama-cpp-dgx-spark-plan.outputs.environment }} timeout-minutes: 180 concurrency: group: llama-cpp-dgx-spark-protected-runner cancel-in-progress: false permissions: contents: read env: BASE_SHA: ${{ inputs.base_sha }} CHECKOUT_SHA: ${{ inputs.checkout_sha }} WORKFLOW_SHA: ${{ github.workflow_sha }} QUALIFICATION_PLAN: ${{ needs.llama-cpp-dgx-spark-plan.outputs.plan }} PLAN_SHA256: ${{ needs.llama-cpp-dgx-spark-plan.outputs.plan_sha256 }} MODEL_HOST_PATH: ${{ needs.llama-cpp-dgx-spark-plan.outputs.model_host_path }} E2E_TARGET_ID: llama-cpp-dgx-spark-qualification E2E_JOB: "1" E2E_DEFAULT_ENABLED: "0" E2E_GATEWAY_RUNTIMES: docker E2E_AGENT_RUNTIME: openclaw E2E_OBSERVABLE_OUTCOME: "Exact candidate plan produces GPU protocol agent and cleanup evidence" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Protected physical DGX Spark GB10; verified local GGUF" steps: - name: Check out trusted Spark qualification harness uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: NVIDIA/NemoClaw ref: ${{ github.workflow_sha }} persist-credentials: false - name: Check out inert Spark candidate data uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: NVIDIA/NemoClaw ref: ${{ inputs.checkout_sha }} path: .candidate-source fetch-depth: 0 persist-credentials: false - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 - name: Set up the isolated Spark registry builder uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 with: driver-opts: network=host buildkitd-config-inline: | [registry."localhost:5000"] http = true - id: qualification name: Run the trusted Spark qualification shell: bash run: | set -euo pipefail artifact_dir="$GITHUB_WORKSPACE/e2e-artifacts/live/llama-cpp-dgx-spark-qualification" install -d -m 0700 "$artifact_dir" printf '%s' "$QUALIFICATION_PLAN" > "$artifact_dir/plan.json" printf '%s\n' "$PLAN_SHA256" > "$artifact_dir/plan-sha256" node --no-warnings --import tsx scripts/checks/run-llama-cpp-dgx-spark-qualification.cts \ --base-sha "$BASE_SHA" --head-sha "$CHECKOUT_SHA" --workflow-sha "$WORKFLOW_SHA" \ --candidate-root "$GITHUB_WORKSPACE/.candidate-source" \ --model-host-path "$MODEL_HOST_PATH" \ --plan "$artifact_dir/plan.json" --plan-sha256 "$PLAN_SHA256" \ --output "$artifact_dir/receipt.json" \ --registry-name "nemoclaw-llama-cpp-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" - id: cleanup name: Remove owned Spark qualification resources if: ${{ always() && steps.qualification.outcome != 'skipped' }} shell: bash run: | node --no-warnings --import tsx scripts/checks/run-llama-cpp-dgx-spark-qualification.cts \ --cleanup-only --registry-name "nemoclaw-llama-cpp-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" \ --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" - id: evidence name: Validate the revision-bound Spark receipt if: ${{ always() && steps.qualification.outcome != 'skipped' }} env: EXECUTION_OUTCOME: ${{ steps.qualification.outcome }} CLEANUP_OUTCOME: ${{ steps.cleanup.outcome }} shell: bash run: | artifact_dir="$GITHUB_WORKSPACE/e2e-artifacts/live/llama-cpp-dgx-spark-qualification" node --no-warnings scripts/checks/validate-llama-cpp-dgx-spark-receipt.mts \ --publish "$artifact_dir" "$EXECUTION_OUTCOME" "$CLEANUP_OUTCOME" "$PLAN_SHA256" - name: Upload validated Spark qualification evidence if: ${{ always() && steps.evidence.outcome == 'success' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-llama-cpp-dgx-spark-qualification path: e2e-artifacts/live/llama-cpp-dgx-spark-qualification/published/ native-runtime-qualification-producer-plan: needs: generate-matrix if: ${{ !cancelled() && needs.generate-matrix.result == 'success' && github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && inputs.jobs == 'native-runtime-qualification-producer' && inputs.targets == '' }} runs-on: ubuntu-latest timeout-minutes: 10 permissions: actions: read contents: read pull-requests: read outputs: matrix: ${{ steps.plan.outputs.matrix }} steps: - name: Check out the trusted qualification producer uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.workflow_sha }} persist-credentials: false sparse-checkout: | .github/actions/setup-reviewed-npm ci/reviewed-npm-audit.json scripts/lib/npm-diagnostics.sh scripts/lib/reviewed-npm-audit.mts src/lib/onboard/runtime-provider/native-qualification-authority.ts test/e2e/registry/native-runtime-qualification.ts tools/e2e/native-runtime-qualification-producer-plan.mts sparse-checkout-cone-mode: false - name: Check out the candidate for source authentication uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} path: .candidate-source fetch-depth: 1 persist-credentials: false - name: Set up Node for trusted qualification planning uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24.22.1 - name: Install reviewed npm uses: ./.github/actions/setup-reviewed-npm - id: source name: Authenticate the candidate and dispatch artifact env: BASE_SHA: ${{ inputs.base_sha }} CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository }} CANDIDATE_SHA: ${{ inputs.checkout_sha }} GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ inputs.pr_number }} PRODUCER_RUN_ATTEMPT: ${{ github.run_attempt }} PRODUCER_RUN_ID: ${{ github.run_id }} WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash run: | set -euo pipefail [[ "$CANDIDATE_REPOSITORY" == "NVIDIA/NemoClaw" ]] || { echo "::error::Native runtime qualification accepts only a branch in NVIDIA/NemoClaw" >&2 exit 1 } [[ "$PRODUCER_RUN_ATTEMPT" == "1" ]] || { echo "::error::Native runtime qualification producer runs cannot be rerun" >&2 exit 1 } [[ "$BASE_SHA" == "$WORKFLOW_SHA" && "$CANDIDATE_SHA" != "$WORKFLOW_SHA" ]] || { echo "::error::Native runtime qualification requires base_sha to equal workflow_sha and checkout_sha to name a different commit" >&2 exit 1 } [[ "$(git -C .candidate-source rev-parse --verify 'HEAD^{commit}')" == "$CANDIDATE_SHA" ]] || { echo "::error::Candidate checkout does not match the authenticated candidate commit" >&2 exit 1 } pull="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" jq -e \ --arg baseSha "$BASE_SHA" \ --arg candidateRepository "$CANDIDATE_REPOSITORY" \ --arg candidateSha "$CANDIDATE_SHA" ' .state == "open" and .head.repo.full_name == $candidateRepository and .head.sha == $candidateSha and .base.repo.full_name == "NVIDIA/NemoClaw" and .base.ref == "main" and .base.sha == $baseSha ' <<<"$pull" >/dev/null || { echo "::error::Pull request identity changed before qualification planning" >&2 exit 1 } artifact_name="e2e-dispatch-${PRODUCER_RUN_ID}-${PRODUCER_RUN_ATTEMPT}" artifacts="$(gh api --method GET \ "repos/${GITHUB_REPOSITORY}/actions/runs/${PRODUCER_RUN_ID}/artifacts" \ -f name="$artifact_name" \ -f per_page=100)" artifact="$(jq -ce \ --arg name "$artifact_name" \ --arg runId "$PRODUCER_RUN_ID" \ --arg workflowSha "$WORKFLOW_SHA" ' select(.total_count == 1 and (.artifacts | length) == 1) | .artifacts[0] | select(.name == $name and .expired == false) | select(.size_in_bytes > 0 and .size_in_bytes <= 1048576) | select((.id | tostring | test("^[1-9][0-9]{0,19}$"))) | select((.digest // "") | test("^sha256:[a-f0-9]{64}$")) | select((.workflow_run.id | tostring) == $runId) | select(.workflow_run.head_sha == $workflowSha) ' <<<"$artifacts")" || { echo "::error::Trusted dispatch artifact is missing or ambiguous" >&2 exit 1 } installer_sha256="$(sha256sum .candidate-source/scripts/install.sh | awk '{print $1}')" [[ "$installer_sha256" =~ ^[a-f0-9]{64}$ ]] || { echo "::error::Candidate installer SHA-256 is invalid" >&2 exit 1 } printf 'artifact_id=%s\n' "$(jq -r '.id | tostring' <<<"$artifact")" >>"$GITHUB_OUTPUT" printf 'artifact_name=%s\n' "$artifact_name" >>"$GITHUB_OUTPUT" printf 'artifact_digest=%s\n' "$(jq -r '.digest' <<<"$artifact")" >>"$GITHUB_OUTPUT" printf 'artifact_size=%s\n' "$(jq -r '.size_in_bytes' <<<"$artifact")" >>"$GITHUB_OUTPUT" printf 'installer_sha256=%s\n' "$installer_sha256" >>"$GITHUB_OUTPUT" - id: plan name: Compile the trusted qualification producer matrix env: BASE_SHA: ${{ inputs.base_sha }} CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository }} CANDIDATE_SHA: ${{ inputs.checkout_sha }} DISPATCH_ARTIFACT_DIGEST: ${{ steps.source.outputs.artifact_digest }} DISPATCH_ARTIFACT_ID: ${{ steps.source.outputs.artifact_id }} DISPATCH_ARTIFACT_NAME: ${{ steps.source.outputs.artifact_name }} DISPATCH_ARTIFACT_SIZE: ${{ steps.source.outputs.artifact_size }} INSTALLER_SHA256: ${{ steps.source.outputs.installer_sha256 }} NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL: ${{ vars.NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL }} PR_NUMBER: ${{ inputs.pr_number }} PRODUCER_RUN_ATTEMPT: ${{ github.run_attempt }} PRODUCER_RUN_ID: ${{ github.run_id }} WORKFLOW_SHA: ${{ github.workflow_sha }} run: node --no-warnings tools/e2e/native-runtime-qualification-producer-plan.mts --ci-output native-runtime-qualification-podman-toolchain: name: Build pinned native Podman toolchain / ${{ matrix.architecture }} needs: [generate-matrix, native-runtime-qualification-producer-plan] if: ${{ !cancelled() && needs.native-runtime-qualification-producer-plan.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }} strategy: fail-fast: false matrix: include: - architecture: amd64 runner: ubuntu-24.04 - architecture: arm64 runner: ubuntu-24.04-arm runs-on: ${{ matrix.runner }} timeout-minutes: 45 permissions: contents: read steps: - name: Check out the pinned Podman source uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: podman-container-tools/podman ref: cade97a52ebdf9dbf9e81de8009015776837a074 # v6.1.0 path: .podman-source fetch-depth: 1 persist-credentials: false - name: Check out the pinned Netavark source uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: containers/netavark ref: 8e91ad1d947ed325327b638f0cb906bea1f7d0ab # v2.1.0 path: .netavark-source fetch-depth: 1 persist-credentials: false - name: Check out the pinned Aardvark DNS source uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: containers/aardvark-dns ref: cd7417681229219059939bdd9f0b3bd9ac9abb08 # v2.1.0 path: .aardvark-source fetch-depth: 1 persist-credentials: false - name: Set up pinned Go for the Podman build uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: 1.25.9 cache: false - name: Set up pinned Rust for the network helper builds uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 with: toolchain: 1.88.0 cache: false rustflags: "" - name: Install build dependencies from the signed runner OS repository shell: bash run: | set -euo pipefail sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \ --yes --no-install-recommends \ curl gcc git libapparmor-dev libbtrfs-dev libc6-dev \ libdevmapper-dev libglib2.0-dev \ libprotobuf-c-dev libprotobuf-dev libseccomp-dev libselinux1-dev \ libsqlite3-dev libsystemd-dev make pkg-config protobuf-compiler - name: Build and package the pinned native toolchain env: AARDVARK_SOURCE_SHA: cd7417681229219059939bdd9f0b3bd9ac9abb08 EXPECTED_ARCHITECTURE: ${{ matrix.architecture }} NETAVARK_SOURCE_SHA: 8e91ad1d947ed325327b638f0cb906bea1f7d0ab PASTA_SOURCE_ARCHIVE_SHA256: 54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958 PASTA_SOURCE_SHA: f8df3f1b228fe19a74a269334fdfe6cc7d0605ce PASTA_VERSION: 2026_07_28.f8df3f1 PODMAN_SOURCE_SHA: cade97a52ebdf9dbf9e81de8009015776837a074 TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain shell: bash run: | set -euo pipefail [[ "$(dpkg --print-architecture)" == "$EXPECTED_ARCHITECTURE" ]] [[ "$(git -C .podman-source rev-parse --verify 'HEAD^{commit}')" == "$PODMAN_SOURCE_SHA" ]] [[ "$(git -C .netavark-source rev-parse --verify 'HEAD^{commit}')" == "$NETAVARK_SOURCE_SHA" ]] [[ "$(git -C .aardvark-source rev-parse --verify 'HEAD^{commit}')" == "$AARDVARK_SOURCE_SHA" ]] [[ ! -e .passt-source && ! -L .passt-source ]] pasta_source_archive="${RUNNER_TEMP}/passt-${PASTA_SOURCE_SHA}.tar.gz" [[ ! -e "$pasta_source_archive" && ! -L "$pasta_source_archive" ]] /usr/bin/curl \ --fail --location --proto '=https' --proto-redir '=https' \ --retry 3 --show-error --silent --tlsv1.2 \ --output "$pasta_source_archive" \ "https://passt.top/passt/snapshot/passt-${PASTA_SOURCE_SHA}.tar.gz" [[ -f "$pasta_source_archive" && ! -L "$pasta_source_archive" ]] [[ "$(sha256sum "$pasta_source_archive" | cut -d' ' -f1)" == "$PASTA_SOURCE_ARCHIVE_SHA256" ]] mkdir .passt-source tar \ --extract --gzip --file="$pasta_source_archive" \ --directory=.passt-source --strip-components=1 \ --no-same-owner --no-same-permissions [[ -f .passt-source/Makefile && ! -L .passt-source/Makefile ]] [[ -f .passt-source/passt.c && ! -L .passt-source/passt.c ]] [[ ! -e .passt-source/passt && ! -L .passt-source/passt ]] for source in .podman-source .netavark-source .aardvark-source; do [[ -z "$(git -C "$source" status --porcelain --untracked-files=no)" ]] done [[ "$(go version)" == go\ version\ go1.25.9\ * ]] [[ "$(rustc --version)" == rustc\ 1.88.0\ * ]] SOURCE_DATE_EPOCH=1786554266 \ BUILD_ORIGIN="NVIDIA/NemoClaw native runtime qualification" \ EXTRA_BUILDTAGS=containers_image_openpgp \ make --directory=.podman-source --jobs=2 \ podman rootlessport PREFIX=/usr/local SOURCE_DATE_EPOCH=1785940686 CI=1 \ make --directory=.netavark-source --jobs=2 build SOURCE_DATE_EPOCH=1785940850 CI=1 \ make --directory=.aardvark-source --jobs=2 build SOURCE_DATE_EPOCH=1785255008 \ make --directory=.passt-source --jobs=2 VERSION="$PASTA_VERSION" passt podman_dependencies="$(ldd .podman-source/bin/podman)" printf '%s\n' "$podman_dependencies" if grep -F "not found" <<<"$podman_dependencies"; then echo "::error::Pinned Podman build has an unresolved runtime dependency" >&2 exit 1 fi if grep -E "libgpgme|libsubid" <<<"$podman_dependencies"; then echo "::error::Pinned Podman build must not require an optional host ABI" >&2 exit 1 fi install -D -m 0755 .podman-source/bin/podman "$TOOLCHAIN_DIRECTORY/bin/podman" [[ -f .passt-source/passt && ! -L .passt-source/passt ]] install -D -m 0755 .passt-source/passt "$TOOLCHAIN_DIRECTORY/bin/pasta" install -D -m 0755 .podman-source/bin/rootlessport \ "$TOOLCHAIN_DIRECTORY/libexec/podman/rootlessport" install -D -m 0755 .netavark-source/bin/netavark \ "$TOOLCHAIN_DIRECTORY/libexec/podman/netavark" install -D -m 0755 .aardvark-source/bin/aardvark-dns \ "$TOOLCHAIN_DIRECTORY/libexec/podman/aardvark-dns" install -D -m 0644 \ .podman-source/vendor/go.podman.io/common/pkg/config/containers.conf \ "$TOOLCHAIN_DIRECTORY/share/containers/containers.conf" [[ "$("$TOOLCHAIN_DIRECTORY/bin/podman" --version)" == "podman version 6.1.0" ]] pasta_version_output="$("$TOOLCHAIN_DIRECTORY/bin/pasta" --version)" [[ "${pasta_version_output%%$'\n'*}" == "pasta $PASTA_VERSION" ]] || { echo "::error::Pinned qualification pasta version is invalid" >&2 exit 1 } [[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/netavark" --version)" == "netavark 2.1.0" ]] [[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/aardvark-dns" --version)" == "aardvark-dns 2.1.0" ]] jq -n \ --arg architecture "$EXPECTED_ARCHITECTURE" \ --arg aardvarkDnsSourceSha "$AARDVARK_SOURCE_SHA" \ --arg netavarkSourceSha "$NETAVARK_SOURCE_SHA" \ --arg pastaSourceArchiveSha256 "$PASTA_SOURCE_ARCHIVE_SHA256" \ --arg pastaSourceSha "$PASTA_SOURCE_SHA" \ --arg pastaVersion "$PASTA_VERSION" \ --arg podmanSourceSha "$PODMAN_SOURCE_SHA" ' { schemaVersion: 1, kind: "nemoclaw-native-podman-toolchain-v1", architecture: $architecture, podmanVersion: "6.1.0", podmanSourceSha: $podmanSourceSha, netavarkVersion: "2.1.0", netavarkSourceSha: $netavarkSourceSha, aardvarkDnsVersion: "2.1.0", aardvarkDnsSourceSha: $aardvarkDnsSourceSha, pastaVersion: $pastaVersion, pastaSourceArchiveSha256: $pastaSourceArchiveSha256, pastaSourceSha: $pastaSourceSha, goVersion: "1.25.9", rustVersion: "1.88.0" } ' >"$TOOLCHAIN_DIRECTORY/manifest.json" ( cd "$TOOLCHAIN_DIRECTORY" sha256sum \ bin/pasta \ bin/podman \ libexec/podman/aardvark-dns \ libexec/podman/netavark \ libexec/podman/rootlessport \ manifest.json \ share/containers/containers.conf >SHA256SUMS ) - name: Upload the pinned native Podman toolchain if: success() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: native-runtime-podman-toolchain-${{ matrix.architecture }} path: ${{ runner.temp }}/native-runtime-podman-toolchain/ portable-podman-toolchain: name: Build pinned Portable Podman 5.7 toolchain needs: generate-matrix if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'portable-hermes-finalization') }} runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: contents: read steps: - name: Check out the pinned Podman 5.7 source uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: podman-container-tools/podman ref: 0370128fc8dcae93533334324ef838db8f8da8cb # v5.7.0 path: .podman-source fetch-depth: 1 persist-credentials: false - name: Set up pinned Go for the Podman 5.7 build uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: 1.25.9 cache: false - name: Install pinned Podman build dependencies from the signed Ubuntu snapshot env: UBUNTU_SNAPSHOT_ID: 20260911T000000Z shell: bash run: | set -euo pipefail packages=( "gcc=4:13.2.0-7ubuntu1" "git=1:2.43.0-1ubuntu7.3" "libapparmor-dev=4.0.1really4.0.1-0ubuntu0.24.04.7" "libbtrfs-dev=6.6.3-1.1build2" "libc6-dev=2.39-0ubuntu8.9" "libdevmapper-dev=2:1.02.185-3ubuntu3.2" "libglib2.0-dev=2.80.0-6ubuntu3.8" "libprotobuf-c-dev=1.4.1-1ubuntu4" "libprotobuf-dev=3.21.12-8.2ubuntu0.3" "libseccomp-dev=2.5.5-1ubuntu3.1" "libselinux1-dev=3.5-2ubuntu2.1" "libsqlite3-dev=3.45.1-1ubuntu2.7" "libsystemd-dev=255.4-1ubuntu8.17" "make=4.3-4.1build2" "pkg-config=1.8.1-2build1" "protobuf-compiler=3.21.12-8.2ubuntu0.3" ) snapshot_sources="$RUNNER_TEMP/ubuntu-snapshot.sources" snapshot_lists="$RUNNER_TEMP/ubuntu-snapshot-lists" snapshot_preferences="$RUNNER_TEMP/ubuntu-snapshot.pref" cat >"$snapshot_sources" <"$snapshot_preferences" <<'EOF' Package: * Pin: origin snapshot.ubuntu.com Pin-Priority: 1001 EOF sudo install -d -m 0755 -o root -g root "$snapshot_lists" sudo install -d -m 0700 -o _apt -g root "$snapshot_lists/partial" apt_sources=( -o "Dir::Etc::sourcelist=$snapshot_sources" -o 'Dir::Etc::sourceparts=-' -o "Dir::Etc::preferences=$snapshot_preferences" -o 'Dir::Etc::preferencesparts=-' -o "Dir::State::lists=$snapshot_lists" -o 'APT::Get::AllowUnauthenticated=false' -o 'Acquire::AllowInsecureRepositories=false' ) sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get \ "${apt_sources[@]}" update sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get \ "${apt_sources[@]}" install --yes --no-install-recommends --allow-downgrades \ "${packages[@]}" for package_spec in "${packages[@]}"; do package_name="${package_spec%%=*}" expected_version="${package_spec#*=}" actual_version="$(dpkg-query --show --showformat='${Version}' "$package_name")" [[ "$actual_version" == "$expected_version" ]] done - name: Download the reviewed native helper toolchain uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: native-podman-e2e-toolchain-amd64 path: ${{ runner.temp }}/portable-podman-e2e-toolchain - name: Replace only Podman with the pinned portable runtime env: PODMAN_SOURCE_SHA: 0370128fc8dcae93533334324ef838db8f8da8cb TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/portable-podman-e2e-toolchain shell: bash run: | set -euo pipefail [[ "$(dpkg --print-architecture)" == amd64 ]] [[ "$(git -C .podman-source rev-parse --verify 'HEAD^{commit}')" == "$PODMAN_SOURCE_SHA" ]] [[ -d "$TOOLCHAIN_DIRECTORY" && ! -L "$TOOLCHAIN_DIRECTORY" ]] [[ -z "$(find -P "$TOOLCHAIN_DIRECTORY" -type l -print -quit)" ]] ( cd "$TOOLCHAIN_DIRECTORY" sha256sum --check --strict SHA256SUMS ) jq -e ' .schemaVersion == 1 and .kind == "nemoclaw-native-podman-toolchain-v1" and .architecture == "amd64" and .podmanVersion == "6.1.0" and .podmanSourceSha == "cade97a52ebdf9dbf9e81de8009015776837a074" and .netavarkVersion == "2.1.0" and .netavarkSourceSha == "8e91ad1d947ed325327b638f0cb906bea1f7d0ab" and .aardvarkDnsVersion == "2.1.0" and .aardvarkDnsSourceSha == "cd7417681229219059939bdd9f0b3bd9ac9abb08" and .pastaVersion == "2026_07_28.f8df3f1" and .pastaSourceArchiveSha256 == "54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958" and .pastaSourceSha == "f8df3f1b228fe19a74a269334fdfe6cc7d0605ce" and .goVersion == "1.25.9" and .rustVersion == "1.88.0" ' "$TOOLCHAIN_DIRECTORY/manifest.json" >/dev/null [[ -z "$(git -C .podman-source status --porcelain --untracked-files=no)" ]] [[ "$(go version)" == go\ version\ go1.25.9\ * ]] SOURCE_DATE_EPOCH=1762886964 \ BUILD_ORIGIN="NVIDIA/NemoClaw Portable Podman qualification" \ EXTRA_BUILDTAGS=containers_image_openpgp \ make --directory=.podman-source --jobs=2 \ podman rootlessport PREFIX=/usr/local podman_dependencies="$(ldd .podman-source/bin/podman)" printf '%s\n' "$podman_dependencies" if grep -F "not found" <<<"$podman_dependencies"; then echo "::error::Pinned Portable Podman build has an unresolved runtime dependency" >&2 exit 1 fi if grep -E "libgpgme|libsubid" <<<"$podman_dependencies"; then echo "::error::Pinned Portable Podman build must not require an optional host ABI" >&2 exit 1 fi install -m 0755 .podman-source/bin/podman "$TOOLCHAIN_DIRECTORY/bin/podman" install -m 0755 .podman-source/bin/rootlessport \ "$TOOLCHAIN_DIRECTORY/libexec/podman/rootlessport" install -m 0644 \ .podman-source/vendor/go.podman.io/common/pkg/config/containers.conf \ "$TOOLCHAIN_DIRECTORY/share/containers/containers.conf" [[ "$("$TOOLCHAIN_DIRECTORY/bin/podman" --version)" == "podman version 5.7.0" ]] jq \ --arg podmanSourceSha "$PODMAN_SOURCE_SHA" \ '.podmanVersion = "5.7.0" | .podmanSourceSha = $podmanSourceSha' \ "$TOOLCHAIN_DIRECTORY/manifest.json" \ >"$TOOLCHAIN_DIRECTORY/manifest.next.json" mv "$TOOLCHAIN_DIRECTORY/manifest.next.json" "$TOOLCHAIN_DIRECTORY/manifest.json" ( cd "$TOOLCHAIN_DIRECTORY" sha256sum \ bin/pasta \ bin/podman \ libexec/podman/aardvark-dns \ libexec/podman/netavark \ libexec/podman/rootlessport \ manifest.json \ share/containers/containers.conf >SHA256SUMS ) - name: Upload the pinned Portable Podman toolchain uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: portable-podman-e2e-toolchain-amd64 path: ${{ runner.temp }}/portable-podman-e2e-toolchain/ if-no-files-found: error retention-days: 3 compression-level: 0 native-runtime-qualification-producer: name: ${{ matrix.jobName }} needs: - generate-matrix - native-runtime-qualification-podman-toolchain - native-runtime-qualification-producer-plan if: ${{ !cancelled() && needs.native-runtime-qualification-podman-toolchain.result == 'success' && needs.native-runtime-qualification-producer-plan.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }} strategy: fail-fast: false matrix: ${{ fromJSON(needs.native-runtime-qualification-producer-plan.outputs.matrix) }} runs-on: ${{ matrix.runner }} timeout-minutes: 120 permissions: contents: read env: NEMOCLAW_RUN_LIVE_E2E: "1" QUALIFICATION_ROW: ${{ toJSON(matrix) }} steps: - name: Require an ephemeral qualification runner env: REVIEWED_EPHEMERAL_POOL: ${{ vars.NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL }} RUNNER_ENVIRONMENT: ${{ runner.environment }} shell: bash run: | set -euo pipefail if [[ "$RUNNER_ENVIRONMENT" != "github-hosted" && "$REVIEWED_EPHEMERAL_POOL" != "enabled" ]]; then echo "::error::Self-hosted native runtime qualification requires the reviewed ephemeral runner pool" >&2 exit 1 fi - name: Check out the trusted qualification harness uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ matrix.source.workflowSha }} path: .qualification-workflow persist-credentials: false sparse-checkout: | .github/actions/setup-reviewed-npm ci/reviewed-npm-audit.json scripts/lib/npm-diagnostics.sh scripts/lib/reviewed-npm-audit.mts src/lib/onboard/runtime-provider/native-qualification-authority.ts scripts/checks/run-native-runtime-installer-qualification.sh test/e2e/registry/native-runtime-qualification.ts tools/e2e/native-runtime-qualification-producer-plan.mts tools/e2e/native-runtime-qualification-producer-evidence.mts sparse-checkout-cone-mode: false - name: Check out the candidate commit uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ matrix.source.candidateRepository }} ref: ${{ matrix.source.candidateSha }} path: .candidate-runtime fetch-depth: 1 persist-credentials: false - name: Set up Node for candidate execution uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24.22.1 - name: Install reviewed npm uses: ./.qualification-workflow/.github/actions/setup-reviewed-npm - name: Require a reviewed Ubuntu runtime host shell: bash run: | set -euo pipefail [[ -r /etc/os-release ]] || { echo "::error::Protected runner does not expose an OS release identity" >&2 exit 1 } # shellcheck disable=SC1091 source /etc/os-release [[ "${ID:-}" == "ubuntu" ]] || { echo "::error::Protected runner must use a reviewed Ubuntu image" >&2 exit 1 } [[ "${VERSION_ID:-}" == "24.04" || "${VERSION_ID:-}" == "26.04" ]] || { echo "::error::Protected runner Ubuntu release is not reviewed for Podman qualification" >&2 exit 1 } - name: Download the pinned native Podman toolchain uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: native-runtime-podman-toolchain-${{ matrix.case.architecture }} path: ${{ runner.temp }}/native-runtime-podman-toolchain - name: Install the pinned native Podman toolchain and rootless prerequisites env: EXPECTED_ARCHITECTURE: ${{ matrix.case.architecture }} TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain shell: bash run: | set -euo pipefail [[ -x /usr/bin/apt-get ]] || { echo "::error::Protected runner cannot install rootless Podman prerequisites from its signed OS repository" >&2 exit 1 } sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \ --yes --no-install-recommends \ acl apparmor btrfs-progs conmon \ golang-github-containers-common iptables nftables runc slirp4netns uidmap [[ -d "$TOOLCHAIN_DIRECTORY" && ! -L "$TOOLCHAIN_DIRECTORY" ]] [[ -z "$(find -P "$TOOLCHAIN_DIRECTORY" -type l -print -quit)" ]] mapfile -t actual_files < <( cd "$TOOLCHAIN_DIRECTORY" find . -type f -print | LC_ALL=C sort ) expected_files=( ./SHA256SUMS ./bin/pasta ./bin/podman ./libexec/podman/aardvark-dns ./libexec/podman/netavark ./libexec/podman/rootlessport ./manifest.json ./share/containers/containers.conf ) [[ "${actual_files[*]}" == "${expected_files[*]}" ]] || { echo "::error::Downloaded native Podman toolchain contains unexpected files" >&2 exit 1 } ( cd "$TOOLCHAIN_DIRECTORY" sha256sum --check --strict SHA256SUMS ) jq -e \ --arg architecture "$EXPECTED_ARCHITECTURE" ' type == "object" and keys == [ "aardvarkDnsSourceSha", "aardvarkDnsVersion", "architecture", "goVersion", "kind", "netavarkSourceSha", "netavarkVersion", "pastaSourceArchiveSha256", "pastaSourceSha", "pastaVersion", "podmanSourceSha", "podmanVersion", "rustVersion", "schemaVersion" ] and .schemaVersion == 1 and .kind == "nemoclaw-native-podman-toolchain-v1" and .architecture == $architecture and .podmanVersion == "6.1.0" and .podmanSourceSha == "cade97a52ebdf9dbf9e81de8009015776837a074" and .netavarkVersion == "2.1.0" and .netavarkSourceSha == "8e91ad1d947ed325327b638f0cb906bea1f7d0ab" and .aardvarkDnsVersion == "2.1.0" and .aardvarkDnsSourceSha == "cd7417681229219059939bdd9f0b3bd9ac9abb08" and .pastaVersion == "2026_07_28.f8df3f1" and .pastaSourceArchiveSha256 == "54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958" and .pastaSourceSha == "f8df3f1b228fe19a74a269334fdfe6cc7d0605ce" and .goVersion == "1.25.9" and .rustVersion == "1.88.0" ' "$TOOLCHAIN_DIRECTORY/manifest.json" >/dev/null for target in \ /usr/local/bin/podman \ /usr/local/libexec/podman/aardvark-dns \ /usr/local/libexec/podman/netavark \ /usr/local/libexec/podman/rootlessport \ /usr/share/containers/containers.conf; do [[ ! -L "$target" ]] || { echo "::error::Native Podman toolchain target must not be a symlink: $target" >&2 exit 1 } done sudo install --owner=root --group=root --mode=0755 \ "$TOOLCHAIN_DIRECTORY/bin/podman" /usr/local/bin/podman for helper in aardvark-dns netavark rootlessport; do sudo install -D --owner=root --group=root --mode=0755 \ "$TOOLCHAIN_DIRECTORY/libexec/podman/$helper" \ "/usr/local/libexec/podman/$helper" done sudo install --owner=root --group=root --mode=0644 \ "$TOOLCHAIN_DIRECTORY/share/containers/containers.conf" \ /usr/share/containers/containers.conf [[ "$(command -v podman)" == "/usr/local/bin/podman" ]] conmon_version="$(conmon --version | awk 'NR == 1 { print $NF }')" runc_version="$(runc --version | awk 'NR == 1 { print $NF }')" dpkg --compare-versions "$conmon_version" ge 2.1.7 dpkg --compare-versions "$runc_version" ge 1.1.11 [[ "$(/usr/local/libexec/podman/netavark --version)" == "netavark 2.1.0" ]] [[ "$(/usr/local/libexec/podman/aardvark-dns --version)" == "aardvark-dns 2.1.0" ]] version="$(podman --version)" [[ "$version" == "podman version 6.1.0" ]] - name: Prepare the credential-free execution account and disable Docker id: boundary env: CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain shell: bash run: | set -euo pipefail for command in apparmor_parser awk cat curl getent git grep groupdel id jq node npm pgrep podman setfacl sha256sum stat systemctl tee unlink useradd userdel usermod; do command -v "$command" >/dev/null || { echo "::error::Protected runner is missing required command: $command" >&2 exit 1 } done [[ "$(podman --version)" == "podman version 6.1.0" ]] || { echo "::error::Protected runner must provide Podman 6.1.0" >&2 exit 1 } sudo systemctl stop docker.service docker.socket 2>/dev/null || true sudo systemctl mask --runtime docker.service docker.socket 2>/dev/null || true sudo pkill -TERM -x dockerd 2>/dev/null || true sudo rm -f /var/run/docker.sock /run/docker.sock ! systemctl is-active --quiet docker.service ! systemctl is-active --quiet docker.socket ! pgrep -x dockerd >/dev/null [[ ! -S /var/run/docker.sock && ! -S /run/docker.sock ]] account="nemoclawq" ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" if getent passwd "$account" >/dev/null || getent group "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then echo "::error::Qualification account, group, or subordinate-ID authorization already exists" >&2 exit 1 fi [[ ! -e "$ownership_marker" && ! -L "$ownership_marker" ]] || { echo "::error::Qualification account ownership marker already exists" >&2 exit 1 } account_created_without_marker=0 rollback_unmarked_account() { local result="$?" trap - EXIT if ((result != 0 && account_created_without_marker == 1)); then sudo userdel --remove "$account" 2>/dev/null || true sudo groupdel "$account" 2>/dev/null || true sudo rm -f -- "$ownership_marker" if getent passwd "$account" >/dev/null || getent group "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then echo "::error::Partially created qualification account could not be rolled back" >&2 exit 1 fi fi exit "$result" } trap rollback_unmarked_account EXIT sudo useradd --create-home --shell /usr/sbin/nologin --user-group "$account" account_created_without_marker=1 uid="$(id -u "$account")" gid="$(id -g "$account")" home="$(getent passwd "$account" | cut -d: -f6)" group_entry="$(getent group "$account")" [[ "$uid" =~ ^[0-9]+$ && "$gid" =~ ^[0-9]+$ && "$home" == "/home/${account}" && -d "$home" && ! -L "$home" ]] || { echo "::error::Qualification account identity is missing or invalid" >&2 exit 1 } [[ "$group_entry" == "${account}:x:${gid}:" ]] || { echo "::error::Qualification private group identity is missing or invalid" >&2 exit 1 } printf '%s:%s:%s\n' "$account" "$uid" "$gid" | sudo tee "$ownership_marker" >/dev/null sudo chown root:root "$ownership_marker" sudo chmod 0400 "$ownership_marker" [[ -f "$ownership_marker" && ! -L "$ownership_marker" && "$(stat -c '%u:%g:%a' "$ownership_marker")" == "0:0:400" ]] || { echo "::error::Qualification account ownership marker is invalid" >&2 exit 1 } account_created_without_marker=0 trap - EXIT ensure_subordinate_range() { local file="$1" local option="$2" local range_start=100000 local range_end local conflict_end [[ -f "$file" && ! -L "$file" ]] || { echo "::error::Rootless Podman subordinate-ID file is missing or invalid: $file" >&2 exit 1 } if awk -F: -v account="$account" ' $1 == account && $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $3 >= 65536 { found = 1 } END { exit found ? 0 : 1 } ' "$file"; then return fi while :; do ((range_start <= 4294901760)) || { echo "::error::Protected runner has no free subordinate-ID range for rootless Podman" >&2 exit 1 } range_end=$((range_start + 65535)) conflict_end="$(awk -F: -v start="$range_start" -v end="$range_end" ' $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ { current_end = $2 + $3 - 1 if ($2 <= end && current_end >= start && current_end > maximum) maximum = current_end } END { if (maximum != "") print maximum } ' "$file")" [[ -n "$conflict_end" ]] || break range_start=$((conflict_end + 1)) done range_end=$((range_start + 65535)) sudo usermod "$option" "${range_start}-${range_end}" "$account" awk -F: -v account="$account" ' $1 == account && $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $3 >= 65536 { found = 1 } END { exit found ? 0 : 1 } ' "$file" || { echo "::error::Protected runner did not provision rootless Podman subordinate IDs" >&2 exit 1 } } ensure_subordinate_range /etc/subuid --add-subuids ensure_subordinate_range /etc/subgid --add-subgids printf 'account=%s\n' "$account" >>"$GITHUB_OUTPUT" printf 'account_created=true\n' >>"$GITHUB_OUTPUT" printf 'uid=%s\n' "$uid" >>"$GITHUB_OUTPUT" printf 'gid=%s\n' "$gid" >>"$GITHUB_OUTPUT" runtime_dir="/run/user/${uid}" runtime_directory_unit="user-runtime-dir@${uid}.service" user_manager_unit="user@${uid}.service" user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d" user_manager_dropin="${user_manager_dropin_directory}/50-nemoclaw-native-runtime.conf" trusted_user_unit_path="/usr/lib/systemd/user:/lib/systemd/user" [[ ! -e "$user_manager_dropin_directory" && ! -L "$user_manager_dropin_directory" ]] || { echo "::error::Qualification systemd user-manager drop-in directory already exists" >&2 exit 1 } sudo install -d --owner=root --group=root --mode=0755 "$user_manager_dropin_directory" printf '[Service]\nEnvironment="SYSTEMD_UNIT_PATH=%s"\n' "$trusted_user_unit_path" | sudo tee "$user_manager_dropin" >/dev/null sudo chown root:root "$user_manager_dropin" sudo chmod 0444 "$user_manager_dropin" [[ -f "$user_manager_dropin" && ! -L "$user_manager_dropin" && "$(stat -c '%u:%g:%a:%h' "$user_manager_dropin")" == "0:0:444:1" ]] || { echo "::error::Qualification systemd user-manager drop-in is invalid" >&2 exit 1 } sudo systemctl daemon-reload verify_user_manager_unit_path() { local environment environment="$(sudo -u "$1" env -i \ HOME="$2" \ LANG=C.UTF-8 \ PATH=/usr/bin:/bin \ XDG_RUNTIME_DIR="$3" \ /usr/bin/systemctl --user show-environment)" tr ' ' '\n' <<<"$environment" | grep -Fx -- "SYSTEMD_UNIT_PATH=$trusted_user_unit_path" >/dev/null || { echo "::error::Qualification systemd user manager did not inherit the trusted unit path" >&2 return 1 } } verify_user_bus() { local execution_account="$1" local expected_uid="$2" local bus="$3" local context="$4" sudo /usr/bin/test -S "$bus" && sudo /usr/bin/test ! -L "$bus" && [[ "$(sudo stat -c '%u' -- "$bus")" == "$expected_uid" ]] && sudo -u "$execution_account" /usr/bin/test -S "$bus" || { echo "::error::Qualification systemd user bus $context" >&2 return 1 } } sudo systemctl start "$user_manager_unit" systemctl is-active --quiet "$runtime_directory_unit" || { echo "::error::Qualification systemd runtime-directory unit is not active" >&2 exit 1 } [[ -d "$runtime_dir" && ! -L "$runtime_dir" && "$(stat -c '%u:%g:%a' "$runtime_dir")" == "${uid}:${gid}:700" ]] || { echo "::error::Qualification runtime directory is missing or invalid" >&2 exit 1 } systemctl is-active --quiet "$user_manager_unit" || { echo "::error::Qualification systemd user manager is not active" >&2 exit 1 } sudo -u "$account" env -i \ HOME="$home" \ LANG=C.UTF-8 \ PATH=/usr/bin:/bin \ XDG_RUNTIME_DIR="$runtime_dir" \ /usr/bin/systemctl --user start dbus.socket sudo -u "$account" env -i \ HOME="$home" \ LANG=C.UTF-8 \ PATH=/usr/bin:/bin \ XDG_RUNTIME_DIR="$runtime_dir" \ /usr/bin/systemctl --user is-active --quiet dbus.socket || { echo "::error::Qualification systemd user bus socket unit is not active" >&2 exit 1 } verify_user_bus "$account" "$uid" "$runtime_dir/bus" \ "is invalid or inaccessible to the execution account" verify_user_manager_unit_path "$account" "$home" "$runtime_dir" storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" storage_config="${storage_config_directory}/storage.conf" containers_config="${storage_config_directory}/containers.conf" podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" apparmor_profile="${storage_config_directory}/podman.apparmor" apparmor_profile_name="nemoclaw-native-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" pasta_executable="${helper_directory}/pasta" pasta_apparmor_profile="${storage_config_directory}/pasta.apparmor" pasta_apparmor_profile_name="nemoclaw-native-pasta-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" [[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { echo "::error::Qualification storage configuration directory already exists" >&2 exit 1 } [[ ! -e "$helper_directory" && ! -L "$helper_directory" ]] || { echo "::error::Qualification helper directory already exists" >&2 exit 1 } sudo install -d -o root -g root -m 0755 "$storage_config_directory" [[ -f "$TOOLCHAIN_DIRECTORY/bin/podman" && ! -L "$TOOLCHAIN_DIRECTORY/bin/podman" ]] || { echo "::error::Pinned qualification Podman executable source is missing or invalid" >&2 exit 1 } [[ ! -e "$podman_executable" && ! -L "$podman_executable" ]] || { echo "::error::Run-owned qualification Podman executable already exists" >&2 exit 1 } sudo install --owner=root --group=root --mode=0555 \ "$TOOLCHAIN_DIRECTORY/bin/podman" "$podman_executable" [[ -f "$podman_executable" && ! -L "$podman_executable" && "$(stat -c '%u:%g:%a' "$podman_executable")" == "0:0:555" ]] || { echo "::error::Run-owned qualification Podman executable is invalid" >&2 exit 1 } [[ "$(sha256sum "$podman_executable" | cut -d' ' -f1)" == "$(sha256sum "$TOOLCHAIN_DIRECTORY/bin/podman" | cut -d' ' -f1)" ]] || { echo "::error::Run-owned qualification Podman executable digest changed during installation" >&2 exit 1 } [[ -f "$TOOLCHAIN_DIRECTORY/bin/pasta" && ! -L "$TOOLCHAIN_DIRECTORY/bin/pasta" ]] || { echo "::error::Pinned qualification pasta executable source is missing or invalid" >&2 exit 1 } sudo install -d --owner=root --group=root --mode=0555 "$helper_directory" sudo install --owner=root --group=root --mode=0555 \ "$TOOLCHAIN_DIRECTORY/bin/pasta" "$pasta_executable" [[ -d "$helper_directory" && ! -L "$helper_directory" && "$(stat -c '%u:%g:%a' "$helper_directory")" == "0:0:555" ]] || { echo "::error::Run-owned qualification helper directory is invalid" >&2 exit 1 } [[ -f "$pasta_executable" && ! -L "$pasta_executable" && "$(stat -c '%u:%g:%a' "$pasta_executable")" == "0:0:555" ]] || { echo "::error::Run-owned qualification pasta executable is invalid" >&2 exit 1 } [[ "$(sha256sum "$pasta_executable" | cut -d' ' -f1)" == "$(sha256sum "$TOOLCHAIN_DIRECTORY/bin/pasta" | cut -d' ' -f1)" ]] || { echo "::error::Run-owned qualification pasta executable digest changed during installation" >&2 exit 1 } printf '%s\n' \ '[storage]' \ 'driver = "overlay"' \ "runroot = \"${home}/.local/share/containers/runroot\"" \ "graphroot = \"${home}/.local/share/containers/storage\"" \ "rootless_storage_path = \"${home}/.local/share/containers/storage\"" | sudo tee "$storage_config" >/dev/null sudo chown root:root "$storage_config" sudo chmod 0444 "$storage_config" [[ -f "$storage_config" && ! -L "$storage_config" && "$(stat -c '%u:%g:%a' "$storage_config")" == "0:0:444" ]] || { echo "::error::Qualification storage configuration is not root-owned and read-only" >&2 exit 1 } printf '%s\n' \ '[network]' \ 'firewall_driver = "nftables"' | sudo tee "$containers_config" >/dev/null sudo chown root:root "$containers_config" sudo chmod 0444 "$containers_config" [[ -f "$containers_config" && ! -L "$containers_config" && "$(stat -c '%u:%g:%a' "$containers_config")" == "0:0:444" ]] || { echo "::error::Qualification containers configuration is not root-owned and read-only" >&2 exit 1 } if [[ -r /sys/module/apparmor/parameters/enabled ]] && grep -q '^Y' /sys/module/apparmor/parameters/enabled; then printf '%s\n' \ '# This ephemeral profile grants user namespaces only to the pinned qualification Podman binary.' \ '' \ 'abi ,' \ 'include ' \ '' \ "profile ${apparmor_profile_name} ${podman_executable} flags=(unconfined) {" \ ' userns,' \ '}' | sudo tee "$apparmor_profile" >/dev/null sudo chown root:root "$apparmor_profile" sudo chmod 0444 "$apparmor_profile" [[ -f "$apparmor_profile" && ! -L "$apparmor_profile" && "$(stat -c '%u:%g:%a' "$apparmor_profile")" == "0:0:444" ]] || { echo "::error::Qualification AppArmor profile is not root-owned and read-only" >&2 exit 1 } sudo apparmor_parser -r "$apparmor_profile" printf '%s\n' \ '# This ephemeral profile is limited to the immutable run-owned pasta helper.' \ '' \ 'abi ,' \ 'include ' \ '' \ "profile ${pasta_apparmor_profile_name} ${pasta_executable} flags=(unconfined) {" \ ' userns,' \ '}' | sudo tee "$pasta_apparmor_profile" >/dev/null sudo chown root:root "$pasta_apparmor_profile" sudo chmod 0444 "$pasta_apparmor_profile" [[ -f "$pasta_apparmor_profile" && ! -L "$pasta_apparmor_profile" && "$(stat -c '%u:%g:%a' "$pasta_apparmor_profile")" == "0:0:444" ]] || { echo "::error::Qualification pasta AppArmor profile is not root-owned and read-only" >&2 exit 1 } sudo apparmor_parser -r "$pasta_apparmor_profile" fi ancestor="$(dirname "$CANDIDATE_DIRECTORY")" while [[ "$ancestor" != "/home" ]]; do [[ ("$ancestor" == "/home/runner" || "$ancestor" == /home/runner/*) && -d "$ancestor" && ! -L "$ancestor" ]] || { echo "::error::Candidate checkout ancestor is outside the reviewed runner workspace" >&2 exit 1 } sudo setfacl --modify "u:${account}:--x" "$ancestor" ancestor="$(dirname "$ancestor")" done sudo chown -R "$uid:$gid" "$CANDIDATE_DIRECTORY" node_directory="$(dirname "$(command -v node)")" [[ "$node_directory" == /* && -x "$node_directory/node" && -x "$node_directory/npm" ]] || { echo "::error::Pinned Node toolchain path is invalid" >&2 exit 1 } guard_dir="${RUNNER_TEMP}/native-runtime-docker-guard" install -d -m 0755 "$guard_dir" printf '%s\n' '#!/usr/bin/env bash' 'exit 97' >"$guard_dir/docker" chmod 0555 "$guard_dir/docker" podman_info="$(sudo -u "$account" env -i \ CONTAINERS_CONF="$containers_config" \ CONTAINERS_STORAGE_CONF="$storage_config" \ HOME="$home" \ LANG=C.UTF-8 \ PATH="$guard_dir:$helper_directory:/usr/local/bin:/usr/bin:/bin" \ XDG_RUNTIME_DIR="$runtime_dir" \ "$podman_executable" info --format json)" || { echo "::error::Credential-free rootless Podman readiness failed" >&2 exit 1 } jq -e ' .host.security.rootless == true and .store.graphDriverName == "overlay" and (((.store.graphOptions // {})["overlay.mount_program"].Executable? // "") == "") ' <<<"$podman_info" >/dev/null || { echo "::error::Qualification requires native rootless overlay storage" >&2 exit 1 } printf 'home=%s\n' "$home" >>"$GITHUB_OUTPUT" printf 'runtime_dir=%s\n' "$runtime_dir" >>"$GITHUB_OUTPUT" printf 'guard_dir=%s\n' "$guard_dir" >>"$GITHUB_OUTPUT" printf 'helper_dir=%s\n' "$helper_directory" >>"$GITHUB_OUTPUT" printf 'node_dir=%s\n' "$node_directory" >>"$GITHUB_OUTPUT" printf 'podman_executable=%s\n' "$podman_executable" >>"$GITHUB_OUTPUT" printf 'containers_config=%s\n' "$containers_config" >>"$GITHUB_OUTPUT" printf 'runtime_directory_unit=%s\n' "$runtime_directory_unit" >>"$GITHUB_OUTPUT" printf 'storage_config=%s\n' "$storage_config" >>"$GITHUB_OUTPUT" printf 'user_manager_unit=%s\n' "$user_manager_unit" >>"$GITHUB_OUTPUT" - name: Prepare GPU resources with the NVIDIA API key id: gpu_resources env: ACCOUNT: ${{ steps.boundary.outputs.account }} ACCOUNT_GID: ${{ steps.boundary.outputs.gid }} ACCOUNT_UID: ${{ steps.boundary.outputs.uid }} ACCELERATION: ${{ matrix.case.acceleration }} ARCHITECTURE: ${{ matrix.case.architecture }} CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }} GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} INFERENCE: ${{ matrix.case.inference }} NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} PODMAN_EXECUTABLE: ${{ steps.boundary.outputs.podman_executable }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} shell: bash run: | set -euo pipefail umask 077 if [[ "$ACCELERATION" != "nvidia-gpu" ]]; then printf 'runner_contract=\n' >>"$GITHUB_OUTPUT" exit 0 fi [[ -n "$NVIDIA_API_KEY" ]] || { echo "::error::Native runtime GPU preparation requires the NVIDIA_API_KEY repository secret" >&2 exit 1 } uid="$(id -u "$ACCOUNT")" gid="$(id -g "$ACCOUNT")" [[ "$uid" == "$ACCOUNT_UID" && "$gid" == "$ACCOUNT_GID" ]] || { echo "::error::Qualification account identity changed before GPU resource preparation" >&2 exit 1 } storage_config_directory="$(dirname "$STORAGE_CONFIG")" runner_contract="${storage_config_directory}/runner-contract.json" registry_auth_directory="${storage_config_directory}/registry-auth" registry_auth_file="${registry_auth_directory}/auth.json" resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" model_directory="${resource_directory}/model" model_revision="7ae557604adf67be50417f59c2c2f167def9a775" model_name="Qwen/Qwen2.5-0.5B-Instruct" case "$ARCHITECTURE" in amd64) probe_image="nvcr.io/nvidia/k8s/cuda-sample@sha256:9855f4c8500addf185360474184b9efdaf4384284779aa9173dcd70164a4ae6f" nim_image="nvcr.io/nim/nvidia/model-free-nim@sha256:a0fdbecdf51792dadc48d284fde3199a58d5a2067007ad5b80319975fe81ce93" vllm_image="nvcr.io/nvidia/vllm@sha256:7be6c2f676c36059a494fe17254e69ae5c677535ba6191044e5fc8e42a91c773" ;; arm64) probe_image="nvcr.io/nvidia/k8s/cuda-sample@sha256:a54fdceac3bc2a8d177f07db942defc2f7237e18d07fca2ff00718ba5aee4940" nim_image="nvcr.io/nim/nvidia/model-free-nim@sha256:8342257b9744e9bc23a02e0f45badad6b88474727473965b4dc83e8fee45956a" vllm_image="nvcr.io/nvidia/vllm@sha256:447995cbb57e6c7cf792cab95e9852e5f62b5fb6d2f39e030fa4eda9a54eadb4" ;; *) echo "::error::GPU resource architecture is unsupported" >&2 exit 1 ;; esac [[ ! -e "$runner_contract" && ! -L "$runner_contract" ]] || { echo "::error::Run-owned GPU runner contract already exists" >&2 exit 1 } [[ ! -e "$registry_auth_directory" && ! -L "$registry_auth_directory" ]] || { echo "::error::Run-owned registry authentication directory already exists" >&2 exit 1 } sudo install -d --owner="$uid" --group="$gid" --mode=0700 "$registry_auth_directory" cleanup_registry_auth() { local result="$?" trap - EXIT if sudo test -e "$registry_auth_file" || sudo test -L "$registry_auth_file"; then sudo test -f "$registry_auth_file" && sudo test ! -L "$registry_auth_file" && [[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { echo "::error::Run-owned registry authentication file cleanup target is invalid" >&2 return 1 } sudo unlink "$registry_auth_file" fi sudo rmdir "$registry_auth_directory" || { echo "::error::Run-owned registry authentication directory was not empty" >&2 return 1 } return "$result" } trap cleanup_registry_auth EXIT printf '%s' "$NVIDIA_API_KEY" | sudo -u "$ACCOUNT" env -i \ CONTAINERS_CONF="$CONTAINERS_CONFIG" \ CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ PATH="$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ REGISTRY_AUTH_FILE="$registry_auth_file" \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ "$PODMAN_EXECUTABLE" login nvcr.io --username '$oauthtoken' --password-stdin >/dev/null sudo test -f "$registry_auth_file" && sudo test ! -L "$registry_auth_file" && [[ "$(sudo stat -c '%u:%g:%h' -- "$registry_auth_file")" == "${uid}:${gid}:1" ]] || { echo "::error::Run-owned registry authentication file is invalid" >&2 exit 1 } sudo chmod 0600 -- "$registry_auth_file" [[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { echo "::error::Run-owned registry authentication file permissions are invalid" >&2 exit 1 } image_to_pull="$probe_image" if [[ "$INFERENCE" == "nim" ]]; then image_to_pull="$nim_image" elif [[ "$INFERENCE" == "vllm" ]]; then image_to_pull="$vllm_image" fi images=("$probe_image") [[ "$image_to_pull" == "$probe_image" ]] || images+=("$image_to_pull") for image in "${images[@]}"; do sudo -u "$ACCOUNT" env -i \ CONTAINERS_CONF="$CONTAINERS_CONFIG" \ CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ REGISTRY_AUTH_FILE="$registry_auth_file" \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ "$PODMAN_EXECUTABLE" pull "$image" done cleanup_registry_auth trap - EXIT unset NVIDIA_API_KEY if [[ "$INFERENCE" == "nim" || "$INFERENCE" == "vllm" ]]; then [[ ! -e "$resource_directory" && ! -L "$resource_directory" ]] || { echo "::error::Run-owned GPU model resource already exists" >&2 exit 1 } sudo install -d --owner=root --group=root --mode=0711 "$resource_directory" sudo install -d --owner="$uid" --group="$gid" --mode=0700 "$model_directory" download_model_file() { local file="$1" local size="$2" local algorithm="$3" local digest="$4" local target="${model_directory}/${file}" sudo -u "$ACCOUNT" env -i \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ /usr/bin/curl \ --fail --location --proto '=https' --retry 3 --show-error --silent --tlsv1.2 \ --output "$target" \ "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/${model_revision}/${file}?download=true" sudo test -f "$target" && sudo test ! -L "$target" && [[ "$(sudo stat -c '%u:%g:%h:%s' -- "$target")" == "${uid}:${gid}:1:${size}" ]] || { echo "::error::Downloaded GPU model file metadata is invalid: $file" >&2 exit 1 } sudo chmod 0600 -- "$target" [[ "$(sudo stat -c '%u:%g:%a:%h:%s' -- "$target")" == "${uid}:${gid}:600:1:${size}" ]] || { echo "::error::Downloaded GPU model file permissions are invalid: $file" >&2 exit 1 } if [[ "$algorithm" == "sha256" ]]; then [[ "$(sudo sha256sum -- "$target" | cut -d' ' -f1)" == "$digest" ]] else [[ "$(sudo git hash-object --no-filters -- "$target")" == "$digest" ]] fi || { echo "::error::Downloaded GPU model file digest is invalid: $file" >&2 exit 1 } } download_model_file config.json 659 sha1 0dbb161213629a23f0fc00ef286e6b1e366d180f download_model_file generation_config.json 242 sha1 dfc11073787daf1b0f9c0f1499487ab5f4c93738 download_model_file merges.txt 1671839 sha1 20024bfe7c83998e9aeaf98a0cd6a2ce6306c2f0 download_model_file model.safetensors 988097824 sha256 fdf756fa7fcbe7404d5c60e26bff1a0c8b8aa1f72ced49e7dd0210fe288fb7fe download_model_file tokenizer.json 7031645 sha1 443909a61d429dff23010e5bddd28ff530edda00 download_model_file tokenizer_config.json 7305 sha1 07bfe0640cb5a0037f9322287fbfc682806cf672 download_model_file vocab.json 2776833 sha1 4783fe10ac3adce15ac8f358ef5462739852c569 for file in config.json generation_config.json merges.txt model.safetensors tokenizer.json tokenizer_config.json vocab.json; do sudo chown root:root "${model_directory}/${file}" sudo chmod 0444 "${model_directory}/${file}" done sudo chown root:root "$model_directory" sudo chmod 0555 "$model_directory" sudo chmod 0555 "$resource_directory" fi jq -n \ --arg architecture "$ARCHITECTURE" \ --arg gpuProbeImageRef "$probe_image" \ --arg model "$model_name" \ --arg modelPath "$model_directory" \ --arg modelRevision "$model_revision" \ --arg nimImageRef "$nim_image" \ --arg vllmImageRef "$vllm_image" ' { schemaVersion: 1, kind: "nemoclaw-native-runtime-qualification-runner-v1", architecture: $architecture, gpuProbeImageRef: $gpuProbeImageRef, nim: { imageRef: $nimImageRef, model: $model, modelPath: $modelPath, modelRevision: $modelRevision }, vllm: { imageRef: $vllmImageRef, model: $model, modelPath: $modelPath, modelRevision: $modelRevision } } ' | sudo tee "$runner_contract" >/dev/null sudo chown root:root "$runner_contract" sudo chmod 0444 "$runner_contract" [[ -f "$runner_contract" && ! -L "$runner_contract" && "$(stat -c '%u:%g:%a' "$runner_contract")" == "0:0:444" ]] || { echo "::error::Run-owned GPU runner contract is invalid" >&2 exit 1 } printf 'runner_contract=%s\n' "$runner_contract" >>"$GITHUB_OUTPUT" - name: Install locked candidate test dependencies without scripts env: ACCOUNT: ${{ steps.boundary.outputs.account }} CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} shell: bash run: | set -euo pipefail sudo -u "$ACCOUNT" env -i \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ /bin/bash --noprofile --norc -c ' set -euo pipefail cd "$1" for file in package.json package-lock.json; do [[ -f "$file" && ! -L "$file" && -O "$file" ]] || { echo "Candidate dependency manifest is missing or invalid: $file" >&2 exit 1 } done exec npm --prefix "$1" ci --ignore-scripts ' bash "$CANDIDATE_DIRECTORY" - name: Run the authenticated installer qualification env: ACCOUNT: ${{ steps.boundary.outputs.account }} ACCOUNT_GID: ${{ steps.boundary.outputs.gid }} ACCOUNT_UID: ${{ steps.boundary.outputs.uid }} ARCHITECTURE: ${{ matrix.case.architecture }} CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime CANDIDATE_SHA: ${{ matrix.source.candidateSha }} CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }} GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} INSTALLER_RECEIPT_PARENT: ${{ runner.temp }}/native-runtime-installer INSTALLER_SHA256: ${{ matrix.installerSha256 }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} RUNTIME_DIRECTORY_UNIT: ${{ steps.boundary.outputs.runtime_directory_unit }} STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} TRUSTED_USER_UNIT_PATH: /usr/lib/systemd/user:/lib/systemd/user USER_MANAGER_UNIT: ${{ steps.boundary.outputs.user_manager_unit }} shell: bash run: | set -euo pipefail install -d -m 0700 "$INSTALLER_RECEIPT_PARENT" sudo chown "$ACCOUNT_UID:$ACCOUNT_GID" "$INSTALLER_RECEIPT_PARENT" sudo -u "$ACCOUNT" env -i \ CONTAINERS_CONF="$CONTAINERS_CONFIG" \ CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ bash .qualification-workflow/scripts/checks/run-native-runtime-installer-qualification.sh \ --candidate-checkout "$CANDIDATE_DIRECTORY" \ --candidate-sha "$CANDIDATE_SHA" \ --installer-sha256 "$INSTALLER_SHA256" \ --architecture "$ARCHITECTURE" \ --artifact-dir "$INSTALLER_RECEIPT_PARENT/receipts" sudo systemctl stop "$USER_MANAGER_UNIT" "$RUNTIME_DIRECTORY_UNIT" ! systemctl is-active --quiet "$USER_MANAGER_UNIT" && ! systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || { echo "::error::Qualification systemd user lifecycle remained active after installer isolation" >&2 exit 1 } sudo pkill -KILL -u "$ACCOUNT_UID" 2>/dev/null || true sudo systemctl start "$USER_MANAGER_UNIT" systemctl is-active --quiet "$USER_MANAGER_UNIT" && systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || { echo "::error::Qualification systemd user lifecycle did not restart after installer isolation" >&2 exit 1 } sudo -u "$ACCOUNT" env -i \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ PATH=/usr/bin:/bin \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ /usr/bin/systemctl --user start dbus.socket sudo -u "$ACCOUNT" env -i \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ PATH=/usr/bin:/bin \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ /usr/bin/systemctl --user is-active --quiet dbus.socket || { echo "::error::Qualification systemd user bus socket unit did not restart after installer isolation" >&2 exit 1 } sudo /usr/bin/test -S "$RUNTIME_DIRECTORY/bus" && sudo /usr/bin/test ! -L "$RUNTIME_DIRECTORY/bus" && [[ "$(sudo stat -c '%u' -- "$RUNTIME_DIRECTORY/bus")" == "$ACCOUNT_UID" ]] && sudo -u "$ACCOUNT" /usr/bin/test -S "$RUNTIME_DIRECTORY/bus" || { echo "::error::Qualification systemd user bus is invalid or inaccessible after installer isolation" >&2 exit 1 } manager_environment="$(sudo -u "$ACCOUNT" env -i \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ PATH=/usr/bin:/bin \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ /usr/bin/systemctl --user show-environment)" tr ' ' '\n' <<<"$manager_environment" | grep -Fx -- "SYSTEMD_UNIT_PATH=$TRUSTED_USER_UNIT_PATH" >/dev/null || { echo "::error::Qualification systemd user manager lost the trusted unit path after installer isolation" >&2 exit 1 } sudo test -d "$INSTALLER_RECEIPT_PARENT/receipts" || { echo "::error::Installer receipt directory is missing or invalid" >&2 exit 1 } sudo test ! -L "$INSTALLER_RECEIPT_PARENT/receipts" || { echo "::error::Installer receipt directory must not be a symlink" >&2 exit 1 } - name: Execute the candidate qualification case without credentials env: ACCOUNT: ${{ steps.boundary.outputs.account }} ACCOUNT_GID: ${{ steps.boundary.outputs.gid }} ACCOUNT_UID: ${{ steps.boundary.outputs.uid }} CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }} GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }} HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }} NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} PODMAN_EXECUTABLE: ${{ steps.boundary.outputs.podman_executable }} QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }} RUNNER_CONTRACT: ${{ steps.gpu_resources.outputs.runner_contract }} RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }} RUNTIME_DIRECTORY_UNIT: ${{ steps.boundary.outputs.runtime_directory_unit }} STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }} USER_MANAGER_UNIT: ${{ steps.boundary.outputs.user_manager_unit }} shell: bash run: | set -euo pipefail live_test="test/e2e/live/native-runtime-qualification-case.test.ts" [[ -f "$CANDIDATE_DIRECTORY/$live_test" && ! -L "$CANDIDATE_DIRECTORY/$live_test" ]] || { echo "::error::Candidate commit does not provide the native runtime qualification case executor" >&2 exit 1 } receipt_directory="${RUNNER_TEMP}/native-runtime-case" install -d -m 0700 "$receipt_directory" sudo chown "$ACCOUNT_UID:$ACCOUNT_GID" "$receipt_directory" cd "$CANDIDATE_DIRECTORY" sudo -u "$ACCOUNT" env -i \ CI=true \ CONTAINERS_CONF="$CONTAINERS_CONFIG" \ CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \ E2E_DEFAULT_ENABLED=0 \ E2E_JOB=1 \ HOME="$QUALIFICATION_HOME" \ LANG=C.UTF-8 \ NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RECEIPT="$receipt_directory/execution.json" \ NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW="$QUALIFICATION_ROW" \ NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE="$PODMAN_EXECUTABLE" \ NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT="$RUNNER_CONTRACT" \ NEMOCLAW_RUN_LIVE_E2E=1 \ PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \ XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \ "$CANDIDATE_DIRECTORY/node_modules/.bin/vitest" run \ --config "$CANDIDATE_DIRECTORY/vitest.config.ts" \ --project e2e-live \ "$live_test" sudo systemctl stop "$USER_MANAGER_UNIT" "$RUNTIME_DIRECTORY_UNIT" ! systemctl is-active --quiet "$USER_MANAGER_UNIT" && ! systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || { echo "::error::Qualification systemd user lifecycle remained active after candidate execution" >&2 exit 1 } sudo pkill -KILL -u "$ACCOUNT_UID" 2>/dev/null || true - name: Verify Docker stayed unavailable shell: bash run: | set -euo pipefail ! systemctl is-active --quiet docker.service ! systemctl is-active --quiet docker.socket ! pgrep -x dockerd >/dev/null [[ ! -S /var/run/docker.sock && ! -S /run/docker.sock ]] - name: Validate receipts and emit bounded evidence env: EVIDENCE_DIRECTORY: ${{ runner.temp }}/native-runtime-evidence EXECUTION_RECEIPT_PATH: ${{ runner.temp }}/native-runtime-case/execution.json INSTALLER_RECEIPT_DIRECTORY: ${{ runner.temp }}/native-runtime-installer/receipts NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }} shell: bash run: | set -euo pipefail sudo --preserve-env=EVIDENCE_DIRECTORY,EXECUTION_RECEIPT_PATH,INSTALLER_RECEIPT_DIRECTORY,QUALIFICATION_ROW \ "$NODE_DIRECTORY/node" --no-warnings \ .qualification-workflow/tools/e2e/native-runtime-qualification-producer-evidence.mts sudo chown -R "$(id -u):$(id -g)" "$EVIDENCE_DIRECTORY" - name: Remove qualification resources if: always() env: ACCOUNT: ${{ steps.boundary.outputs.account }} ACCOUNT_CREATED: ${{ steps.boundary.outputs.account_created }} shell: bash run: | set -euo pipefail reported_account="${ACCOUNT:-}" reported_created="${ACCOUNT_CREATED:-}" ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" account="" uid="" gid="" if [[ -e "$ownership_marker" || -L "$ownership_marker" ]]; then [[ -f "$ownership_marker" && ! -L "$ownership_marker" && "$(stat -c '%u:%g:%a' "$ownership_marker")" == "0:0:400" ]] || { echo "::error::Qualification account ownership marker cleanup target is invalid" >&2 exit 1 } ownership="$(sudo cat "$ownership_marker")" [[ "$ownership" =~ ^nemoclawq:([0-9]+):([0-9]+)$ ]] || { echo "::error::Qualification account ownership marker content is invalid" >&2 exit 1 } account="nemoclawq" uid="${BASH_REMATCH[1]}" gid="${BASH_REMATCH[2]}" [[ -z "$reported_account" || "$reported_account" == "$account" ]] || { echo "::error::Qualification account output does not match its ownership marker" >&2 exit 1 } if getent passwd "$uid" >/dev/null && ! getent passwd "$account" >/dev/null; then echo "::error::Qualification account UID belongs to a different host account" >&2 exit 1 fi if getent passwd "$account" >/dev/null; then [[ "$(id -u "$account")" == "$uid" && "$(id -g "$account")" == "$gid" ]] || { echo "::error::Qualification account identity changed before cleanup" >&2 exit 1 } fi if getent group "$account" >/dev/null; then [[ "$(getent group "$account")" == "${account}:x:${gid}:" ]] || { echo "::error::Qualification private group identity changed before cleanup" >&2 exit 1 } elif getent passwd "$account" >/dev/null; then echo "::error::Qualification private group disappeared before cleanup" >&2 exit 1 fi runtime_dir="/run/user/${uid}" runtime_directory_unit="user-runtime-dir@${uid}.service" user_manager_unit="user@${uid}.service" user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d" user_manager_dropin="${user_manager_dropin_directory}/50-nemoclaw-native-runtime.conf" trusted_user_unit_path="/usr/lib/systemd/user:/lib/systemd/user" storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" apparmor_profile="${storage_config_directory}/podman.apparmor" pasta_apparmor_profile="${storage_config_directory}/pasta.apparmor" helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" pasta_executable="${helper_directory}/pasta" registry_auth_directory="${storage_config_directory}/registry-auth" registry_auth_file="${registry_auth_directory}/auth.json" runner_contract="${storage_config_directory}/runner-contract.json" resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" model_directory="${resource_directory}/model" sudo systemctl stop "$user_manager_unit" "$runtime_directory_unit" 2>/dev/null || true ! systemctl is-active --quiet "$user_manager_unit" && ! systemctl is-active --quiet "$runtime_directory_unit" || { echo "::error::Qualification systemd user lifecycle remained active during cleanup" >&2 exit 1 } if getent passwd "$account" >/dev/null; then sudo pkill -KILL -u "$uid" 2>/dev/null || true fi if [[ -e "$user_manager_dropin_directory" || -L "$user_manager_dropin_directory" ]]; then [[ -d "$user_manager_dropin_directory" && ! -L "$user_manager_dropin_directory" && "$(stat -c '%u:%g:%a' "$user_manager_dropin_directory")" == "0:0:755" ]] || { echo "::error::Qualification systemd user-manager drop-in directory cleanup target is invalid" >&2 exit 1 } [[ -f "$user_manager_dropin" && ! -L "$user_manager_dropin" && "$(stat -c '%u:%g:%a:%h' "$user_manager_dropin")" == "0:0:444:1" ]] || { echo "::error::Qualification systemd user-manager drop-in cleanup target is invalid" >&2 exit 1 } expected_user_manager_dropin="$(printf '[Service]\nEnvironment="SYSTEMD_UNIT_PATH=%s"' "$trusted_user_unit_path")" [[ "$(cat "$user_manager_dropin")" == "$expected_user_manager_dropin" ]] || { echo "::error::Qualification systemd user-manager drop-in content changed before cleanup" >&2 exit 1 } sudo unlink "$user_manager_dropin" sudo rmdir "$user_manager_dropin_directory" sudo systemctl daemon-reload fi if [[ -e "$storage_config_directory" || -L "$storage_config_directory" ]]; then [[ -d "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { echo "::error::Qualification storage configuration cleanup target is invalid" >&2 exit 1 } if [[ -e "$apparmor_profile" || -L "$apparmor_profile" ]]; then [[ -f "$apparmor_profile" && ! -L "$apparmor_profile" && "$(stat -c '%u:%g:%a' "$apparmor_profile")" == "0:0:444" ]] || { echo "::error::Qualification AppArmor profile cleanup target is invalid" >&2 exit 1 } sudo apparmor_parser -R "$apparmor_profile" sudo rm -f -- "$apparmor_profile" fi if [[ -e "$pasta_apparmor_profile" || -L "$pasta_apparmor_profile" ]]; then [[ -f "$pasta_apparmor_profile" && ! -L "$pasta_apparmor_profile" && "$(stat -c '%u:%g:%a' "$pasta_apparmor_profile")" == "0:0:444" ]] || { echo "::error::Qualification pasta AppArmor profile cleanup target is invalid" >&2 exit 1 } sudo apparmor_parser -R "$pasta_apparmor_profile" sudo rm -f -- "$pasta_apparmor_profile" fi if [[ -e "$registry_auth_directory" || -L "$registry_auth_directory" ]]; then [[ -d "$registry_auth_directory" && ! -L "$registry_auth_directory" && "$(stat -c '%u:%g:%a' "$registry_auth_directory")" == "${uid}:${gid}:700" ]] || { echo "::error::Qualification registry authentication directory cleanup target is invalid" >&2 exit 1 } if sudo test -e "$registry_auth_file" || sudo test -L "$registry_auth_file"; then sudo test -f "$registry_auth_file" && sudo test ! -L "$registry_auth_file" && [[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || { echo "::error::Qualification registry authentication file cleanup target is invalid" >&2 exit 1 } sudo unlink "$registry_auth_file" fi sudo rmdir "$registry_auth_directory" fi if [[ -e "$runner_contract" || -L "$runner_contract" ]]; then [[ -f "$runner_contract" && ! -L "$runner_contract" && "$(stat -c '%u:%g:%a' "$runner_contract")" == "0:0:444" ]] || { echo "::error::Qualification GPU runner contract cleanup target is invalid" >&2 exit 1 } sudo unlink "$runner_contract" fi if [[ -e "$storage_config_directory/containers.conf" || -L "$storage_config_directory/containers.conf" ]]; then [[ -f "$storage_config_directory/containers.conf" && ! -L "$storage_config_directory/containers.conf" && "$(stat -c '%u:%g:%a' "$storage_config_directory/containers.conf")" == "0:0:444" ]] || { echo "::error::Qualification containers configuration cleanup target is invalid" >&2 exit 1 } sudo unlink "$storage_config_directory/containers.conf" fi sudo rm -f -- "$storage_config_directory/storage.conf" sudo rmdir "$storage_config_directory" fi if [[ -e "$resource_directory" || -L "$resource_directory" ]]; then [[ -d "$resource_directory" && ! -L "$resource_directory" && ("$(stat -c '%u:%g:%a' "$resource_directory")" == "0:0:711" || "$(stat -c '%u:%g:%a' "$resource_directory")" == "0:0:555") ]] || { echo "::error::Qualification GPU resource directory cleanup target is invalid" >&2 exit 1 } if sudo test -e "$model_directory" || sudo test -L "$model_directory"; then model_mode="$(sudo stat -c '%u:%g:%a' -- "$model_directory")" sudo test -d "$model_directory" && sudo test ! -L "$model_directory" && [[ "$model_mode" == "${uid}:${gid}:700" || "$model_mode" == "0:0:555" ]] || { echo "::error::Qualification GPU model directory cleanup target is invalid" >&2 exit 1 } for file in config.json generation_config.json merges.txt model.safetensors tokenizer.json tokenizer_config.json vocab.json; do target="${model_directory}/${file}" if sudo test -e "$target" || sudo test -L "$target"; then file_mode="$(sudo stat -c '%u:%g:%a:%h' -- "$target")" sudo test -f "$target" && sudo test ! -L "$target" && [[ "$file_mode" == "${uid}:${gid}:600:1" || "$file_mode" == "0:0:444:1" ]] || { echo "::error::Qualification GPU model file cleanup target is invalid: $file" >&2 exit 1 } sudo unlink "$target" fi done sudo rmdir "$model_directory" || { echo "::error::Qualification GPU model directory contains unexpected entries" >&2 exit 1 } fi sudo rmdir "$resource_directory" || { echo "::error::Qualification GPU resource directory contains unexpected entries" >&2 exit 1 } fi if [[ -e "$podman_executable" || -L "$podman_executable" ]]; then [[ -f "$podman_executable" && ! -L "$podman_executable" && "$(stat -c '%u:%g:%a' "$podman_executable")" == "0:0:555" ]] || { echo "::error::Qualification Podman executable cleanup target is invalid" >&2 exit 1 } sudo rm -f -- "$podman_executable" fi if [[ -e "$helper_directory" || -L "$helper_directory" ]]; then [[ -d "$helper_directory" && ! -L "$helper_directory" && "$(stat -c '%u:%g:%a' "$helper_directory")" == "0:0:555" ]] || { echo "::error::Qualification helper directory cleanup target is invalid" >&2 exit 1 } [[ -f "$pasta_executable" && ! -L "$pasta_executable" && "$(stat -c '%u:%g:%a' "$pasta_executable")" == "0:0:555" ]] || { echo "::error::Qualification pasta executable cleanup target is invalid" >&2 exit 1 } sudo rm -f -- "$pasta_executable" sudo rmdir "$helper_directory" fi if [[ -e "$runtime_dir" || -L "$runtime_dir" ]]; then echo "::error::Qualification runtime directory remains after its systemd cleanup" >&2 exit 1 fi if getent passwd "$account" >/dev/null; then sudo userdel --remove "$account" fi if getent group "$account" >/dev/null; then sudo groupdel "$account" fi if getent passwd "$account" >/dev/null; then echo "::error::Qualification account still exists after cleanup" >&2 exit 1 fi if getent group "$account" >/dev/null; then echo "::error::Qualification private group still exists after cleanup" >&2 exit 1 fi if grep -q "^${account}:" /etc/subuid /etc/subgid; then echo "::error::Qualification subordinate-ID authorization remains after cleanup" >&2 exit 1 fi [[ ! -e "/run/user/${uid}" && ! -L "/run/user/${uid}" ]] || { echo "::error::Qualification runtime directory remains after cleanup" >&2 exit 1 } storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}" [[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || { echo "::error::Qualification storage configuration remains after cleanup" >&2 exit 1 } [[ ! -e "$podman_executable" && ! -L "$podman_executable" ]] || { echo "::error::Qualification Podman executable remains after cleanup" >&2 exit 1 } [[ ! -e "$helper_directory" && ! -L "$helper_directory" ]] || { echo "::error::Qualification helper directory remains after cleanup" >&2 exit 1 } [[ ! -e "$resource_directory" && ! -L "$resource_directory" ]] || { echo "::error::Qualification GPU resource directory remains after cleanup" >&2 exit 1 } sudo rm -f -- "$ownership_marker" elif [[ -n "$reported_account" || "$reported_created" == "true" ]]; then echo "::error::Qualification account output exists without its ownership marker" >&2 exit 1 fi [[ ! -e "$ownership_marker" && ! -L "$ownership_marker" ]] || { echo "::error::Qualification account ownership marker remains after cleanup" >&2 exit 1 } - name: Upload the qualification case evidence if: success() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: ${{ matrix.artifactName }} path: ${{ runner.temp }}/native-runtime-evidence/ native-runtime-qualification-producer-aggregate: name: Aggregate native runtime qualification evidence needs: [ generate-matrix, native-runtime-qualification-producer-plan, native-runtime-qualification-producer, ] if: ${{ always() && needs.native-runtime-qualification-producer-plan.result == 'success' && needs.native-runtime-qualification-producer.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }} runs-on: ubuntu-24.04 timeout-minutes: 15 permissions: actions: read contents: read pull-requests: read steps: - name: Check out the qualification aggregator uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ github.repository }} ref: ${{ github.workflow_sha }} path: .qualification-aggregate persist-credentials: false sparse-checkout: | .github/actions/setup-reviewed-npm ci/reviewed-npm-audit.json scripts/lib/npm-diagnostics.sh scripts/lib/reviewed-npm-audit.mts src/lib/onboard/runtime-provider/native-qualification-authority.ts test/e2e/registry/native-runtime-qualification.ts tools/e2e/native-runtime-qualification-producer-plan.mts tools/e2e/native-runtime-qualification-producer-aggregate.mts sparse-checkout-cone-mode: false - name: Download the exact case evidence cohort uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: native-runtime-qualification-evidence-${{ inputs.checkout_sha }}-* path: ${{ runner.temp }}/native-runtime-case-artifacts merge-multiple: false - name: Resolve this aggregate job identity id: aggregate-job env: GH_TOKEN: ${{ github.token }} PRODUCER_RUN_ATTEMPT: ${{ github.run_attempt }} PRODUCER_RUN_ID: ${{ github.run_id }} shell: bash run: | set -euo pipefail jobs="$(gh api --method GET \ "repos/${GITHUB_REPOSITORY}/actions/runs/${PRODUCER_RUN_ID}/attempts/${PRODUCER_RUN_ATTEMPT}/jobs" \ -f per_page=100)" total_count="$(jq -er '.total_count | select(type == "number" and . >= 1)' <<<"$jobs")" || { echo "::error::Aggregate job lookup returned an invalid job count" >&2 exit 1 } (( total_count <= 100 )) || { echo "::error::Aggregate job lookup exceeds the bounded 100-job page" >&2 exit 1 } job_id="$(jq -er \ --arg name 'Aggregate native runtime qualification evidence' \ --argjson runId "$PRODUCER_RUN_ID" \ --argjson attempt "$PRODUCER_RUN_ATTEMPT" ' [.jobs[] | select( .name == $name and .run_id == $runId and .run_attempt == $attempt and .status == "in_progress" )] | select(length == 1) | .[0].id ' <<<"$jobs")" || { echo "::error::Could not resolve one in-progress aggregate job identity" >&2 exit 1 } [[ "$job_id" =~ ^[1-9][0-9]{0,19}$ ]] || { echo "::error::Aggregate job identity is invalid" >&2 exit 1 } printf 'job_id=%s\n' "$job_id" >>"$GITHUB_OUTPUT" - name: Set up Node for qualification aggregation uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24.22.1 - name: Install reviewed npm uses: ./.qualification-aggregate/.github/actions/setup-reviewed-npm - name: Validate and aggregate all 24 case receipts working-directory: .qualification-aggregate env: AGGREGATE_JOB_ID: ${{ steps.aggregate-job.outputs.job_id }} CASE_ARTIFACT_ROOT: ${{ runner.temp }}/native-runtime-case-artifacts EVIDENCE_DIRECTORY: ${{ runner.temp }}/native-runtime-aggregate QUALIFICATION_PLAN: ${{ needs.native-runtime-qualification-producer-plan.outputs.matrix }} run: >- node --no-warnings tools/e2e/native-runtime-qualification-producer-aggregate.mts - name: Upload aggregate evidence uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: native-runtime-qualification-${{ inputs.checkout_sha }} path: ${{ runner.temp }}/native-runtime-aggregate/ if-no-files-found: error retention-days: 30 compression-level: 9 staging-brev-launchable: name: Exact staging Brev Launchable needs: generate-matrix if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }} runs-on: ubuntu-latest timeout-minutes: 180 permissions: contents: read concurrency: group: staging-brev-launchable-cpu queue: max cancel-in-progress: false env: CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} E2E_JOB: "1" E2E_GATEWAY_RUNTIMES: docker INSTANCE_NAME: nclaw-e2e-${{ github.run_id }}-${{ github.run_attempt }} E2E_AGENT_RUNTIME: "openclaw" E2E_OBSERVABLE_OUTCOME: "The staging image boots and completes the full E2E scenario" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Brev Launchable Docker host; NVIDIA hosted inference" steps: - name: Checkout trusted Launchable lane uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.workflow_sha }} persist-credentials: false sparse-checkout: | tools/e2e/brev-launchable-e2e.sh sparse-checkout-cone-mode: false - id: workspace name: Prepare the trusted lane env: BREV_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.BREV_API_KEY || '' }} BREV_CLI_SHA256: d4aa49db1716f10308a6587778a676a0c0076bd48a212d86a421ab9550bc8f32 BREV_CLI_VERSION: 0.6.334 BREV_ORG_ID: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.BREV_ORG_ID || '' }} run: | set -euo pipefail work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-launchable-e2e.XXXXXX")" chmod 700 "$work_dir" archive="${RUNNER_TEMP}/brev-cli.tar.gz" curl -fsSL -o "$archive" "https://github.com/brevdev/brev-cli/releases/download/v${BREV_CLI_VERSION}/brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz" printf '%s %s\n' "$BREV_CLI_SHA256" "$archive" | sha256sum -c - tar -xzf "$archive" -C "${RUNNER_TEMP}" brev && sudo install -m 0755 "${RUNNER_TEMP}/brev" /usr/local/bin/brev brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID" printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" - name: Build, deploy, verify, test, and clean up env: BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }} GH_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN || '' }} NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NVIDIA_API_KEY || '' }} WORK_DIR: ${{ steps.workspace.outputs.work_dir }} run: tools/e2e/brev-launchable-e2e.sh - name: Upload Launchable evidence if: ${{ always() && steps.workspace.outputs.work_dir != '' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: staging-brev-launchable-${{ env.CANDIDATE_SHA }}-${{ github.run_id }}-${{ github.run_attempt }} path: | ${{ steps.workspace.outputs.work_dir }}/lane.log ${{ steps.workspace.outputs.work_dir }}/launchable-e2e.json ${{ steps.workspace.outputs.work_dir }}/full-e2e.log ${{ steps.workspace.outputs.work_dir }}/cleanup.json staging-brev-launchable-identity: name: Exact staging Brev Launchable identity needs: generate-matrix if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && inputs.jobs == 'staging-brev-launchable-identity' && inputs.targets == '' }} runs-on: ubuntu-latest timeout-minutes: 180 permissions: contents: read concurrency: group: staging-brev-launchable-cpu queue: max cancel-in-progress: false env: CANDIDATE_SHA: ${{ github.sha }} E2E_DEFAULT_ENABLED: "0" E2E_JOB: "1" E2E_GATEWAY_RUNTIMES: agnostic INSTANCE_NAME: nclaw-identity-${{ github.run_id }}-${{ github.run_attempt }} E2E_AGENT_RUNTIME: "none" E2E_OBSERVABLE_OUTCOME: "The staging image boots, passes the SSH access probe, and matches the baked runtime identity" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Brev Launchable host; no inference endpoint" steps: - name: Checkout trusted Launchable identity lane uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.workflow_sha }} persist-credentials: false sparse-checkout: | tools/e2e/brev-launchable-e2e.sh sparse-checkout-cone-mode: false - id: workspace name: Prepare the trusted identity lane env: BREV_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && secrets.BREV_API_KEY || '' }} BREV_CLI_SHA256: d4aa49db1716f10308a6587778a676a0c0076bd48a212d86a421ab9550bc8f32 BREV_CLI_VERSION: 0.6.334 BREV_ORG_ID: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && secrets.BREV_ORG_ID || '' }} run: | set -euo pipefail install -d -m 0700 "$HOME/.brev" work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-launchable-identity.XXXXXX")" chmod 700 "$work_dir" archive="${RUNNER_TEMP}/brev-cli.tar.gz" curl -fsSL -o "$archive" "https://github.com/brevdev/brev-cli/releases/download/v${BREV_CLI_VERSION}/brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz" printf '%s %s\n' "$BREV_CLI_SHA256" "$archive" | sha256sum -c - tar -xzf "$archive" -C "${RUNNER_TEMP}" brev && sudo install -m 0755 "${RUNNER_TEMP}/brev" /usr/local/bin/brev brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID" printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" - name: Build, boot, and verify identity timeout-minutes: 150 env: BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }} GH_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN || '' }} NEMOCLAW_BREV_DEFER_CLEANUP: "1" NEMOCLAW_BREV_LAUNCHABLE_IDENTITY_ONLY: "1" WORK_DIR: ${{ steps.workspace.outputs.work_dir }} run: tools/e2e/brev-launchable-e2e.sh - name: Verify identity workspace cleanup if: ${{ always() && steps.workspace.outputs.work_dir != '' }} timeout-minutes: 15 env: BREV_CREATE_RECONCILE_SECONDS: "120" BREV_DELETE_TIMEOUT_SECONDS: "600" POLL_SECONDS: "15" WORK_DIR: ${{ steps.workspace.outputs.work_dir }} run: tools/e2e/brev-launchable-e2e.sh cleanup-owned-workspace - name: Remove Brev API credentials if: always() run: | set -euo pipefail credentials="$HOME/.brev/credentials.json" rm -f -- "$credentials" test ! -e "$credentials" - name: Upload Launchable identity evidence if: ${{ always() && steps.workspace.outputs.work_dir != '' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: staging-brev-launchable-identity-${{ env.CANDIDATE_SHA }}-${{ github.run_id }}-${{ github.run_attempt }} path: | ${{ steps.workspace.outputs.work_dir }}/lane.log ${{ steps.workspace.outputs.work_dir }}/launchable-identity.json ${{ steps.workspace.outputs.work_dir }}/cleanup.json live: name: ${{ matrix.label }} needs: [base-image-publication, generate-matrix, package-openshell-sdk] if: ${{ needs.generate-matrix.outputs.matrix != '[]' }} runs-on: ${{ matrix.runner }} timeout-minutes: ${{ matrix.timeout_minutes }} strategy: fail-fast: false matrix: include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} env: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live E2E_EXECUTION_ID: ${{ matrix.execution_id }} E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} NEMOCLAW_LANGCHAIN_DEEPAGENTS_CODE_SANDBOX_BASE_IMAGE_REF: ${{ needs.generate-matrix.outputs.workload_source == 'managed-image' && needs.base-image-publication.outputs.dcode_base_ref || '' }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - name: Set up pinned v1 compatibility toolchain uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 with: toolchain: 1.98.1 cache: false rustflags: "" # Keep only the credential-bearing step anchored. Cleanup mappings stay # explicit because strict YAML decoders reject 100 or more aliases here. - &dockerhub-auth name: Authenticate to Docker Hub uses: NVIDIA/NemoClaw/.github/actions/docker-auth-setup@05fa6b810017752ab21148cb7e9d82d12a88c92f with: auth-required: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && '1' || '0' }} username: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} token: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} - name: Configure live E2E trace directory env: TARGET_ID: ${{ matrix.id }} shell: bash run: | set -euo pipefail printf 'NEMOCLAW_TRACE_DIR=%s\n' "${RUNNER_TEMP}/nemoclaw-e2e-traces/${TARGET_ID}" >> "${GITHUB_ENV}" # invalidState: the selected PR-modifiable TUI check needs a PTY driver, # but the fixed GitHub-hosted runner image does not provide expect. # sourceBoundary: privileged host setup runs from the first-party # host-dependency-setup action pinned to an immutable full SHA, never the # PR-controlled target ref; the check only consumes expect without privilege. # whyNotSourceFix: GitHub-hosted jobs cannot use a repository-owned host # image, and caching privileged dpkg state between clean runners is not # supported. # regressionTest: the workflow-boundary suite pins this target, condition, # ordering, action provenance, and package mapping. # removalCondition: remove the install when the hosted runner supplies # expect or the acceptance check no longer requires a PTY. - name: Install Deep Agents Code TUI host dependencies if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }} uses: NVIDIA/NemoClaw/.github/actions/host-dependency-setup@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: packages: expect # Configure NEMOCLAW_TRACE_DIR before workspace prep so every child # command writes raw traces under runner temp, never under upload roots. - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" install-dependencies: "false" - name: Install reviewed OpenShell SDK archive without package credentials uses: NVIDIA/NemoClaw/.github/actions/install-reviewed-openshell-sdk@342c8f0ab115febb905eb4729d32478f22207589 - name: Record immutable Deep Agents Code base evidence if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' && needs.generate-matrix.outputs.workload_source == 'managed-image' && needs.base-image-publication.outputs.managed_image_catalog == '' }} env: BASE_CONTRACT: ${{ needs.base-image-publication.outputs.dcode_base_contract }} CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} TARGET_ID: ${{ matrix.id }} TARGET_LABEL: ${{ matrix.label }} shell: bash run: | set -euo pipefail evidence_dir="${E2E_ARTIFACT_DIR}/${TARGET_ID}" install -d -m 0700 "${evidence_dir}" jq -e -n \ --arg candidateSha "${CANDIDATE_SHA}" \ --argjson baseContract "${BASE_CONTRACT}" \ '{contractVersion: 1, candidateSha: $candidateSha, base: $baseContract} | select( (.candidateSha | test("^[0-9a-f]{40}$")) and (.base.reference | test("@sha256:[0-9a-f]{64}$")) )' >"${evidence_dir}/dcode-base-image.json" test_artifact_name="$(node -e ' const slug = process.argv[1] .trim() .toLowerCase() .replace(/[^a-z0-9._-]+/g, "-") .replace(/^-+|-+$/g, ""); process.stdout.write(slug || "unnamed-test"); ' "${TARGET_LABEL}")" test_evidence_dir="${E2E_ARTIFACT_DIR}/${test_artifact_name}" install -d -m 0700 "${test_evidence_dir}" install -m 0600 \ "${evidence_dir}/dcode-base-image.json" \ "${test_evidence_dir}/dcode-base-image.json" - name: Restore exact-commit CLI artifact uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@3c6d991640414a2d78f95f629d67194d44829554 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Prepare native Podman E2E runtime uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} # invalidState: a profile plugin installed with --no-deps can import even # when an incomplete base image omitted its required upstream packages. # sourceBoundary: this trusted workflow scopes the repo-owned stripped-base # build to the exact DCode target; the production Dockerfile must reject it # at the isolated import gate before its later dependency-consistency check. # whyNotSourceFix: dependency completeness belongs to the hash-locked base; # resolving dependencies during local plugin install would duplicate that # trust boundary, so the regression constructs the invalid input instead. # regressionTest: workflow-boundary tests pin the target, script, and # ordering; build-gate tests pin the base build and failure contract. # removalCondition: remove only if package installation no longer uses # --no-deps or an equivalent earlier build gate proves both imports. - name: Verify DCode profile import gate rejects missing base dependencies if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' && matrix.runtime_provider == 'docker' }} shell: bash run: bash scripts/check-dcode-profile-import-gate.sh - name: Run live E2E tests env: E2E_TARGET_ID: ${{ matrix.id }} NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} TARGET_ID: ${{ matrix.id }} run: | set -euo pipefail npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/registry-targets.test.ts --selector "^${TARGET_ID}:" # The sanitizer reads raw traces only after checking the workflow-owned # runner-temp path, then writes the timing-only file into upload roots. - name: Build trusted live E2E timing summary if: always() env: TARGET_ID: ${{ matrix.id }} shell: bash run: | set -euo pipefail NEMOCLAW_TRACE_DIR="${NEMOCLAW_TRACE_DIR:-}" expected_trace_dir="${RUNNER_TEMP}/nemoclaw-e2e-traces/${TARGET_ID}" if [ -z "${RUNNER_TEMP}" ] || [ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]; then echo "::error title=E2E trace sanitization refused::NEMOCLAW_TRACE_DIR does not match its workflow-owned RUNNER_TEMP path. No raw traces were read or uploaded. Correct the trace path configuration before rerunning." >&2 printf 'Expected trace path: %s\n' "${expected_trace_dir}" >&2 exit 1 fi python3 scripts/e2e/sanitize-trace-timing.py \ "${NEMOCLAW_TRACE_DIR}" \ "${E2E_ARTIFACT_DIR}/${TARGET_ID}" # Cleanup intentionally runs after sanitization and before upload so raw # trace JSON never becomes part of the uploaded artifact surface. - name: Delete raw live E2E traces if: always() env: TARGET_ID: ${{ matrix.id }} shell: bash run: | set -euo pipefail NEMOCLAW_TRACE_DIR="${NEMOCLAW_TRACE_DIR:-}" expected_trace_dir="${RUNNER_TEMP}/nemoclaw-e2e-traces/${TARGET_ID}" if [ -z "${RUNNER_TEMP}" ] || [ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]; then echo "::error title=E2E trace cleanup refused::NEMOCLAW_TRACE_DIR does not match its workflow-owned RUNNER_TEMP path. No files were deleted. Correct the trace path configuration before rerunning." >&2 printf 'Expected trace path: %s\n' "${expected_trace_dir}" >&2 exit 1 fi rm -rf -- "${NEMOCLAW_TRACE_DIR}" - name: Summarize artifacts if: always() env: TARGET_ID: ${{ matrix.id }} TARGET_LABEL: ${{ matrix.label }} run: | python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY" import json import os from pathlib import Path root = Path(os.environ["E2E_ARTIFACT_DIR"]) / os.environ["TARGET_ID"] plan_path = root / "run-plan.json" print("## E2E Targets") print() print("- Project: `e2e-live`") print(f"- Target: `{os.environ['TARGET_ID']}`") print(f"- Label: `{os.environ['TARGET_LABEL']}`") print(f"- Artifact root: `{root}`") print() print("| Target | Manifest | Expected state | Suites | Phases |") print("| --- | --- | --- | --- | --- |") if plan_path.exists(): plan = json.loads(plan_path.read_text(encoding="utf-8")) suites = ", ".join(plan.get("suiteIds") or []) or "(none)" phases = ", ".join(plan.get("phases") or []) or "(none)" print( "| " f"`{plan.get('targetId') or os.environ['TARGET_ID']}` | " f"`{plan.get('manifestPath') or 'not-yet-defined'}` | " f"`{plan.get('expectedStateId') or 'not-yet-defined'}` | " f"{suites} | {phases} |" ) else: print( "| " f"`{os.environ['TARGET_ID']}` | `(missing run-plan.json)` | " "`(missing run-plan.json)` | `(missing)` | `(missing)` |" ) PY - name: Require automatic config export evidence if: ${{ success() }} shell: bash run: | set -euo pipefail evidence="e2e-artifacts/live/${{ matrix.id }}/config-export-evidence.v1.json" yaml="e2e-artifacts/live/${{ matrix.id }}/config-export.yaml" test -f "$evidence" jq -e '.passed == true' "$evidence" >/dev/null case "$(jq -er '.classification' "$evidence")" in success) test -f "$yaml" jq -e '.consumer.passed == true' "$evidence" >/dev/null jq -er '.consumer.revision | strings | select(test("^[0-9a-f]{40}$"))' "$evidence" >/dev/null expected_sha="$(jq -er '.export.sha256 | strings | select(test("^[0-9a-f]{64}$"))' "$evidence")" actual_sha="$(sha256sum -- "$yaml")" actual_sha="${actual_sha%% *}" test "$actual_sha" = "$expected_sha" ;; expected-refusal|no-usable-sandbox) ;; *) exit 1 ;; esac - name: Upload E2E artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-${{ matrix.execution_id }} path: | e2e-artifacts/live/${{ matrix.id }}/run-plan.json e2e-artifacts/live/${{ matrix.id }}/target.json e2e-artifacts/live/${{ matrix.id }}/target-result.json e2e-artifacts/live/${{ matrix.id }}/test-progress.json e2e-artifacts/live/${{ matrix.id }}/environment.result.json e2e-artifacts/live/${{ matrix.id }}/onboarding.result.json e2e-artifacts/live/${{ matrix.id }}/state-validation.result.json e2e-artifacts/live/${{ matrix.id }}/config-export-evidence.v1.json e2e-artifacts/live/${{ matrix.id }}/config-export.yaml e2e-artifacts/live/${{ matrix.id }}/dcode-base-image.json e2e-artifacts/live/${{ matrix.id }}/cloud-onboard-trace-timing-summary.json e2e-artifacts/live/${{ matrix.id }}/onboard-progress-budget.json e2e-artifacts/live/risk-signal.json e2e-artifacts/live/${{ matrix.id }}/actions/ e2e-artifacts/live/${{ matrix.id }}/logs/ e2e-artifacts/live/${{ matrix.id }}/shell/ - name: Restore Docker CLI after native Podman E2E if: ${{ always() && matrix.runtime_provider == 'podman' }} uses: NVIDIA/NemoClaw/.github/actions/restore-native-podman-e2e@9650336899bf836db5844381a97cbc2b0fe4a2b8 with: enabled: "true" - name: Clean up Docker auth if: always() shell: bash run: bash .github/scripts/docker-auth-cleanup.sh # Credential-free tests opt in with a tag beside the test. Discovery supplies # only a validated test ID, file, and Vitest project; this E2E workflow owns # the shared job's runner, setup, timeout, permissions, and artifact policy. shared-e2e: name: Shared E2E (${{ matrix.execution_id }}) needs: [generate-matrix, package-openshell-sdk] if: ${{ needs.generate-matrix.outputs.test_matrix != '[]' }} runs-on: ubuntu-latest timeout-minutes: 15 strategy: fail-fast: false matrix: include: ${{ fromJSON(needs.generate-matrix.outputs.test_matrix) }} env: CHECK_DOC_LINKS_REMOTE: "0" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/${{ matrix.execution_id }} E2E_EXECUTION_ID: ${{ matrix.execution_id }} E2E_TARGET_ID: ${{ matrix.id }} NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" install-dependencies: "false" - name: Install reviewed OpenShell SDK archive without package credentials uses: NVIDIA/NemoClaw/.github/actions/install-reviewed-openshell-sdk@342c8f0ab115febb905eb4729d32478f22207589 - name: Restore exact-commit CLI artifact uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@3c6d991640414a2d78f95f629d67194d44829554 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Prepare native Podman E2E runtime uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} - name: Run tagged credential-free test env: TEST_FILE: ${{ matrix.file }} TEST_PROJECT: ${{ matrix.project }} run: | set -euo pipefail npx vitest run --project "${TEST_PROJECT}" "${TEST_FILE}" \ --tags-filter=e2e/credential-free \ --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload test artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-${{ matrix.execution_id }} path: e2e-artifacts/live/${{ matrix.execution_id }}/ - name: Restore Docker CLI after native Podman E2E if: ${{ always() && matrix.runtime_provider == 'podman' }} uses: NVIDIA/NemoClaw/.github/actions/restore-native-podman-e2e@9650336899bf836db5844381a97cbc2b0fe4a2b8 with: enabled: "true" catalogue-hosted-inference: name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix, package-openshell-sdk] if: ${{ needs.generate-matrix.outputs.catalogue_hosted_inference_matrix != '[]' }} strategy: fail-fast: false matrix: include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_hosted_inference_matrix) }} uses: ./.github/workflows/e2e-standard-profile.yaml with: candidate_repository: ${{ inputs.checkout_repository || github.repository }} candidate_sha: ${{ inputs.checkout_sha || github.sha }} runtime_provider: ${{ matrix.runtime_provider }} execution_id: ${{ matrix.execution_id }} coverage_variant: ${{ matrix.coverage_variant }} risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} openshell_sdk_artifact_name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }} managed_image_catalog: ${{ needs.base-image-publication.outputs.managed_image_catalog }} workload_source: ${{ needs.generate-matrix.outputs.workload_source }} credential_boundary: selected hosted inference API key target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} hosted_inference_model: ${{ vars[matrix.model_env] }} runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }} checkout_sha: ${{ inputs.checkout_sha }} workflow_sha: ${{ inputs.workflow_sha }} test_file: ${{ matrix.test_file }} timeout_minutes: ${{ matrix.timeout_minutes }} install_mode: ${{ matrix.install_mode }} install_non_interactive: ${{ matrix.install_non_interactive }} restore_cli: ${{ matrix.restore_cli }} cloudflared: ${{ matrix.cloudflared }} host_packages: ${{ matrix.host_packages }} host_preparation: ${{ matrix.host_preparation }} runner_comparison: ${{ matrix.runner_comparison }} compatible_api_key: ${{ matrix.compatible_api_key }} github_token: false shard: ${{ matrix.shard }} artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.ref == 'refs/heads/main' && github.workflow_ref == 'NVIDIA/NemoClaw/.github/workflows/e2e.yaml@refs/heads/main' && github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: DOCKERHUB_USERNAME: ${{ github.ref == 'refs/heads/main' && github.workflow_ref == 'NVIDIA/NemoClaw/.github/workflows/e2e.yaml@refs/heads/main' && github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} DOCKERHUB_TOKEN: ${{ github.ref == 'refs/heads/main' && github.workflow_ref == 'NVIDIA/NemoClaw/.github/workflows/e2e.yaml@refs/heads/main' && github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} HOSTED_INFERENCE_API_KEY: ${{ github.ref == 'refs/heads/main' && github.workflow_ref == 'NVIDIA/NemoClaw/.github/workflows/e2e.yaml@refs/heads/main' && github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && matrix.id == 'hosted-inference-openrouter' && secrets.OPENROUTER_API_KEY || '' }} catalogue-standard: name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix, package-openshell-sdk] if: ${{ needs.generate-matrix.outputs.catalogue_standard_matrix != '[]' }} strategy: fail-fast: false matrix: include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_standard_matrix) }} uses: ./.github/workflows/e2e-standard-profile.yaml with: candidate_repository: ${{ inputs.checkout_repository || github.repository }} candidate_sha: ${{ inputs.checkout_sha || github.sha }} runtime_provider: ${{ matrix.runtime_provider }} execution_id: ${{ matrix.execution_id }} coverage_variant: ${{ matrix.coverage_variant }} risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} openshell_sdk_artifact_name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }} managed_image_catalog: ${{ needs.base-image-publication.outputs.managed_image_catalog }} workload_source: ${{ needs.generate-matrix.outputs.workload_source }} credential_boundary: no provider credential target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }} checkout_sha: ${{ inputs.checkout_sha }} workflow_sha: ${{ inputs.workflow_sha }} test_file: ${{ matrix.test_file }} timeout_minutes: ${{ matrix.timeout_minutes }} install_mode: ${{ matrix.install_mode }} install_non_interactive: ${{ matrix.install_non_interactive }} restore_cli: ${{ matrix.restore_cli }} cloudflared: ${{ matrix.cloudflared }} host_packages: ${{ matrix.host_packages }} host_preparation: ${{ matrix.host_preparation }} runner_comparison: ${{ matrix.runner_comparison }} compatible_api_key: ${{ matrix.compatible_api_key }} github_token: false shard: ${{ matrix.shard }} artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} catalogue-nvidia-api: name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix, package-openshell-sdk] if: ${{ needs.generate-matrix.outputs.catalogue_nvidia_api_matrix != '[]' }} strategy: fail-fast: false matrix: include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_nvidia_api_matrix) }} uses: ./.github/workflows/e2e-standard-profile.yaml with: candidate_repository: ${{ inputs.checkout_repository || github.repository }} candidate_sha: ${{ inputs.checkout_sha || github.sha }} runtime_provider: ${{ matrix.runtime_provider }} execution_id: ${{ matrix.execution_id }} coverage_variant: ${{ matrix.coverage_variant }} risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} openshell_sdk_artifact_name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }} managed_image_catalog: ${{ needs.base-image-publication.outputs.managed_image_catalog }} workload_source: ${{ needs.generate-matrix.outputs.workload_source }} credential_boundary: NVIDIA API key target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }} checkout_sha: ${{ inputs.checkout_sha }} workflow_sha: ${{ inputs.workflow_sha }} test_file: ${{ matrix.test_file }} timeout_minutes: ${{ matrix.timeout_minutes }} install_mode: ${{ matrix.install_mode }} install_non_interactive: ${{ matrix.install_non_interactive }} restore_cli: ${{ matrix.restore_cli }} cloudflared: ${{ matrix.cloudflared }} host_packages: ${{ matrix.host_packages }} host_preparation: ${{ matrix.host_preparation }} runner_comparison: ${{ matrix.runner_comparison }} compatible_api_key: ${{ matrix.compatible_api_key }} github_token: false shard: ${{ matrix.shard }} artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} NVIDIA_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_API_KEY || '' }} catalogue-nvidia-inference: name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix, package-openshell-sdk] if: ${{ needs.generate-matrix.outputs.catalogue_nvidia_inference_matrix != '[]' }} strategy: fail-fast: false matrix: include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_nvidia_inference_matrix) }} uses: ./.github/workflows/e2e-standard-profile.yaml with: candidate_repository: ${{ inputs.checkout_repository || github.repository }} candidate_sha: ${{ inputs.checkout_sha || github.sha }} runtime_provider: ${{ matrix.runtime_provider }} execution_id: ${{ matrix.execution_id }} coverage_variant: ${{ matrix.coverage_variant }} risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} openshell_sdk_artifact_name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }} managed_image_catalog: ${{ needs.base-image-publication.outputs.managed_image_catalog }} workload_source: ${{ needs.generate-matrix.outputs.workload_source }} credential_boundary: NVIDIA inference API key target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }} checkout_sha: ${{ inputs.checkout_sha }} workflow_sha: ${{ inputs.workflow_sha }} test_file: ${{ matrix.test_file }} timeout_minutes: ${{ matrix.timeout_minutes }} install_mode: ${{ matrix.install_mode }} install_non_interactive: ${{ matrix.install_non_interactive }} restore_cli: ${{ matrix.restore_cli }} cloudflared: ${{ matrix.cloudflared }} host_packages: ${{ matrix.host_packages }} host_preparation: ${{ matrix.host_preparation }} runner_comparison: ${{ matrix.runner_comparison }} compatible_api_key: ${{ matrix.compatible_api_key }} github_token: false shard: ${{ matrix.shard }} artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} catalogue-github-read: name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix, package-openshell-sdk] if: ${{ needs.generate-matrix.outputs.catalogue_github_read_matrix != '[]' }} strategy: fail-fast: false matrix: include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_github_read_matrix) }} uses: ./.github/workflows/e2e-standard-profile.yaml with: candidate_repository: ${{ inputs.checkout_repository || github.repository }} candidate_sha: ${{ inputs.checkout_sha || github.sha }} runtime_provider: ${{ matrix.runtime_provider }} execution_id: ${{ matrix.execution_id }} coverage_variant: ${{ matrix.coverage_variant }} risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} openshell_sdk_artifact_name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} managed_image_revision: ${{ needs.base-image-publication.outputs.managed_image_revision }} managed_image_receipt: ${{ needs.base-image-publication.outputs.managed_image_receipt }} managed_image_catalog: ${{ needs.base-image-publication.outputs.managed_image_catalog }} workload_source: ${{ needs.generate-matrix.outputs.workload_source }} credential_boundary: GitHub read token target_id: ${{ matrix.target_id }} catalogue_id: ${{ matrix.id }} runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }} checkout_sha: ${{ inputs.checkout_sha }} workflow_sha: ${{ inputs.workflow_sha }} test_file: ${{ matrix.test_file }} timeout_minutes: ${{ matrix.timeout_minutes }} install_mode: ${{ matrix.install_mode }} install_non_interactive: ${{ matrix.install_non_interactive }} restore_cli: ${{ matrix.restore_cli }} cloudflared: ${{ matrix.cloudflared }} host_packages: ${{ matrix.host_packages }} host_preparation: ${{ matrix.host_preparation }} runner_comparison: ${{ matrix.runner_comparison }} compatible_api_key: ${{ matrix.compatible_api_key }} github_token: true shard: ${{ matrix.shard }} artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} openshell-gateway-auth-contract: needs: [generate-matrix, package-openshell-sdk] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'openshell-gateway-auth-contract') }} runs-on: ubuntu-latest timeout-minutes: 20 env: E2E_JOB: "1" E2E_GATEWAY_RUNTIMES: docker E2E_TARGET_ID: "openshell-gateway-auth-contract" E2E_AGENT_RUNTIME: "none" E2E_OBSERVABLE_OUTCOME: "Gateway mTLS and sandbox JWT authentication boundaries hold" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; no inference endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-gateway-auth-contract NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_CANDIDATE_VERSION: ${{ needs.generate-matrix.outputs.openshell_version }} NEMOCLAW_OPENSHELL_PIN_VERSION: ${{ needs.generate-matrix.outputs.openshell_version }} DOCKER_GRPC_PROBE_IMAGE: "node:24.22.1-trixie-slim@sha256:43b9ba1142b8dcf33fa8f922ca1be85714adc444e0586c7cf7726daaeabc1bea" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - *dockerhub-auth - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" install-dependencies: "false" - name: Install reviewed OpenShell SDK archive without package credentials uses: NVIDIA/NemoClaw/.github/actions/install-reviewed-openshell-sdk@342c8f0ab115febb905eb4729d32478f22207589 - name: Restore exact-commit CLI artifact uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@3c6d991640414a2d78f95f629d67194d44829554 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Install OpenShell CLI run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh - name: Pre-pull pinned gateway auth probe image run: docker pull "$DOCKER_GRPC_PROBE_IMAGE" - name: Run OpenShell gateway auth contract live test run: | set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" if command -v openshell-gateway >/dev/null 2>&1; then OPENSHELL_GATEWAY_BIN="$(command -v openshell-gateway)" elif [ -x "$HOME/.local/bin/openshell-gateway" ]; then OPENSHELL_GATEWAY_BIN="$HOME/.local/bin/openshell-gateway" else echo "::error::OpenShell gateway binary not found after install" ls -la /usr/local/bin/openshell-gateway "$HOME/.local/bin/openshell-gateway" 2>&1 || true exit 1 fi export OPENSHELL_GATEWAY_BIN echo "Using OPENSHELL_GATEWAY_BIN=$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_GATEWAY_BIN" --version npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openshell-gateway-auth-source-contract.test.ts - id: artifact_safety name: Validate final OpenShell gateway auth contract artifacts if: always() run: node --no-warnings tools/e2e/openshell-gateway-auth-artifact-safety.mts "$E2E_ARTIFACT_DIR" - name: Upload OpenShell gateway auth contract artifacts if: ${{ always() && steps.artifact_safety.outcome == 'success' && steps.artifact_safety.outputs.approved_path != '' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-openshell-gateway-auth-contract path: ${{ steps.artifact_safety.outputs.approved_path }} - name: Clean up Docker auth if: always() shell: bash run: bash .github/scripts/docker-auth-cleanup.sh external-gateway-health: needs: [generate-matrix, package-openshell-sdk] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'external-gateway-health') }} runs-on: ubuntu-latest timeout-minutes: 15 env: E2E_JOB: "1" E2E_DEFAULT_ENABLED: "0" E2E_TARGET_ID: "external-gateway-health" E2E_AGENT_RUNTIME: "none" E2E_OBSERVABLE_OUTCOME: "The exact Blueprint Runner observes public gateway health over explicit HTTPS and CA" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu host with reviewed candidate OpenShell; no inference endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/external-gateway-health NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_OPENSHELL_PIN_VERSION: ${{ needs.generate-matrix.outputs.openshell_version }} NEMOCLAW_E2E_REQUIRE_EXECUTED_TEST: "1" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" install-dependencies: "false" - name: Install reviewed OpenShell SDK archive without package credentials uses: NVIDIA/NemoClaw/.github/actions/install-reviewed-openshell-sdk@342c8f0ab115febb905eb4729d32478f22207589 - name: Restore exact-commit CLI artifact uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@3c6d991640414a2d78f95f629d67194d44829554 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Install OpenShell CLI run: env -u NODE_AUTH_TOKEN -u GITHUB_TOKEN bash scripts/install-openshell.sh - name: Run external gateway health live test run: | set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" npx tsx tools/e2e/live-vitest-invocation.mts run \ --test-path test/e2e/live/external-gateway-health.test.ts - name: Upload external gateway health artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-external-gateway-health path: e2e-artifacts/live/external-gateway-health/ mcp-bridge: name: MCP bridge (${{ matrix.agent }}, ${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix, package-openshell-sdk] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'mcp-bridge') }} runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('mcp-bridge-{0}', matrix.agent)] }} permissions: contents: read # Keep each destructive agent lifecycle on a fresh runner. This bounds the # blast radius of runner loss and gives every adapter independent evidence. timeout-minutes: 90 strategy: fail-fast: false matrix: agent: [openclaw, hermes, deepagents] runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['mcp-bridge'] }} include: - agent: openclaw agent_runtime: openclaw coverage_variant: openclaw - agent: hermes agent_runtime: hermes coverage_variant: hermes - agent: deepagents agent_runtime: langchain-deepagents-code coverage_variant: deepagents env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" E2E_GATEWAY_RUNTIMES: docker,podman E2E_TARGET_ID: "mcp-bridge" E2E_OBSERVABLE_OUTCOME: "Stable OpenShell MCP bridge reaches tools and inference" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu managed runtime host; local compatible inference and MCP endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}/${{ matrix.runtime_provider }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} NEMOCLAW_MCP_BRIDGE_AGENT: ${{ matrix.agent }} NEMOCLAW_OPENSHELL_CHANNEL: stable NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: ${{ matrix.runtime_provider == 'docker' && '1' || '0' }} NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} OPENSHELL_DOCKER_SUPERVISOR_IMAGE: ${{ needs.generate-matrix.outputs.openshell_supervisor_image }} steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap if: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && matrix.agent == 'hermes' }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null CHECKOUT_SHA: ${{ inputs.checkout_sha }} DISPATCH_SHA: ${{ github.sha }} ENV: /dev/null EVENT_NAME: ${{ github.event_name }} EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} LC_ALL: C REF: ${{ github.ref }} REPOSITORY: ${{ github.repository }} RUNNER_ARCH_KIND: ${{ runner.arch }} RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} RUNNER_OS_KIND: ${{ runner.os }} WORKFLOW_SHA: ${{ github.workflow_sha }} run: &trusted-hermes-e2e-swap | set -euo pipefail readonly swap_dir="/mnt/nemoclaw-hermes-e2e-swap" readonly required_swap_bytes=34359738368 readonly swap_file_bytes=34359742464 readonly reserve_bytes=17179869184 readonly activation_observation_attempts=5 readonly activation_observation_delay_seconds=1 swap_file="" swap_activation_succeeded=0 fail() { printf 'Trusted Hermes E2E swap setup failed: %s\n' "$1" >&2 exit 1 } if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" ]]; then fail "workflow must run from NVIDIA/NemoClaw" fi if [[ "${EVENT_NAME}" != "push" && "${EVENT_NAME}" != "workflow_dispatch" ]]; then fail "workflow event must be push or workflow_dispatch" fi if [[ "${EVENT_NAME}" == "push" && "${REF}" != "refs/heads/main" ]]; then fail "push workflow must run from NVIDIA/NemoClaw main" fi if [[ "${EVENT_NAME}" == "workflow_dispatch" && "${REF}" != refs/heads/* ]]; then fail "manual workflow must run from an NVIDIA/NemoClaw branch" fi # PR E2E mode: maintainer-dispatched PR commit. if [[ "${EVENT_NAME}" == "workflow_dispatch" && -n "${CHECKOUT_SHA}" ]]; then if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then fail "checkout SHA must be lowercase 40-hex" fi if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then fail "workflow source must match the trusted dispatch revision" fi else # Direct-main mode: push or manual trigger on main. if [[ -n "${CHECKOUT_SHA}" || -n "${EXPECTED_WORKFLOW_SHA}" ]]; then fail "direct main runs must not request an alternate checkout or workflow revision" fi if [[ ! "${WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then fail "direct main workflow source must match the run revision" fi fi if [[ "${RUNNER_ENVIRONMENT_KIND}" != "github-hosted" || "${RUNNER_OS_KIND}" != "Linux" || "${RUNNER_ARCH_KIND}" != "X64" ]]; then fail "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner" fi mnt_metadata="$(/usr/bin/stat -c "%F:%u:%g" -- /mnt)" if [[ "${mnt_metadata}" != "directory:0:0" ]]; then fail "/mnt must be a root-owned directory" fi read_active_swap_bytes() { /usr/bin/sudo -n /usr/sbin/swapon --show=SIZE --bytes --noheadings | /usr/bin/awk '{ total += $1 } END { printf "%.0f", total }' } active_swap_bytes="$(read_active_swap_bytes)" active_swap_bytes="${active_swap_bytes:-0}" if [[ ! "${active_swap_bytes}" =~ ^[0-9]+$ ]]; then fail "unable to determine active swap capacity" fi if (( active_swap_bytes >= required_swap_bytes )); then printf 'Hermes E2E swap is already sufficient: %s bytes active\n' "${active_swap_bytes}" exit 0 fi available_bytes="$(/usr/bin/df --block-size=1 --output=avail /mnt | /usr/bin/tail -n 1 | /usr/bin/tr -d "[:space:]")" if [[ ! "${available_bytes}" =~ ^[0-9]+$ ]]; then fail "unable to determine available disk capacity under /mnt" fi required_disk_bytes=$((swap_file_bytes + reserve_bytes)) if (( available_bytes < required_disk_bytes )); then fail "insufficient disk capacity: ${available_bytes} bytes available, ${required_disk_bytes} required" fi if /usr/bin/sudo -n /usr/bin/test -e "${swap_dir}" || /usr/bin/sudo -n /usr/bin/test -L "${swap_dir}"; then fail "refusing unexpected pre-existing swap path" fi directory_created=0 cleanup_partial_swap() { status="$?" if (( status != 0 && directory_created == 1 )); then if active_swap_names="$(/usr/bin/sudo -n /usr/sbin/swapon --show=NAME --noheadings --raw 2>/dev/null)"; then fixed_swap_active=0 while IFS= read -r active_swap_name; do if [[ -n "${swap_file}" && "${active_swap_name}" == "${swap_file}" ]]; then fixed_swap_active=1 break fi done <<< "${active_swap_names}" if (( fixed_swap_active == 1 || swap_activation_succeeded == 1 )); then if /usr/bin/sudo -n /usr/sbin/swapoff "${swap_file}" 2>/dev/null; then /usr/bin/sudo -n /usr/bin/rm -f -- "${swap_file}" || true /usr/bin/sudo -n /usr/bin/rmdir -- "${swap_dir}" || true else printf 'Preserving active Hermes E2E swap after setup failure: %s\n' "${swap_file}" >&2 fi else if [[ -n "${swap_file}" ]]; then /usr/bin/sudo -n /usr/bin/rm -f -- "${swap_file}" || true fi /usr/bin/sudo -n /usr/bin/rmdir -- "${swap_dir}" || true fi else printf 'Preserving Hermes E2E swap because active swap could not be queried: %s\n' "${swap_file}" >&2 fi fi trap - EXIT exit "${status}" } trap cleanup_partial_swap EXIT /usr/bin/sudo -n /usr/bin/mkdir -m 0700 -- "${swap_dir}" directory_created=1 directory_metadata="$(/usr/bin/sudo -n /usr/bin/stat -c "%F:%u:%g:%a" -- "${swap_dir}")" if [[ "${directory_metadata}" != "directory:0:0:700" ]]; then fail "swap directory must be a root-owned mode-0700 directory" fi swap_file="$(/usr/bin/sudo -n /usr/bin/mktemp --tmpdir="${swap_dir}" nemoclaw-hermes.XXXXXXXX.swap)" if ! /usr/bin/sudo -n /usr/bin/test -f "${swap_file}" || /usr/bin/sudo -n /usr/bin/test -L "${swap_file}"; then fail "swap file must be a regular non-symlink" fi file_metadata="$(/usr/bin/sudo -n /usr/bin/stat -c "%u:%g:%a" -- "${swap_file}")" if [[ "${file_metadata}" != "0:0:600" ]]; then fail "swap file must be root-owned mode 0600" fi /usr/bin/sudo -n /usr/bin/fallocate -l "${swap_file_bytes}" "${swap_file}" file_size_bytes="$(/usr/bin/sudo -n /usr/bin/stat -c "%s" -- "${swap_file}")" if [[ ! "${file_size_bytes}" =~ ^[0-9]+$ || "${file_size_bytes}" -ne "${swap_file_bytes}" ]]; then fail "swap file size does not match the fixed backing allocation" fi remaining_bytes="$(/usr/bin/df --block-size=1 --output=avail /mnt | /usr/bin/tail -n 1 | /usr/bin/tr -d "[:space:]")" if [[ ! "${remaining_bytes}" =~ ^[0-9]+$ || "${remaining_bytes}" -lt "${reserve_bytes}" ]]; then fail "swap allocation did not preserve the required disk reserve" fi /usr/bin/sudo -n /usr/sbin/mkswap --quiet "${swap_file}" /usr/bin/sudo -n /usr/sbin/swapon "${swap_file}" swap_activation_succeeded=1 observe_provisioned_swap() { activation_observation_attempt=1 while (( activation_observation_attempt <= activation_observation_attempts )); do provisioned_swap_active=0 if active_swap_names="$(/usr/bin/sudo -n /usr/sbin/swapon --show=NAME --noheadings --raw 2>/dev/null)"; then while IFS= read -r active_swap_name; do if [[ "${active_swap_name}" == "${swap_file}" ]]; then provisioned_swap_active=1 break fi done <<< "${active_swap_names}" fi if observed_swap_bytes="$(read_active_swap_bytes 2>/dev/null)"; then observed_swap_bytes="${observed_swap_bytes:-0}" if [[ "${observed_swap_bytes}" =~ ^[0-9]+$ ]] && (( provisioned_swap_active == 1 && observed_swap_bytes >= required_swap_bytes )); then active_swap_bytes="${observed_swap_bytes}" return 0 fi fi if (( activation_observation_attempt < activation_observation_attempts )); then /usr/bin/sleep "${activation_observation_delay_seconds}" fi activation_observation_attempt=$((activation_observation_attempt + 1)) done return 1 } if ! observe_provisioned_swap; then fail "unable to verify the required active swap capacity after bounded observation" fi trap - EXIT printf 'Hermes E2E swap ready: %s bytes active\n' "${active_swap_bytes}" /usr/bin/sudo -n /usr/sbin/swapon --show - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - *dockerhub-auth - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" install-dependencies: "false" - name: Install reviewed OpenShell SDK archive without package credentials uses: NVIDIA/NemoClaw/.github/actions/install-reviewed-openshell-sdk@342c8f0ab115febb905eb4729d32478f22207589 - name: Restore exact-commit CLI artifact uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@3c6d991640414a2d78f95f629d67194d44829554 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Initialize runner comparison telemetry if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && (matrix.agent == 'hermes' || matrix.agent == 'deepagents') }} continue-on-error: true shell: bash run: npx tsx tools/e2e/runner-comparison.mts initialize - name: Prepare native Podman E2E runtime uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} - name: Install and verify cloudflared prerequisite # Update posture: maintainers review upstream cloudflared releases and # update the version and reviewed SHA256 together in both explicit MCP # lanes; mutable package repositories and unreviewed latest releases # are intentionally rejected by the workflow-contract tests. env: CLOUDFLARED_VERSION: "2026.6.1" CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526" run: | set -euo pipefail cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb" curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}" printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c - package="$(dpkg-deb -f "${cloudflared_deb}" Package)" version="$(dpkg-deb -f "${cloudflared_deb}" Version)" architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)" if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2 exit 1 fi sudo dpkg -i "${cloudflared_deb}" cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}" - name: Generate MCP test TLS run: bash test/e2e/setup-mcp-test-tls.sh - name: Install OpenShell CLI env: NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1" OPENSHELL_RUNTIME_PROVENANCE: ${{ needs.generate-matrix.outputs.openshell_runtime_provenance }} run: | set -euo pipefail bash scripts/install-openshell.sh export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" node --input-type=module - "$(command -v openshell)" <<'NODE' import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; const directory = dirname(process.argv[2]); const expected = JSON.parse(process.env.OPENSHELL_RUNTIME_PROVENANCE); for (const [role, binary] of [ ['cli', 'openshell'], ['gateway', 'openshell-gateway'], ['standaloneSandbox', 'openshell-sandbox'], ]) { const bytes = readFileSync(join(directory, binary)); const actual = createHash('sha256').update(bytes).digest('hex'); if (actual !== expected.artifacts?.[role]?.binarySha256) { throw new Error(`Installed OpenShell ${role} differs from reviewed provenance`); } } NODE if [[ "$NEMOCLAW_MCP_BRIDGE_AGENT" == "deepagents" ]]; then mkdir -p "$E2E_ARTIFACT_DIR/mcp-bridge-deepagents" printf '%s\n' "$OPENSHELL_RUNTIME_PROVENANCE" > "$E2E_ARTIFACT_DIR/mcp-bridge-deepagents/openshell-exact-main-provenance.json" fi - name: Run MCP OpenShell provider live test run: | set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" if command -v openshell >/dev/null 2>&1; then OPENSHELL_BIN="$(command -v openshell)" elif [ -x "$HOME/.local/bin/openshell" ]; then OPENSHELL_BIN="$HOME/.local/bin/openshell" else echo "::error::OpenShell CLI not found after install" exit 1 fi export OPENSHELL_BIN "$OPENSHELL_BIN" --version npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/mcp-bridge.test.ts - name: Finalize runner comparison telemetry if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && (matrix.agent == 'hermes' || matrix.agent == 'deepagents') }} continue-on-error: true shell: bash run: npx tsx tools/e2e/runner-comparison.mts finalize - id: mcp_artifact_secret_scan name: Scan MCP artifacts for fixture credentials if: always() run: >- npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}/${{ matrix.runtime_provider }} - name: Upload MCP server artifacts if: ${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-mcp-bridge-${{ matrix.agent }}-${{ matrix.runtime_provider }} path: e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}/${{ matrix.runtime_provider }}/ - name: Restore Docker CLI after native Podman E2E if: ${{ always() && matrix.runtime_provider == 'podman' }} uses: NVIDIA/NemoClaw/.github/actions/restore-native-podman-e2e@9650336899bf836db5844381a97cbc2b0fe4a2b8 with: enabled: "true" - name: Clean up Docker auth if: always() shell: bash run: bash .github/scripts/docker-auth-cleanup.sh openshell-credential-generation-window: name: OpenShell credential generation window (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix, package-openshell-sdk] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'openshell-credential-generation-window') }} runs-on: ubuntu-latest permissions: contents: read # Keep the credential-generation lifecycle on a fresh runner so it can # execute in parallel with, and fail independently from, the Deep Agents # MCP lifecycle without sharing destructive sandbox state. timeout-minutes: 90 strategy: fail-fast: false matrix: runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['openshell-credential-generation-window'] }} env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" E2E_GATEWAY_RUNTIMES: docker,podman E2E_TARGET_ID: "openshell-credential-generation-window" E2E_AGENT_RUNTIME: "openclaw" E2E_OBSERVABLE_OUTCOME: "Stable-handle refresh, revocation, detach, re-add, and valid rebuild preserve authorization epochs; expired inference credentials block rebuild before source deletion" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu managed runtime host; local compatible inference and MCP endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-credential-generation-window/${{ matrix.runtime_provider }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_OPENSHELL_CHANNEL: stable NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: "1" NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} OPENSHELL_DOCKER_SUPERVISOR_IMAGE: ${{ needs.generate-matrix.outputs.openshell_supervisor_image }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - *dockerhub-auth - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" install-dependencies: "false" - name: Install reviewed OpenShell SDK archive without package credentials uses: NVIDIA/NemoClaw/.github/actions/install-reviewed-openshell-sdk@342c8f0ab115febb905eb4729d32478f22207589 - name: Restore exact-commit CLI artifact uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@3c6d991640414a2d78f95f629d67194d44829554 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Prepare native Podman E2E runtime uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} - name: Install and verify cloudflared prerequisite env: CLOUDFLARED_VERSION: "2026.6.1" CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526" run: | set -euo pipefail cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb" curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}" printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c - package="$(dpkg-deb -f "${cloudflared_deb}" Package)" version="$(dpkg-deb -f "${cloudflared_deb}" Version)" architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)" if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2 exit 1 fi sudo dpkg -i "${cloudflared_deb}" cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}" - name: Generate MCP test TLS run: bash test/e2e/setup-mcp-test-tls.sh - name: Install OpenShell CLI env: NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1" OPENSHELL_RUNTIME_PROVENANCE: ${{ needs.generate-matrix.outputs.openshell_runtime_provenance }} run: | set -euo pipefail bash scripts/install-openshell.sh export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" node --input-type=module - "$(command -v openshell)" <<'NODE' import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; const directory = dirname(process.argv[2]); const expected = JSON.parse(process.env.OPENSHELL_RUNTIME_PROVENANCE); for (const [role, binary] of [ ['cli', 'openshell'], ['gateway', 'openshell-gateway'], ['standaloneSandbox', 'openshell-sandbox'], ]) { const bytes = readFileSync(join(directory, binary)); const actual = createHash('sha256').update(bytes).digest('hex'); if (actual !== expected.artifacts?.[role]?.binarySha256) { throw new Error(`Installed OpenShell ${role} differs from reviewed provenance`); } } NODE mkdir -p "$E2E_ARTIFACT_DIR/openshell-credential-generation-window" printf '%s\n' "$OPENSHELL_RUNTIME_PROVENANCE" > "$E2E_ARTIFACT_DIR/openshell-credential-generation-window/openshell-exact-main-provenance.json" - name: Run OpenShell credential generation-window live test run: | set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" if command -v openshell >/dev/null 2>&1; then OPENSHELL_BIN="$(command -v openshell)" elif [ -x "$HOME/.local/bin/openshell" ]; then OPENSHELL_BIN="$HOME/.local/bin/openshell" else echo "::error::OpenShell CLI not found after install" exit 1 fi export OPENSHELL_BIN "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/openshell-credential-generation-window.test.ts \ -t '^openshell-credential-generation-window$' \ --no-file-parallelism \ --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - id: credential_window_artifact_secret_scan name: Scan credential-window artifacts for fixture credentials if: always() run: >- npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts e2e-artifacts/live/openshell-credential-generation-window/${{ matrix.runtime_provider }} - name: Upload credential-window artifacts if: ${{ always() && steps.credential_window_artifact_secret_scan.outcome == 'success' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-openshell-credential-generation-window-${{ matrix.runtime_provider }} path: e2e-artifacts/live/openshell-credential-generation-window/${{ matrix.runtime_provider }}/ - name: Restore Docker CLI after native Podman E2E if: ${{ always() && matrix.runtime_provider == 'podman' }} uses: NVIDIA/NemoClaw/.github/actions/restore-native-podman-e2e@9650336899bf836db5844381a97cbc2b0fe4a2b8 with: enabled: "true" - name: Clean up Docker auth if: always() shell: bash run: bash .github/scripts/docker-auth-cleanup.sh openshell-dev-artifact: needs: generate-matrix if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'mcp-bridge-dev') }} runs-on: ubuntu-latest permissions: contents: read timeout-minutes: 15 outputs: artifact_name: ${{ steps.resolve_openshell_dev_artifact.outputs.artifact_name }} source_commit: ${{ steps.resolve_openshell_dev_artifact.outputs.source_commit }} manifest_sha256: ${{ steps.resolve_openshell_dev_artifact.outputs.manifest_sha256 }} steps: - name: Checkout trusted OpenShell dev tooling uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ github.repository }} ref: ${{ inputs.workflow_sha || github.workflow_sha }} path: .trusted-openshell-dev-artifact persist-credentials: false sparse-checkout: | .github/actions/setup-reviewed-npm ci/reviewed-npm-audit.json scripts/lib/npm-diagnostics.sh scripts/lib/reviewed-npm-audit.mts scripts/install-openshell.sh tools/e2e/openshell-dev-artifact.mts - name: Set up Node for OpenShell dev artifact resolution uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24.22.1 - name: Install reviewed npm uses: ./.trusted-openshell-dev-artifact/.github/actions/setup-reviewed-npm - id: resolve_openshell_dev_artifact name: Resolve immutable OpenShell dev artifact run: >- node --no-warnings "${{ github.workspace }}/.trusted-openshell-dev-artifact/tools/e2e/openshell-dev-artifact.mts" resolve "${{ runner.temp }}/openshell-dev-artifact" - name: Upload OpenShell dev artifact resolution if: ${{ always() }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: ${{ steps.resolve_openshell_dev_artifact.outputs.artifact_name || format('openshell-dev-infrastructure-failure-{0}-{1}', github.run_id, github.run_attempt) }} path: ${{ runner.temp }}/openshell-dev-artifact/ mcp-bridge-dev: name: MCP bridge dev (${{ matrix.agent }}, ${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix, openshell-dev-artifact, package-openshell-sdk] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'mcp-bridge-dev') }} runs-on: ubuntu-latest permissions: contents: read timeout-minutes: 90 strategy: fail-fast: false matrix: agent: [openclaw, hermes, deepagents] runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['mcp-bridge-dev'] }} include: - agent: openclaw agent_runtime: openclaw coverage_variant: openclaw - agent: hermes agent_runtime: hermes coverage_variant: hermes - agent: deepagents agent_runtime: langchain-deepagents-code coverage_variant: deepagents env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" E2E_GATEWAY_RUNTIMES: docker,podman E2E_DEFAULT_ENABLED: "0" E2E_TARGET_ID: "mcp-bridge-dev" E2E_OBSERVABLE_OUTCOME: "Development OpenShell MCP bridge reaches tools and inference" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu managed runtime host; local compatible inference and MCP endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/${{ matrix.runtime_provider }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} NEMOCLAW_MCP_BRIDGE_AGENT: ${{ matrix.agent }} NEMOCLAW_OPENSHELL_CHANNEL: dev NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} steps: # setup-node probes package managers in the workspace. # Run it before candidate checkout with automatic caching disabled. - name: Set up Node.js for trusted OpenShell verification uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24.22.1 package-manager-cache: false - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - *dockerhub-auth - name: Install reviewed npm for trusted OpenShell verification uses: NVIDIA/NemoClaw/.github/actions/setup-reviewed-npm@98669f24d35f18e49b6b2769cd68709509ea24f2 - name: Checkout trusted OpenShell dev tooling uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ github.repository }} ref: ${{ inputs.workflow_sha || github.workflow_sha }} path: .trusted-openshell-dev-artifact persist-credentials: false sparse-checkout: | .github/scripts/copy-openshell-dev-asset.sh .github/scripts/docker-auth-cleanup.sh scripts/install-openshell.sh tools/e2e/openshell-dev-artifact.mts - name: Restore immutable OpenShell dev artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.openshell-dev-artifact.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-dev-artifact digest-mismatch: error - name: Verify immutable OpenShell dev artifact env: OPENSHELL_DEV_ARTIFACT_DIR: ${{ runner.temp }}/openshell-dev-artifact OPENSHELL_DEV_EXPECTED_MANIFEST_SHA256: ${{ needs.openshell-dev-artifact.outputs.manifest_sha256 }} OPENSHELL_DEV_EXPECTED_SOURCE_COMMIT: ${{ needs.openshell-dev-artifact.outputs.source_commit }} run: >- node --no-warnings "${{ github.workspace }}/.trusted-openshell-dev-artifact/tools/e2e/openshell-dev-artifact.mts" verify "$OPENSHELL_DEV_ARTIFACT_DIR" "$OPENSHELL_DEV_EXPECTED_SOURCE_COMMIT" "$OPENSHELL_DEV_EXPECTED_MANIFEST_SHA256" - name: Revoke Docker auth before OpenShell development tooling shell: bash run: >- bash "${{ github.workspace }}/.trusted-openshell-dev-artifact/.github/scripts/docker-auth-cleanup.sh" - name: Install immutable OpenShell dev artifact env: NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL: "1" NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1" OPENSHELL_DEV_ASSET_DIR: ${{ runner.temp }}/openshell-dev-artifact/assets run: | set -euo pipefail shim_dir="$(mktemp -d)" trap 'rm -rf "$shim_dir"' EXIT cat >"$shim_dir/gh" <<'EOF' #!/usr/bin/env bash set -euo pipefail if [[ "$#" -ne 10 || "$1" != "release" || "$2" != "download" || "$3" != "dev" || "$4" != "--repo" || "$5" != "NVIDIA/OpenShell" || "$6" != "--pattern" || "$8" != "--dir" || "${10}" != "--clobber" ]]; then printf 'Unsupported gh invocation for retained OpenShell assets.\n' >&2 exit 64 fi asset="$7" destination="$9" case "$asset" in openshell-x86_64-unknown-linux-musl.tar.gz | openshell-checksums-sha256.txt | openshell-gateway-x86_64-unknown-linux-gnu.tar.gz | openshell-gateway-checksums-sha256.txt | openshell-sandbox-x86_64-unknown-linux-musl.tar.gz | openshell-sandbox-checksums-sha256.txt) ;; *) printf 'Unsupported retained OpenShell asset: %s\n' "$asset" >&2 exit 64 ;; esac bash "${{ github.workspace }}/.trusted-openshell-dev-artifact/.github/scripts/copy-openshell-dev-asset.sh" \ "$OPENSHELL_DEV_ASSET_DIR" "$asset" "$destination" EOF cat >"$shim_dir/curl" <<'EOF' #!/usr/bin/env bash printf 'Network fallback is disabled for retained OpenShell assets.\n' >&2 exit 1 EOF chmod 700 "$shim_dir/gh" "$shim_dir/curl" PATH="$shim_dir:$PATH" \ bash "${{ github.workspace }}/.trusted-openshell-dev-artifact/scripts/install-openshell.sh" # Dependency preparation can read candidate project configuration. Keep # it after the trusted installer has consumed the verified OpenShell # assets and Docker credentials have been revoked. - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" install-dependencies: "false" - name: Install reviewed OpenShell SDK archive without package credentials uses: NVIDIA/NemoClaw/.github/actions/install-reviewed-openshell-sdk@342c8f0ab115febb905eb4729d32478f22207589 - name: Restore exact-commit CLI artifact uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@3c6d991640414a2d78f95f629d67194d44829554 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Prepare native Podman E2E runtime uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} - name: Install and verify cloudflared prerequisite # Update posture: keep this dev compatibility lane on the same reviewed # version/SHA256 pair as the stable lane; workflow-contract tests fail # if the pins diverge or installation becomes mutable. env: CLOUDFLARED_VERSION: "2026.6.1" CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526" run: | set -euo pipefail cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb" curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}" printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c - package="$(dpkg-deb -f "${cloudflared_deb}" Package)" version="$(dpkg-deb -f "${cloudflared_deb}" Version)" architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)" if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2 exit 1 fi sudo dpkg -i "${cloudflared_deb}" cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}" - name: Generate MCP test TLS run: bash test/e2e/setup-mcp-test-tls.sh - id: mcp_runtime_compatibility name: Classify OpenShell credential-boundary compatibility run: | set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" npx tsx tools/e2e/mcp-bridge-runtime-compatibility.mts - name: Run MCP OpenShell provider live test if: ${{ steps.mcp_runtime_compatibility.outputs.mode == 'full-lifecycle' }} run: | set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" if command -v openshell >/dev/null 2>&1; then OPENSHELL_BIN="$(command -v openshell)" elif [ -x "$HOME/.local/bin/openshell" ]; then OPENSHELL_BIN="$HOME/.local/bin/openshell" else echo "::error::OpenShell CLI not found after install" exit 1 fi export OPENSHELL_BIN "$OPENSHELL_BIN" --version npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/mcp-bridge.test.ts - id: mcp_artifact_secret_scan name: Scan MCP artifacts for fixture credentials if: always() run: >- npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/${{ matrix.runtime_provider }} - name: Upload MCP server artifacts if: ${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-mcp-bridge-dev-${{ matrix.agent }}-${{ matrix.runtime_provider }} path: e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/${{ matrix.runtime_provider }}/ - name: Restore Docker CLI after native Podman E2E if: ${{ always() && matrix.runtime_provider == 'podman' }} uses: NVIDIA/NemoClaw/.github/actions/restore-native-podman-e2e@9650336899bf836db5844381a97cbc2b0fe4a2b8 with: enabled: "true" - name: Clean up Docker auth if: always() shell: bash run: bash .github/scripts/docker-auth-cleanup.sh # Manual PR qualification also requires the exact candidate activation contract. managed-image-multiarch-startup: name: Protected managed-image startup (${{ matrix.platform }}) needs: [base-image-publication, generate-matrix, package-openshell-sdk] if: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'managed-image-multiarch-startup') || contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'managed-image-protected-runtime')) }} runs-on: ${{ matrix.runner }} timeout-minutes: 210 permissions: contents: read strategy: fail-fast: false matrix: include: - platform: linux/amd64 runner: ubuntu-24.04 shard: linux-amd64 environment_or_inference_endpoint: "AMD64 Ubuntu; exact managed image startup" coverage_variant: linux-amd64 - platform: linux/arm64 runner: ubuntu-24.04-arm shard: linux-arm64 environment_or_inference_endpoint: "Arm64 Ubuntu; exact managed image startup" coverage_variant: linux-arm64 env: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }} E2E_JOB: "1" E2E_GATEWAY_RUNTIMES: docker E2E_TARGET_ID: "managed-image-multiarch-startup" E2E_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code" E2E_OBSERVABLE_OUTCOME: "Exact managed images start directly on the native architecture" RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-multiarch-activation-v1.json NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha || github.sha }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_HEAD_SHA: ${{ inputs.checkout_sha || github.sha }} NEMOCLAW_E2E_SHARD: ${{ matrix.shard }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE: ${{ github.workspace }}/.protected-managed-image-build-cache/${{ matrix.shard }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE_ARTIFACT: protected-managed-image-build-cache-${{ github.run_id }}-${{ inputs.checkout_sha || github.sha }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: protected-${{ github.run_id }}-${{ github.run_attempt }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/contracts.json NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/evidence.json NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM: ${{ matrix.platform }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }} NEMOCLAW_PROTECTED_REGISTRY_NAME: nemoclaw-managed-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }} NEMOCLAW_RUN_LIVE_E2E: "1" steps: - name: Bind protected E2E correlation identity shell: bash env: REQUESTED_CORRELATION_ID: ${{ inputs.correlation_id }} run: | set -euo pipefail correlation_id="$REQUESTED_CORRELATION_ID" if [[ -z "$correlation_id" ]]; then correlation_id="$(node --input-type=module -e \ 'import { randomUUID } from "node:crypto"; console.log(randomUUID())')" fi [[ "$correlation_id" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$ ]] printf 'NEMOCLAW_E2E_CORRELATION_ID=%s\n' "$correlation_id" >>"$GITHUB_ENV" - name: Validate protected exact-head dispatch env: BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }} CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }} EVENT_NAME: ${{ github.event_name }} EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }} PLATFORM: ${{ matrix.platform }} REF: ${{ github.ref }} REPOSITORY: ${{ github.repository }} RUNNER_ARCH_KIND: ${{ runner.arch }} WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash run: | set -euo pipefail [[ "$REPOSITORY" == "NVIDIA/NemoClaw" && ( ( "$EVENT_NAME" == "push" && "$REF" == "refs/heads/main" ) || ( "$EVENT_NAME" == "workflow_dispatch" && "$REF" == refs/heads/* ) ) ]] || { echo "::error::Protected managed-image startup must run from NVIDIA/NemoClaw" >&2 exit 1 } [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::Protected managed-image startup requires candidate and base SHAs" >&2 exit 1 } [[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA" ]] || { echo "::error::Protected managed-image startup requires the exact trusted workflow SHA" >&2 exit 1 } case "${PLATFORM}:${RUNNER_ARCH_KIND}" in linux/amd64:X64 | linux/arm64:ARM64) ;; *) echo "::error::Protected managed-image startup requires a native ${PLATFORM} runner" >&2 exit 1 ;; esac - name: Checkout protected managed-image candidate source uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - name: Validate trusted Hermes resolver checkout path shell: bash run: | set -euo pipefail trusted_resolver_root="$GITHUB_WORKSPACE/.trusted-hermes-resolver" [[ ! -e "$trusted_resolver_root" && ! -L "$trusted_resolver_root" ]] || { echo "::error::Trusted Hermes resolver checkout path already exists" >&2 exit 1 } - name: Checkout trusted Hermes resolver uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ github.repository }} ref: ${{ inputs.workflow_sha || github.workflow_sha }} path: .trusted-hermes-resolver sparse-checkout: | .github/actions/resolve-reviewed-hermes-platform scripts/checks/resolve-managed-pr-base.sh scripts/lib/dockerfile-copy-sources.mts fetch-depth: 1 persist-credentials: false - name: Bind protected managed-image Buildx configuration env: SHARD: ${{ matrix.shard }} shell: bash run: | set -euo pipefail [[ "$SHARD" =~ ^linux-(amd64|arm64)$ ]] || { echo "::error::Protected managed-image Buildx shard is invalid" >&2 exit 1 } buildx_config="${RUNNER_TEMP}/nemoclaw-protected-buildx-${SHARD}" [[ ! -e "$buildx_config" && ! -L "$buildx_config" ]] || { echo "::error::Protected managed-image Buildx configuration path already exists" >&2 exit 1 } install -d -m 0700 "$buildx_config" printf 'BUILDX_CONFIG=%s\n' "$buildx_config" >>"$GITHUB_ENV" - name: Set up protected managed-image Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 with: driver-opts: network=host buildkitd-config-inline: | [registry."localhost:5000"] http = true - id: openclaw-base name: Reuse only an input-equivalent published OpenClaw base if: ${{ inputs.checkout_sha != '' }} env: AGENT: openclaw BASE_ALIAS: ghcr.io/nvidia/nemoclaw/sandbox-base:latest BASE_DOCKERFILE: Dockerfile.base BASE_REPOSITORY: ghcr.io/nvidia/nemoclaw/sandbox-base BASE_SHA: ${{ inputs.base_sha }} CANDIDATE_SHA: ${{ inputs.checkout_sha }} DISPLAY_NAME: OpenClaw LOCAL_BASE_REFERENCE: nemoclaw-managed-pr/openclaw-base:${{ inputs.checkout_sha }} PLATFORM: ${{ matrix.platform }} RESOLUTION_MODE: reference-or-candidate shell: bash run: .trusted-hermes-resolver/scripts/checks/resolve-managed-pr-base.sh - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" install-dependencies: "false" - name: Install reviewed OpenShell SDK archive without package credentials uses: NVIDIA/NemoClaw/.github/actions/install-reviewed-openshell-sdk@342c8f0ab115febb905eb4729d32478f22207589 - name: Build shared policy boundary shell: bash run: | set -euo pipefail [[ ! -e nemoclaw/dist && ! -L nemoclaw/dist ]] || { echo "::error::Shared policy boundary output exists before the candidate build" >&2 exit 1 } npm run build:policy-boundary for artifact in \ nemoclaw/dist/shared/openshell-policy-boundary.cjs \ nemoclaw/dist/shared/sandbox-name.cjs; do [[ -f "$artifact" && ! -L "$artifact" && -s "$artifact" ]] || { echo "::error::Shared policy boundary artifact is missing or invalid: $artifact" >&2 exit 1 } done - *dockerhub-auth - id: candidate-contract name: Validate candidate activation contract env: CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }} shell: bash run: | set -euo pipefail activation="ci/protected-managed-image-multiarch-activation-v1.json" [[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::Protected managed-image checkout does not match the candidate SHA" >&2 exit 1 } [[ -f "$activation" && ! -L "$activation" ]] || { echo "::error::Protected managed-image activation contract is absent" >&2 exit 1 } jq -e ' ( ((keys | sort) == ["agents", "contractVersion", "jobId", "platforms"] and .contractVersion == 1) or ((keys | sort) == ["agents", "contractVersion", "dcodeBaseSource", "jobId", "platforms"] and .contractVersion == 2 and .dcodeBaseSource == "candidate") ) and .jobId == "managed-image-multiarch-startup" and .agents == ["openclaw", "hermes", "langchain-deepagents-code"] and .platforms == ["linux/amd64", "linux/arm64"] ' "$activation" >/dev/null || { echo "::error::Protected managed-image activation contract is invalid" >&2 exit 1 } printf 'dcode_source=%s\n' "$(jq -r '.dcodeBaseSource // "published"' "$activation")" >> "$GITHUB_OUTPUT" install -d -m 0700 "$E2E_ARTIFACT_DIR" - id: hermes-base name: Resolve reviewed Hermes platform base image if: ${{ inputs.checkout_sha == '' }} uses: ./.trusted-hermes-resolver/.github/actions/resolve-reviewed-hermes-platform with: dockerfile-path: agents/hermes/Dockerfile platform: ${{ matrix.platform }} - name: Remove trusted Hermes resolver checkout if: always() shell: bash run: | set -euo pipefail trusted_resolver_root="$GITHUB_WORKSPACE/.trusted-hermes-resolver" if [[ -e "$trusted_resolver_root" || -L "$trusted_resolver_root" ]]; then [[ -d "$trusted_resolver_root" && ! -L "$trusted_resolver_root" ]] || { echo "::error::Trusted Hermes resolver checkout path changed type" >&2 exit 1 } rm -rf -- "$trusted_resolver_root" fi [[ ! -e "$trusted_resolver_root" && ! -L "$trusted_resolver_root" ]] || { echo "::error::Trusted Hermes resolver checkout remained after cleanup" >&2 exit 1 } - id: bases name: Resolve digest-pinned platform base images env: CANDIDATE_BASES: ${{ inputs.checkout_sha != '' }} DCODE_BASE_CONTRACT: ${{ needs.base-image-publication.outputs.dcode_base_contract }} HERMES_BASE_REF: ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@${{ steps.hermes-base.outputs.digest }} OPENCLAW_BASE_REF: ${{ steps.openclaw-base.outputs.ref }} PLATFORM: ${{ matrix.platform }} shell: bash run: | set -euo pipefail case "$CANDIDATE_BASES" in true) [[ "$OPENCLAW_BASE_REF" == candidate || "$OPENCLAW_BASE_REF" =~ ^ghcr[.]io/nvidia/nemoclaw/sandbox-base@sha256:[a-f0-9]{64}$ ]] || { echo "::error::Trusted OpenClaw base resolver returned an invalid reference" >&2 exit 1 } printf 'openclaw=%s\nhermes=candidate\ndcode=candidate\n' "$OPENCLAW_BASE_REF" >> "$GITHUB_OUTPUT" exit 0 ;; false) ;; *) echo "::error::Invalid candidate base selection" >&2; exit 1 ;; esac arch="${PLATFORM#linux/}" work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-protected-bases.XXXXXX")" trap 'rm -rf -- "$work_dir"' EXIT resolve_base() { local output_name="$1" local alias="$2" local repository="$3" local alias_raw="$work_dir/${output_name}-alias.raw" local exact_raw="$work_dir/${output_name}-exact.raw" docker buildx imagetools inspect "$alias" --raw > "$alias_raw" local digest digest="$( jq -er --arg arch "$arch" ' if ( .mediaType == "application/vnd.oci.image.index.v1+json" or .mediaType == "application/vnd.docker.distribution.manifest.list.v2+json" ) then [.manifests[] | select(.platform.os == "linux" and .platform.architecture == $arch)] | if length == 1 then .[0].digest else error("not one exact platform descriptor") end else error("base alias is not a platform index") end ' "$alias_raw" )" [[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]] || { echo "::error::${output_name} base alias returned an invalid digest" >&2 exit 1 } local reference="${repository}@${digest}" docker buildx imagetools inspect "$reference" --raw > "$exact_raw" [[ "sha256:$(sha256sum "$exact_raw" | awk '{print $1}')" == "$digest" ]] || { echo "::error::${output_name} exact base bytes do not match the selected digest" >&2 exit 1 } printf '%s=%s\n' "$output_name" "$reference" >> "$GITHUB_OUTPUT" } resolve_base openclaw \ ghcr.io/nvidia/nemoclaw/sandbox-base:latest \ ghcr.io/nvidia/nemoclaw/sandbox-base dcode_reference="$( jq -er --arg platform "$PLATFORM" \ '.platformReferences[$platform]' <<< "$DCODE_BASE_CONTRACT" )" [[ "$dcode_reference" =~ ^ghcr[.]io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base@sha256:[a-f0-9]{64}$ ]] || { echo "::error::dcode base publication returned an invalid exact platform reference" >&2 exit 1 } dcode_digest="${dcode_reference##*@}" docker buildx imagetools inspect "$dcode_reference" --raw > "$work_dir/dcode-exact.raw" [[ "sha256:$(sha256sum "$work_dir/dcode-exact.raw" | awk '{print $1}')" == "$dcode_digest" ]] || { echo "::error::dcode exact base bytes do not match the published platform digest" >&2 exit 1 } printf 'dcode=%s\n' "$dcode_reference" >> "$GITHUB_OUTPUT" [[ "$HERMES_BASE_REF" =~ ^ghcr[.]io/nvidia/nemoclaw/hermes-sandbox-base@sha256:[a-f0-9]{64}$ ]] printf 'hermes=%s\n' "$HERMES_BASE_REF" >> "$GITHUB_OUTPUT" - name: Start isolated protected managed-image registry shell: bash run: | set -euo pipefail if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then echo "::error::Protected registry name already exists" >&2 exit 1 fi if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then echo "::error::Refusing to reuse an existing localhost:5000 registry" >&2 exit 1 fi docker run --detach \ --name "$NEMOCLAW_PROTECTED_REGISTRY_NAME" \ --label "io.nvidia.nemoclaw.managed-image.cohort=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT}" \ --label "io.nvidia.nemoclaw.e2e-platform=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM}" \ --publish 127.0.0.1:5000:5000 \ docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373 for _ in $(seq 1 30); do if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null; then exit 0 fi sleep 1 done docker logs "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >&2 exit 1 - name: Build exact all-agent protected managed images env: BASE_DCODE: ${{ steps.candidate-contract.outputs.dcode_source == 'candidate' && 'candidate' || steps.bases.outputs.dcode }} BASE_HERMES: ${{ steps.bases.outputs.hermes }} BASE_OPENCLAW: ${{ steps.bases.outputs.openclaw }} CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }} shell: bash run: | set -euo pipefail cache_args=() if [[ "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" == "linux/amd64" ]]; then cache_args=(--cache-to "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE") fi scripts/checks/build-protected-managed-images.sh \ --output "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" \ --revision "$CHECKOUT_SHA" \ --cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \ --platform "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" \ --runtime-user sandbox \ --openclaw-base "$BASE_OPENCLAW" \ --hermes-base "$BASE_HERMES" \ --dcode-base "$BASE_DCODE" \ "${cache_args[@]}" - name: Run every exact managed-image contract directly env: BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }} CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }} WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }} shell: bash run: | set -euo pipefail direct_runs="${RUNNER_TEMP}/protected-managed-image-direct-runs.jsonl" : > "$direct_runs" while IFS= read -r entry; do agent="$(jq -er '.agent' <<< "$entry")" digest="$(jq -er '.digest' <<< "$entry")" platform="$(jq -er '.platform' <<< "$entry")" reference="$(jq -er '.reference' <<< "$entry")" [[ "$platform" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" ]] || { echo "::error::Protected managed-image contract changed platform" >&2 exit 1 } npx --no-install tsx scripts/checks/run-managed-image-direct-e2e.ts \ --agent "$agent" \ --image "$reference" \ --platform "$platform" jq -nc \ --arg agent "$agent" \ --arg digest "$digest" \ --arg platform "$platform" \ --arg reference "$reference" \ '{agent: $agent, digest: $digest, platform: $platform, reference: $reference}' \ >> "$direct_runs" done < <(jq -c '.[]' "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT") contract_sha="sha256:$(sha256sum "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" | awk '{print $1}')" jq -n \ --arg baseSha "$BASE_SHA" \ --arg cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \ --arg contractSha256 "$contract_sha" \ --arg headSha "$CHECKOUT_SHA" \ --arg platform "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" \ --arg workflowSha "$WORKFLOW_SHA" \ --argjson runAttempt "$GITHUB_RUN_ATTEMPT" \ --argjson runId "$GITHUB_RUN_ID" \ --slurpfile contracts "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" \ --slurpfile directRuns "$direct_runs" \ '{ kind: "nemoclaw-protected-managed-image-multiarch-v1", headSha: $headSha, baseSha: $baseSha, workflowSha: $workflowSha, platform: $platform, cohort: $cohort, contractSha256: $contractSha256, contracts: $contracts[0], directRuns: $directRuns, run: {id: $runId, attempt: $runAttempt} }' > "${NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE}.tmp" mv \ "${NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE}.tmp" \ "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE" - name: Remove isolated protected managed-image registry if: always() shell: bash run: | set -euo pipefail if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then owner="$(docker container inspect --format '{{index .Config.Labels "io.nvidia.nemoclaw.managed-image.cohort"}}' "$NEMOCLAW_PROTECTED_REGISTRY_NAME")" [[ "$owner" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" ]] || { echo "::error::Refusing to remove a registry not owned by this protected shard" >&2 exit 1 } docker rm -f "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null fi if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1 || curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then echo "::error::Protected managed-image registry remained after cleanup" >&2 exit 1 fi - name: Validate protected managed-image evidence shell: bash run: >- npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/managed-image-multiarch-startup.test.ts - name: Remove owned Docker Engine 27 receipt daemon if: always() shell: bash run: | set -euo pipefail node --import tsx scripts/checks/docker-engine-27-receipt-transfer-e2e.ts \ --cleanup-only \ --run-id "$GITHUB_RUN_ID" \ --run-attempt "$GITHUB_RUN_ATTEMPT" \ --platform "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" - name: Validate OpenClaw managed-image security boundary shell: bash run: >- env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN npx vitest run --project integration test/e2e-runtime/managed-image-openclaw-security.test.ts --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Validate managed-image glibc probe lifecycle if: ${{ !cancelled() }} shell: bash run: | set -euo pipefail export NEMOCLAW_RUN_GLIBC_PROBE_DOCKER_E2E=1 NEMOCLAW_TEST_IMAGE="$(jq -er '.[] | select(.agent == "openclaw") | .reference' "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT")" export NEMOCLAW_TEST_IMAGE env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN npx vitest run --project integration test/e2e-runtime/image-compatibility-docker-lifecycle.test.ts --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Remove protected managed-image cohort resources if: always() shell: bash run: | set -euo pipefail label="io.nvidia.nemoclaw.managed-image.cohort=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT}" cleanup_failed=0 mapfile -t owned_containers < <(docker ps -aq --filter "label=$label") ((${#owned_containers[@]} == 0)) || docker rm -f "${owned_containers[@]}" >/dev/null || cleanup_failed=1 mapfile -t owned_images < <(docker image ls -q --filter "label=$label" | sort -u) ((${#owned_images[@]} == 0)) || docker image rm -f "${owned_images[@]}" >/dev/null || cleanup_failed=1 mapfile -t owned_volumes < <(docker volume ls -q --filter "label=$label") ((${#owned_volumes[@]} == 0)) || docker volume rm -f "${owned_volumes[@]}" >/dev/null || cleanup_failed=1 if docker ps -aq --filter "label=$label" | grep -q . || docker image ls -q --filter "label=$label" | grep -q . || docker volume ls -q --filter "label=$label" | grep -q .; then echo "::error::Protected managed-image cohort resources remained after cleanup" >&2 exit 1 fi ((cleanup_failed == 0)) || exit 1 - name: Publish exact amd64 protected runtime build cache if: ${{ matrix.platform == 'linux/amd64' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE_ARTIFACT }} path: ${{ env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE }}/ if-no-files-found: error retention-days: 1 compression-level: 0 overwrite: true - name: Upload protected managed-image evidence if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-managed-image-multiarch-startup-${{ matrix.shard }} path: e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/ - name: Clean up Docker auth if: always() shell: bash run: bash .github/scripts/docker-auth-cleanup.sh # This lane runs after managed-image startup succeeds. Manual PR qualification # requires the exact candidate activation contract. Its # hosted amd64 prerequisite exports the exact candidate build cache. # This GPU lane imports the exact candidate build cache and its complete # locked npm cache seed, then rebuilds with BuildKit networking disabled. # invalidState: Cold managed-image builds on the protected GPU runner can # stall in npm and terminate with "Exit handler never called!" before any # GPU, Ollama, NIM, or vLLM assertion executes. # sourceBoundary: The hosted amd64 lane builds the exact candidate. It adds # every reachable Linux x64 glibc registry archive from the candidate # lockfile to the same-run cache artifact. The GPU lane verifies the manifest, # lock digest, archive set, and # each SHA-512 integrity before the candidate build can consume the seed. # BuildKit records can accelerate the rebuild, while the seed makes a missing # npm layer reproducible without weakening the fail-closed network boundary. # whyNotBuildCacheOnly: Protected runs 31156103003, 31158136234, and # 31159822518 proved that network-key drift and incomplete local cache exports # can miss the npm layer before runtime qualification. # regressionTest: The materializer and build-script behavior tests reject an # incomplete, substituted, unexpected, or symlinked archive handoff and pin # exact --cache-from/--network none arguments. Workflow boundary tests pin the # same-run cache artifact. # lifecycleDecision: PR #8261 owns removal after three consecutive protected # GPU cold builds of all three images reach GPU, Ollama, NIM, and vLLM # assertions without the hosted cache. managed-image-protected-runtime: name: Protected managed-image GPU and local inference needs: [ base-image-publication, generate-matrix, managed-image-multiarch-startup, package-openshell-sdk, ] if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && needs['base-image-publication'].result == 'success' && needs['generate-matrix'].result == 'success' && needs['managed-image-multiarch-startup'].result == 'success' && needs['package-openshell-sdk'].result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'managed-image-protected-runtime') }} runs-on: linux-amd64-gpu-rtxpro6000-latest-1 timeout-minutes: 75 permissions: contents: read pull-requests: read env: CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }} E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime E2E_JOB: "1" E2E_GATEWAY_RUNTIMES: docker E2E_TARGET_ID: "managed-image-protected-runtime" E2E_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code" E2E_OBSERVABLE_OUTCOME: "Protected GPU runtime supports Ollama vLLM NIM rollback and cleanup" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local and hosted inference services" RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-runtime-activation-v1.json NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} NEMOCLAW_E2E_SHARD: linux-amd64-gpu NEMOCLAW_E2E_TESTED_ROOT: ${{ github.workspace }}/.candidate-runtime NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE: ${{ github.workspace }}/.protected-managed-image-build-cache/linux-amd64 NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE_ARTIFACT: protected-managed-image-build-cache-${{ github.run_id }}-${{ inputs.checkout_sha || github.sha }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: protected-${{ github.run_id }}-${{ github.run_attempt }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime/contracts.json NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM: linux/amd64 NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }} NEMOCLAW_PROTECTED_REGISTRY_NAME: nemoclaw-managed-runtime-${{ github.run_id }}-${{ github.run_attempt }} NEMOCLAW_RUN_LIVE_E2E: "1" OPENSHELL_GATEWAY: nemoclaw steps: - name: Validate protected runtime exact-head dispatch env: BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }} CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }} EVENT_NAME: ${{ github.event_name }} EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }} REF: ${{ github.ref }} REPOSITORY: ${{ github.repository }} RUNNER_ARCH_KIND: ${{ runner.arch }} WORKFLOW_SHA: ${{ github.workflow_sha }} shell: bash run: | set -euo pipefail printf 'NEMOCLAW_PROTECTED_RUNTIME_STARTED_AT_MS=%s000\n' "$(date +%s)" >> "$GITHUB_ENV" [[ "$REPOSITORY" == "NVIDIA/NemoClaw" && ( ( "$EVENT_NAME" == "push" && "$REF" == "refs/heads/main" ) || ( "$EVENT_NAME" == "workflow_dispatch" && "$REF" == refs/heads/* ) ) ]] || { echo "::error::Protected managed-image runtime must run from NVIDIA/NemoClaw" >&2 exit 1 } [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::Protected managed-image runtime requires candidate and base SHAs" >&2 exit 1 } [[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA" ]] || { echo "::error::Protected managed-image runtime requires the exact trusted workflow SHA" >&2 exit 1 } [[ "$RUNNER_ARCH_KIND" == "X64" ]] || { echo "::error::Protected managed-image runtime requires a native linux/amd64 GPU runner" >&2 exit 1 } - name: Checkout trusted protected runtime qualification uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ github.repository }} ref: ${{ inputs.workflow_sha || github.workflow_sha }} fetch-depth: 0 persist-credentials: false - name: Checkout exact protected runtime candidate source uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} path: .candidate-runtime fetch-depth: 0 persist-credentials: false - name: Reuse or refresh reviewed audit evidence before the offline build uses: ./.github/actions/ci-reviewed-npm-audit with: target-root: ${{ github.workspace }}/.candidate-runtime report-dir: artifacts/reviewed-npm-audit cache-directory: ${{ runner.temp }}/reviewed-npm-audit-cache - name: Download exact protected runtime build cache uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE_ARTIFACT }} path: ${{ env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE }} - *dockerhub-auth - name: Set up protected runtime Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 with: driver-opts: network=host buildkitd-config-inline: | [registry."localhost:5000"] http = true - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" - id: runtime-contract name: Validate protected runtime activation contract env: CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }} shell: bash run: | set -euo pipefail candidate_root=".candidate-runtime" activation="$candidate_root/ci/protected-managed-image-runtime-activation-v1.json" [[ "$(git -C "$candidate_root" rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::Protected managed-image runtime checkout does not match the candidate SHA" >&2 exit 1 } [[ -f "$activation" && ! -L "$activation" ]] || { echo "::error::Protected managed-image runtime activation contract is absent" >&2 exit 1 } runtime_user="$( jq -er ' ( .jobId == "managed-image-protected-runtime" and .agents == ["openclaw", "hermes", "langchain-deepagents-code"] and .platform == "linux/amd64" and .providers == ["ollama", "nim", "vllm"] ) as $common | if $common and (keys | sort) == ["agents", "contractVersion", "jobId", "platform", "providers"] and .contractVersion == 1 then "root" elif $common and (keys | sort) == ["agents", "contractVersion", "jobId", "platform", "providers", "runtimeUser"] and .contractVersion == 2 and .runtimeUser == "sandbox" then "sandbox" else error("invalid protected runtime activation contract") end ' "$activation" )" || { echo "::error::Protected managed-image runtime activation contract is invalid" >&2 exit 1 } printf 'runtime_user=%s\n' "$runtime_user" >> "$GITHUB_OUTPUT" install -d -m 0700 "$E2E_ARTIFACT_DIR" - id: openshell-projection name: Bind reviewed candidate OpenShell runtime shell: bash env: GITHUB_TOKEN: ${{ github.token }} UPGRADE_PR_NUMBER: ${{ inputs.pr_number }} run: | set -euo pipefail node tools/e2e/protected-openshell-workspace.mts prepare > "$E2E_ARTIFACT_DIR/openshell-projection.json" - name: Verify reviewed candidate OpenShell runtime before consumption shell: bash env: GITHUB_TOKEN: ${{ github.token }} UPGRADE_PR_NUMBER: ${{ inputs.pr_number }} run: | set -euo pipefail node tools/e2e/protected-openshell-workspace.mts verify > "$E2E_ARTIFACT_DIR/openshell-projection-initially-verified.json" - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Install reviewed OpenShell SDK from trusted controller uses: ./.github/actions/install-reviewed-openshell-sdk - name: Build trusted CLI with reviewed OpenShell runtime shell: bash run: npm run build:cli - id: runtime-hermes-base name: Resolve reviewed Hermes runtime base image if: ${{ inputs.checkout_sha == '' }} uses: ./.github/actions/resolve-reviewed-hermes-platform with: dockerfile-path: .candidate-runtime/agents/hermes/Dockerfile platform: linux/amd64 - id: runtime-bases name: Resolve digest-pinned amd64 runtime base images env: CANDIDATE_BASES: ${{ inputs.checkout_sha != '' }} CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }} DCODE_BASE_REF: ${{ needs.base-image-publication.outputs.dcode_base_ref }} HERMES_BASE_REF: ghcr.io/nvidia/nemoclaw/hermes-sandbox-base@${{ steps.runtime-hermes-base.outputs.digest }} shell: bash run: | set -euo pipefail work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-runtime-bases.XXXXXX")" trap 'rm -rf -- "$work_dir"' EXIT [[ "$CANDIDATE_BASES" == true || "$CANDIDATE_BASES" == false ]] || exit 1 # Consume the CPU job's immutable bases, not aliases resolved again on the GPU host. prepared_inputs="$NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE/prepared-inputs" [[ -f "$prepared_inputs" && ! -L "$prepared_inputs" ]] || { echo "::error::protected build cache has no safe prepared-input identity" >&2 exit 1 } read -r cached_revision cached_platform cached_openclaw cached_hermes cached_dcode extra < "$prepared_inputs" [[ -z "$extra" && "$cached_revision" == "$CHECKOUT_SHA" && "$cached_platform" == "linux/amd64" && "$(cat "$prepared_inputs")" == "$cached_revision $cached_platform $cached_openclaw $cached_hermes $cached_dcode" ]] || { echo "::error::protected build-cache identity does not match this run" >&2 exit 1 } resolve_cached_base() { local name="$1" agent="$2" reference="$3" repository="$4" expected_published="$5" local candidate_prefix="localhost:5000/nemoclaw-managed-protected-base/${agent}@" local digest="${reference##*@}" [[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]] || exit 1 if [[ "$reference" == "${candidate_prefix}${digest}" ]]; then local verified verified="$(node scripts/checks/protected-dcode-base-receipt.mts verify-reusable \ "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE/${name}-base" \ "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE/${name}-base-receipt.json" "$agent")" [[ "$verified" == "$reference" ]] || { echo "::error::prepared ${agent} base does not match this run's verified OCI artifact" >&2 exit 1 } else [[ ( "$CANDIDATE_BASES" == false || "$agent" == openclaw ) && "$reference" == "${repository}@${digest}" && ( -z "$expected_published" || "$reference" == "$expected_published" ) ]] || { echo "::error::prepared ${agent} base does not match the selected source" >&2 exit 1 } docker buildx imagetools inspect "$reference" --raw > "$work_dir/${name}-exact.raw" [[ "sha256:$(sha256sum "$work_dir/${name}-exact.raw" | awk '{print $1}')" == "$digest" ]] || { echo "::error::${agent} exact base bytes do not match the prepared digest" >&2 exit 1 } fi printf '%s=%s\n' "$name" "$reference" >> "$GITHUB_OUTPUT" } resolve_cached_base openclaw openclaw "$cached_openclaw" ghcr.io/nvidia/nemoclaw/sandbox-base '' resolve_cached_base hermes hermes "$cached_hermes" ghcr.io/nvidia/nemoclaw/hermes-sandbox-base "$HERMES_BASE_REF" resolve_cached_base dcode langchain-deepagents-code "$cached_dcode" ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base "$DCODE_BASE_REF" - name: Start isolated protected runtime registry shell: bash run: | set -euo pipefail if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then echo "::error::Protected runtime registry name already exists" >&2 exit 1 fi if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then echo "::error::Refusing to reuse an existing localhost:5000 registry" >&2 exit 1 fi docker run --detach \ --name "$NEMOCLAW_PROTECTED_REGISTRY_NAME" \ --label "io.nvidia.nemoclaw.managed-image.cohort=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT}" \ --label "io.nvidia.nemoclaw.e2e-platform=linux/amd64" \ --publish 127.0.0.1:5000:5000 \ docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373 for _ in $(seq 1 30); do if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null; then exit 0 fi sleep 1 done docker logs "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >&2 exit 1 - name: Build exact all-agent protected runtime images env: BASE_DCODE: ${{ steps.runtime-bases.outputs.dcode }} BASE_HERMES: ${{ steps.runtime-bases.outputs.hermes }} BASE_OPENCLAW: ${{ steps.runtime-bases.outputs.openclaw }} CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }} RUNTIME_USER: ${{ steps.runtime-contract.outputs.runtime_user }} shell: bash run: | set -euo pipefail scripts/checks/build-protected-managed-images.sh \ --output "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" \ --revision "$CHECKOUT_SHA" \ --cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \ --platform linux/amd64 \ --runtime-user "$RUNTIME_USER" \ --source-root "$GITHUB_WORKSPACE/.candidate-runtime" \ --cache-from "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE" \ --audit-evidence-from "$GITHUB_WORKSPACE/.candidate-runtime/artifacts/reviewed-npm-audit" \ --openclaw-base "$BASE_OPENCLAW" \ --hermes-base "$BASE_HERMES" \ --dcode-base "$BASE_DCODE" - name: Install OpenShell CLI shell: bash run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh - name: Verify reviewed candidate OpenShell runtime shell: bash env: GITHUB_TOKEN: ${{ github.token }} UPGRADE_PR_NUMBER: ${{ inputs.pr_number }} run: | set -euo pipefail node tools/e2e/protected-openshell-workspace.mts verify > "$E2E_ARTIFACT_DIR/openshell-projection-verified.json" - name: Run all-agent GPU, local inference, rollback, and cleanup qualification env: NVIDIA_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_API_KEY || '' }} shell: bash run: | set -euo pipefail [[ "$(git rev-parse --verify HEAD)" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA" ]] || { echo "::error::Protected NIM qualification must execute trusted workflow code" >&2 exit 1 } export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" OPENSHELL_BIN="$(command -v openshell)" export OPENSHELL_BIN "$OPENSHELL_BIN" --version npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/managed-image-protected-runtime.test.ts - name: Remove isolated protected runtime registry timeout-minutes: 2 if: always() shell: bash run: | set -euo pipefail if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then owner="$( docker container inspect \ --format '{{index .Config.Labels "io.nvidia.nemoclaw.managed-image.cohort"}}' \ "$NEMOCLAW_PROTECTED_REGISTRY_NAME" )" [[ "$owner" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" ]] || { echo "::error::Refusing to remove a runtime registry not owned by this protected job" >&2 exit 1 } docker rm -f "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null fi if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then echo "::error::Protected managed-image runtime registry remained after cleanup" >&2 exit 1 fi if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then echo "::error::Protected managed-image runtime registry listener remained after cleanup" >&2 exit 1 fi - name: Restore trusted OpenShell sources timeout-minutes: 1 if: ${{ always() && steps.openshell-projection.outcome == 'success' }} shell: bash env: GITHUB_TOKEN: ${{ github.token }} UPGRADE_PR_NUMBER: ${{ inputs.pr_number }} run: | set -euo pipefail node tools/e2e/protected-openshell-workspace.mts restore > "$E2E_ARTIFACT_DIR/openshell-projection-restored.json" - name: Upload protected managed-image runtime artifacts timeout-minutes: 4 if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-managed-image-protected-runtime path: e2e-artifacts/live/managed-image-protected-runtime/ - name: Clean up Docker auth timeout-minutes: 1 if: always() shell: bash run: bash .github/scripts/docker-auth-cleanup.sh hermes-e2e: name: Hermes E2E (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix, package-openshell-sdk] if: ${{ needs.generate-matrix.outputs.hermes_selected == 'true' }} runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-e2e'] }} timeout-minutes: 85 strategy: fail-fast: false matrix: runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['hermes-e2e'] }} env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" E2E_GATEWAY_RUNTIMES: docker,podman E2E_TARGET_ID: "hermes-e2e" E2E_AGENT_RUNTIME: "hermes" E2E_OBSERVABLE_OUTCOME: "Install onboarding health inference lifecycle dashboard and security succeed" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; mock or NVIDIA hosted inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-e2e/${{ matrix.runtime_provider }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} NEMOCLAW_E2E_INFERENCE_MODE: ${{ inputs.inference_mode || 'mock' }} NEMOCLAW_E2E_HERMES_DASHBOARD: "1" NEMOCLAW_HERMES_DASHBOARD: "1" NEMOCLAW_DASHBOARD_PORT: "19000" NEMOCLAW_HERMES_DASHBOARD_PORT: "19000" NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT: "19120" NEMOCLAW_HERMES_DASHBOARD_TUI: "1" NEMOCLAW_HERMES_API_PORT: "8643" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_AGENT: hermes NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_RECREATE_SANDBOX: "1" NEMOCLAW_SANDBOX_NAME: e2e-hermes NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60" steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap if: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (github.event_name == 'push' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,') || contains(format(',{0},', inputs.jobs), ',hermes-dashboard,') || contains(format(',{0},', inputs.targets), ',hermes-dashboard,')))) }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null CHECKOUT_SHA: ${{ inputs.checkout_sha }} DISPATCH_SHA: ${{ github.sha }} ENV: /dev/null EVENT_NAME: ${{ github.event_name }} EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} LC_ALL: C REF: ${{ github.ref }} REPOSITORY: ${{ github.repository }} RUNNER_ARCH_KIND: ${{ runner.arch }} RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} RUNNER_OS_KIND: ${{ runner.os }} WORKFLOW_SHA: ${{ github.workflow_sha }} run: *trusted-hermes-e2e-swap - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - name: Set up pinned v1 compatibility toolchain uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 with: toolchain: 1.98.1 cache: false rustflags: "" - *dockerhub-auth - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" install-dependencies: "false" - name: Install reviewed OpenShell SDK archive without package credentials uses: NVIDIA/NemoClaw/.github/actions/install-reviewed-openshell-sdk@342c8f0ab115febb905eb4729d32478f22207589 - name: Restore exact-commit CLI artifact uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@3c6d991640414a2d78f95f629d67194d44829554 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Initialize runner comparison telemetry if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} continue-on-error: true shell: bash run: npx tsx tools/e2e/runner-comparison.mts initialize - name: Prepare native Podman E2E runtime uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} - name: Run Hermes live Vitest test env: NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }} run: | set -euo pipefail npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/hermes-e2e.test.ts - name: Finalize runner comparison telemetry if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} continue-on-error: true shell: bash run: npx tsx tools/e2e/runner-comparison.mts finalize - name: Upload Hermes live Vitest artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-hermes-e2e-${{ matrix.runtime_provider }} path: e2e-artifacts/live/hermes-e2e/${{ matrix.runtime_provider }}/ - name: Restore Docker CLI after native Podman E2E if: ${{ always() && matrix.runtime_provider == 'podman' }} uses: NVIDIA/NemoClaw/.github/actions/restore-native-podman-e2e@9650336899bf836db5844381a97cbc2b0fe4a2b8 with: enabled: "true" - name: Clean up Docker auth if: always() shell: bash run: bash .github/scripts/docker-auth-cleanup.sh portable-hermes-finalization: name: Portable Hermes finalization (Podman 5.7, NVIDIA GPU) needs: [generate-matrix, portable-podman-toolchain, package-openshell-sdk] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'portable-hermes-finalization') }} runs-on: linux-amd64-gpu-rtxpro6000-latest-1 timeout-minutes: 90 permissions: contents: read env: E2E_JOB: "1" E2E_DEFAULT_ENABLED: "0" E2E_GATEWAY_RUNTIMES: podman E2E_TARGET_ID: portable-hermes-finalization E2E_AGENT_RUNTIME: hermes E2E_OBSERVABLE_OUTCOME: "Portable Hermes lifecycle completes on the supported NVIDIA GPU and Podman 5.7 boundary" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "x86-64 NVIDIA GPU runner; rootless Podman 5.7; local Ollama" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/portable-hermes-finalization E2E_SOURCE_REVISION: ${{ inputs.checkout_sha || github.sha }} NEMOCLAW_NATIVE_PODMAN_SERVICE_TASKS_MAX: "1024" NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_GATEWAY_RUNTIME: podman NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_NON_INTERACTIVE: "1" steps: - name: Install Portable Podman host dependencies uses: NVIDIA/NemoClaw/.github/actions/host-dependency-setup@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: packages: conmon crun fuse-overlayfs golang-github-containers-common iptables nftables slirp4netns uidmap - name: Check out trusted workflow cleanup authority uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.workflow_sha }} fetch-depth: 1 persist-credentials: false - name: Install immutable native Podman cleanup fixture env: CLEANUP_FIXTURE: /usr/local/libexec/nemoclaw/native-podman-e2e-restore.${{ github.run_id }}.${{ github.run_attempt }} TRUSTED_FIXTURE_SHA256: f9b26c07e5b84660a0f2710307cefc9c0c811d3d88628bb0a9500e031ec02ba8 shell: /bin/bash --noprofile --norc -e -o pipefail {0} run: | set -euo pipefail source_action=.github/actions/restore-native-podman-e2e/action.yaml extracted_fixture="$(mktemp)" cleanup_extracted_fixture() { rm -f -- "$extracted_fixture" } trap cleanup_extracted_fixture EXIT awk ' found { sub(/^ /, ""); print } /^ run: \|$/ { found = 1; next } ' "$source_action" >"$extracted_fixture" printf '%s %s\n' "$TRUSTED_FIXTURE_SHA256" "$extracted_fixture" | sha256sum --check --strict sudo install -d --owner=root --group=root --mode=0755 /usr/local/libexec/nemoclaw sudo install --owner=root --group=root --mode=0555 "$extracted_fixture" "$CLEANUP_FIXTURE" [[ "$(sudo stat -c '%u:%g:%a' "$CLEANUP_FIXTURE")" == "0:0:555" ]] sudo cmp -s "$extracted_fixture" "$CLEANUP_FIXTURE" - name: Install pinned OpenShell env: XDG_BIN_HOME: ${{ runner.temp }}/nemoclaw-portable-hermes-openshell-bin.${{ github.run_id }}.${{ github.run_attempt }} run: | set -euo pipefail install_root="${XDG_BIN_HOME:?}" [[ "$install_root" == "$RUNNER_TEMP"/nemoclaw-portable-hermes-openshell-bin.* ]] rm -rf -- "$install_root" install -d -m 0700 "$install_root" installer_sentinel="nemoclaw-openshell-install-sentinel:${install_root}" cat >"$install_root/openshell" <<'EOF' #!/bin/sh printf '%s\n' 'openshell 0.0.0-dev' EOF printf '# %s\n' "$installer_sentinel" >>"$install_root/openshell" chmod 0700 "$install_root/openshell" env -u GH_TOKEN -u GITHUB_TOKEN \ PATH="$install_root:$PATH" \ NEMOCLAW_NON_INTERACTIVE=1 \ bash scripts/install-openshell.sh openshell_install_root_is_complete() { local root="$1" component for component in openshell openshell-gateway openshell-sandbox; do [[ -f "$root/$component" && ! -L "$root/$component" && -x "$root/$component" ]] \ || return 1 done } if ! openshell_install_root_is_complete "$install_root"; then echo "::error::The pinned OpenShell installer did not populate the isolated install root." >&2 exit 1 fi if grep -Fq "$installer_sentinel" "$install_root/openshell"; then echo "::error::The pinned OpenShell installer did not replace its trusted reinstall sentinel." >&2 exit 1 fi for component in openshell openshell-gateway openshell-sandbox; do source="$install_root/$component" [[ -f "$source" && ! -L "$source" && -x "$source" ]] destination="/usr/bin/$component" if [[ "$source" -ef "$destination" ]]; then [[ -f "$destination" && ! -L "$destination" ]] sudo chown root:root "$destination" sudo chmod 0755 "$destination" else sudo install -o root -g root -m 0755 "$source" "$destination" fi done echo /usr/bin >>"$GITHUB_PATH" - name: Recover stale Docker isolation before Portable Podman E2E uses: NVIDIA/NemoClaw/.github/actions/restore-native-podman-e2e@9650336899bf836db5844381a97cbc2b0fe4a2b8 with: enabled: "true" - name: Prepare Portable Podman 5.7 runtime uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: cleanup-fixture: /usr/local/libexec/nemoclaw/native-podman-e2e-restore.${{ github.run_id }}.${{ github.run_attempt }} disable-container-cgroups: "true" enabled: "true" toolchain: portable-5.7 isolate-docker-cli: "false" service-tasks-max: ${{ env.NEMOCLAW_NATIVE_PODMAN_SERVICE_TASKS_MAX }} - name: Select pinned compatible Portable crun shell: /bin/bash --noprofile --norc -e -o pipefail {0} run: | set -euo pipefail # Ubuntu Noble's crun 1.14.1 rejects the OCI spec emitted by Podman 5.7. # Keep the replacement beside the immutable cleanup fixture, outside # the shared Podman toolchain root whose restore contract is closed. runtime=/usr/local/libexec/nemoclaw/portable-crun.${{ github.run_id }}.${{ github.run_attempt }} config="$RUNNER_TEMP/native-podman-e2e-containers.conf" socket="${OPENSHELL_PODMAN_SOCKET:?}" service=nemoclaw-native-podman-e2e.service expected_sha=2aa6b7024a9c9f153895c0d11ae233d3758f54844011c3a039e3e89048d01d42 [[ "$(uname -m)" == x86_64 ]] [[ "${CONTAINERS_CONF:?}" == "$config" && -f "$config" && ! -L "$config" ]] [[ ! -e "$runtime" && ! -L "$runtime" ]] ! grep -Fq '[engine.runtimes]' "$config" download="$(mktemp "$RUNNER_TEMP/portable-crun.XXXXXXXX")" trap 'rm -f -- "$download"' EXIT curl --fail --location --silent --show-error --proto '=https' \ --proto-redir '=https' --tlsv1.2 \ --max-time 120 \ https://github.com/containers/crun/releases/download/1.28/crun-1.28-linux-amd64 \ --output "$download" printf '%s %s\n' "$expected_sha" "$download" | sha256sum --check --strict sudo install --owner=root --group=root --mode=0755 "$download" "$runtime" [[ "$(sudo stat -c '%u:%g:%a' "$runtime")" == 0:0:755 ]] [[ "$(sudo sha256sum "$runtime" | cut -d ' ' -f1)" == "$expected_sha" ]] [[ "$("$runtime" --version | head -n 1)" == 'crun version 1.28' ]] printf '\n[engine.runtimes]\ncrun = ["%s"]\n' "$runtime" >>"$config" /usr/bin/systemctl --user restart "$service" /usr/bin/systemctl --user is-active --quiet "$service" podman --url "unix://$socket" info --format json | jq -e --arg runtime "$runtime" ' (.host.ociRuntime.name // .Host.OCIRuntime.Name) == "crun" and (.host.ociRuntime.path // .Host.OCIRuntime.Path) == $runtime ' >/dev/null podman info --format json | jq -e --arg runtime "$runtime" ' (.host.ociRuntime.path // .Host.OCIRuntime.Path) == $runtime ' >/dev/null jq -n --arg path "$runtime" --arg sha256 "$expected_sha" \ '{schemaVersion: 1, runtime: "crun", version: "1.28", path: $path, sha256: $sha256, remoteServiceSelected: true, localCliSelected: true}' \ >"$RUNNER_TEMP/portable-podman-crun-authority.json" - name: Set Portable Podman registry process headroom shell: /bin/bash --noprofile --norc -e -o pipefail {0} run: | set -euo pipefail # The shared setup action is pinned and runs before candidate checkout. # The job selects the limit; the action owns service-unit rendering and startup. minimum=128 service_tasks_max="${NEMOCLAW_NATIVE_PODMAN_SERVICE_TASKS_MAX:?}" config="$RUNNER_TEMP/native-podman-e2e-containers.conf" [[ "${CONTAINERS_CONF:?}" == "$config" && -f "$config" && ! -L "$config" ]] [[ "$(grep -Fxc '[containers]' "$config")" == 1 ]] [[ "$(grep -Fxc '[engine]' "$config")" == 1 ]] if grep -Eq '^[[:space:]]*(pids_limit|cgroup_manager)[[:space:]]*=' "$config"; then echo "::error::Portable Podman containers.conf already sets process or cgroup policy" >&2 exit 1 fi [[ "$(grep -Fxc 'cgroups = "disabled"' "$config")" == 1 ]] # Podman can place a rootless registry in a sibling scope even with # container cgroups disabled. The live probe verifies its actual # cgroup and every limiting ancestor after registry HTTP readiness. service=nemoclaw-native-podman-e2e.service service_pid="$(/usr/bin/systemctl --user show "$service" --property=MainPID --value)" [[ "$service_pid" =~ ^[1-9][0-9]*$ ]] # Portable host preparation changes the client configuration and podman.service. # The registry socket belongs to this separate service and must retain this configuration. service_config="$(tr '\0' '\n' <"/proc/$service_pid/environ" | sed -n 's/^CONTAINERS_CONF=//p')" [[ "$service_config" == "$config" ]] || { echo "::error::Portable registry Podman service is using a different containers.conf" >&2 exit 1 } printf 'NEMOCLAW_NATIVE_PODMAN_SERVICE_PID=%s\n' "$service_pid" >>"$GITHUB_ENV" nproc_soft="$(awk '$1 == "Max" && $2 == "processes" { print $3; exit }' "/proc/$service_pid/limits" || true)" user_tasks="$(ps -u "$(id -u)" -L --no-headers | wc -l | tr -d ' ')" || user_tasks=unavailable host_max="$(cat /proc/sys/kernel/threads-max 2>/dev/null || true)" host_tasks="$(awk '{ split($4, tasks, "/"); print tasks[2] }' /proc/loadavg 2>/dev/null || true)" tasks_max="$(/usr/bin/systemctl --user show "$service" --property=TasksMax --value || true)" tasks_current="$(/usr/bin/systemctl --user show "$service" --property=TasksCurrent --value || true)" limit_nproc="$(/usr/bin/systemctl --user show "$service" --property=LimitNPROC --value || true)" failure=none if [[ "$tasks_max" != "$service_tasks_max" ]]; then failure=service_tasks_configuration fi if [[ ! "$user_tasks" =~ ^[0-9]+$ || -z "$tasks_max" || -z "$tasks_current" || -z "$limit_nproc" ]] || \ { [[ "$nproc_soft" != unlimited ]] && \ { [[ ! "$nproc_soft" =~ ^[0-9]+$ ]] || (( nproc_soft - user_tasks < minimum )); }; }; then failure=rlimit_nproc fi if [[ ! "$tasks_max" =~ ^[0-9]+$ || ! "$tasks_current" =~ ^[0-9]+$ ]] || \ (( tasks_max - tasks_current < minimum )); then failure=service_tasks fi if [[ ! "$host_max" =~ ^[0-9]+$ || ! "$host_tasks" =~ ^[0-9]+$ ]] || \ (( host_max - host_tasks < minimum )); then failure=host_threads fi group="$(awk -F: '$1 == "0" { print $3; exit }' "/proc/$service_pid/cgroup")" cgroups='[]' if [[ "$group" != /* || "$group" == *'..'* ]]; then failure=service_cgroup group=/ fi depth=0 while (( depth < 24 )); do [[ "$group" != / ]] || break current="$(cat "/sys/fs/cgroup$group/pids.current" 2>/dev/null || true)" maximum="$(cat "/sys/fs/cgroup$group/pids.max" 2>/dev/null || true)" cgroups="$(jq -cn --argjson rows "$cgroups" \ --arg path "$group" --arg current "$current" --arg maximum "$maximum" \ '$rows + [{path: $path, current: $current, maximum: $maximum}]')" if [[ ! "$current" =~ ^[0-9]+$ ]] || \ { [[ "$maximum" != max ]] && \ { [[ ! "$maximum" =~ ^[0-9]+$ ]] || (( maximum - current < minimum )); }; }; then failure=cgroup_pids fi group="${group%/*}" [[ -n "$group" ]] || group=/ depth=$((depth + 1)) done if [[ "$group" != / ]]; then failure=cgroup_depth; fi jq -n --arg failure "$failure" --argjson minimum "$minimum" \ --arg containerCgroups observed-after-registry-launch \ --arg tasksMax "$tasks_max" --arg tasksCurrent "$tasks_current" \ --arg limitNproc "$limit_nproc" \ --arg nprocSoft "$nproc_soft" --arg userTasks "$user_tasks" \ --arg hostMax "$host_max" --arg hostTasks "$host_tasks" --argjson cgroups "$cgroups" \ '{schemaVersion: 1, failure: $failure, minimumHeadroom: $minimum, containerCgroups: $containerCgroups, service: {tasksMax: $tasksMax, tasksCurrent: $tasksCurrent, limitNproc: $limitNproc, nprocSoft: $nprocSoft, userTasks: $userTasks}, host: {threadsMax: $hostMax, tasks: $hostTasks}, cgroups: $cgroups}' \ >"$RUNNER_TEMP/portable-podman-process-headroom.json" if [[ "$failure" != none ]]; then echo "::error::Portable Podman infrastructure has insufficient process headroom ($failure)" >&2 exit 1 fi printf 'PORTABLE_REGISTRY_MINIMUM_PROCESS_HEADROOM=%s\n' "$minimum" >>"$GITHUB_ENV" - name: Check out the exact candidate uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - *dockerhub-auth - name: Prove Portable Podman OCI execution before Hermes shell: /bin/bash --noprofile --norc -e -o pipefail {0} run: | set -euo pipefail # Exercise the API service used by the managed registry before candidate behavior. podman --url "unix://${OPENSHELL_PODMAN_SOCKET:?}" run --rm --pull=always \ --entrypoint /bin/sh \ docker.io/library/registry:2@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373 \ -c ':' - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" install-dependencies: "false" - name: Install reviewed OpenShell SDK archive without package credentials uses: NVIDIA/NemoClaw/.github/actions/install-reviewed-openshell-sdk@342c8f0ab115febb905eb4729d32478f22207589 - name: Restore exact-commit CLI artifact uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@3c6d991640414a2d78f95f629d67194d44829554 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Compile Portable Hermes authorities run: | set -euo pipefail npm run catalog:compile npm run build:policy-boundary - name: Run Portable Hermes finalization live Vitest test shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: CLEANUP_FIXTURE: /usr/local/libexec/nemoclaw/native-podman-e2e-restore.${{ github.run_id }}.${{ github.run_attempt }} E2E_HERMES_BASE_STORAGE_HOME: ${{ runner.temp }}/nemoclaw-hermes-base-storage NEMOCLAW_HERMES_E2E_BASE_IMAGE: localhost/nemoclaw-hermes-base:portable-e2e XDG_DATA_HOME: ${{ runner.temp }}/nemoclaw-hermes-base-storage run: | set -euo pipefail restore_runner() { local command_status=$? local cleanup_status=0 trap - EXIT INT TERM set +e /usr/bin/env -i \ HOME="$HOME" \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ RUNNER_TEMP="$RUNNER_TEMP" \ /bin/bash "$CLEANUP_FIXTURE" cleanup_status=$? set -e if [[ "$cleanup_status" -ne 0 ]]; then echo "::error::Portable Podman cleanup failed after restoring Docker" >&2 exit "$cleanup_status" fi exit "$command_status" } [[ "$(sudo stat -c '%u:%g:%a' "$CLEANUP_FIXTURE")" == "0:0:555" ]] trap restore_runner EXIT trap 'exit 130' INT trap 'exit 143' TERM [[ "$(uname -m)" == x86_64 ]] gpu_name="$(nvidia-smi --query-gpu=name --format=csv,noheader | sed -n '1p')" [[ -n "$gpu_name" ]] mkdir -p "$E2E_ARTIFACT_DIR" jq -n \ --arg architecture x86_64 \ --arg gpuName "$gpu_name" \ --arg podmanVersion "$(podman --version)" \ '{schemaVersion: 1, architecture: $architecture, gpuName: $gpuName, podmanVersion: $podmanVersion}' \ >"$E2E_ARTIFACT_DIR/host-authority.json" podman build \ --runtime "/usr/local/libexec/nemoclaw/portable-crun.${{ github.run_id }}.${{ github.run_attempt }}" \ --file agents/hermes/Dockerfile.base \ --tag localhost/nemoclaw-hermes-base:portable-e2e \ . npx tsx tools/e2e/live-vitest-invocation.mts run \ --test-path test/e2e/live/portable-profile-rootless-linux.test.ts - name: Retain Portable Podman process headroom if: always() shell: bash run: | set -euo pipefail for name in portable-podman-process-headroom portable-podman-crun-authority; do source="$RUNNER_TEMP/$name.json" if [[ ! -f "$source" || -L "$source" ]]; then continue; fi [[ ! -L "$E2E_ARTIFACT_DIR" ]] install -d -m 0700 "$E2E_ARTIFACT_DIR" install -m 0600 "$source" "$E2E_ARTIFACT_DIR/$name.json" done - name: Upload Portable Hermes finalization artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-portable-hermes-finalization path: e2e-artifacts/live/portable-hermes-finalization/ - name: Remove pinned Portable crun if: always() shell: /bin/bash --noprofile --norc -e -o pipefail {0} run: | set -euo pipefail runtime=/usr/local/libexec/nemoclaw/portable-crun.${{ github.run_id }}.${{ github.run_attempt }} if ! sudo test -e "$runtime" && ! sudo test -L "$runtime"; then exit 0; fi uid="$(id -u)" manager_state="$(/usr/bin/systemctl is-active "user@${uid}.service" 2>/dev/null || true)" if [[ "$manager_state" == active ]]; then service=nemoclaw-native-podman-e2e runtime_dir="/run/user/$uid" user_systemctl() { env DBUS_SESSION_BUS_ADDRESS="unix:path=$runtime_dir/bus" \ XDG_RUNTIME_DIR="$runtime_dir" /usr/bin/systemctl --user "$@" } user_systemctl stop "$service.socket" "$service.service" 2>/dev/null || true socket_state="$(user_systemctl is-active "$service.socket" 2>/dev/null || true)" service_state="$(user_systemctl is-active "$service.service" 2>/dev/null || true)" if [[ "$socket_state" != inactive && "$socket_state" != unknown ]] || [[ "$service_state" != inactive && "$service_state" != unknown ]]; then echo "::error::Portable crun cleanup could not prove Podman socket and service inactive" >&2 exit 1 fi elif [[ "$manager_state" != inactive ]]; then echo "::error::Portable crun cleanup could not prove user manager inactive" >&2 exit 1 fi sudo test -f "$runtime" sudo test ! -L "$runtime" # A failed install can leave a partial root-owned file before chmod. [[ "$(sudo stat -c '%u:%g' "$runtime")" == 0:0 ]] sudo rm -f -- "$runtime" sudo test ! -e "$runtime" - name: Restore Docker and retire Portable Podman runtime if: always() uses: NVIDIA/NemoClaw/.github/actions/restore-native-podman-e2e@9650336899bf836db5844381a97cbc2b0fe4a2b8 with: enabled: "true" - name: Remove pinned Portable crun after restoration if: always() shell: /bin/bash --noprofile --norc -e -o pipefail {0} run: | set -euo pipefail runtime=/usr/local/libexec/nemoclaw/portable-crun.${{ github.run_id }}.${{ github.run_attempt }} if ! sudo test -e "$runtime" && ! sudo test -L "$runtime"; then exit 0; fi uid="$(id -u)" manager_state="$(/usr/bin/systemctl is-active "user@${uid}.service" 2>/dev/null || true)" if [[ "$manager_state" == active ]]; then runtime_dir="/run/user/$uid" user_systemctl() { env DBUS_SESSION_BUS_ADDRESS="unix:path=$runtime_dir/bus" \ XDG_RUNTIME_DIR="$runtime_dir" /usr/bin/systemctl --user "$@" } socket_state="$(user_systemctl is-active nemoclaw-native-podman-e2e.socket 2>/dev/null || true)" service_state="$(user_systemctl is-active nemoclaw-native-podman-e2e.service 2>/dev/null || true)" if [[ "$socket_state" != inactive && "$socket_state" != unknown ]] || [[ "$service_state" != inactive && "$service_state" != unknown ]]; then echo "::error::Portable crun cleanup after restoration could not prove Podman inactive" >&2 exit 1 fi elif [[ "$manager_state" != inactive ]]; then echo "::error::Portable crun cleanup after restoration could not prove user manager inactive" >&2 exit 1 fi sudo test -f "$runtime" sudo test ! -L "$runtime" [[ "$(sudo stat -c '%u:%g' "$runtime")" == 0:0 ]] sudo rm -f -- "$runtime" sudo test ! -e "$runtime" - name: Remove immutable native Podman cleanup fixture if: always() env: CLEANUP_FIXTURE: /usr/local/libexec/nemoclaw/native-podman-e2e-restore.${{ github.run_id }}.${{ github.run_attempt }} shell: bash run: sudo rm -f -- "$CLEANUP_FIXTURE" - name: Clean up Docker auth if: always() shell: bash run: bash .github/scripts/docker-auth-cleanup.sh hermes-gpu-startup: name: Hermes GPU startup (${{ matrix.scenario }}, ${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix, package-openshell-sdk] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'hermes-gpu-startup') }} runs-on: linux-amd64-gpu-rtxpro6000-latest-1 timeout-minutes: 90 strategy: fail-fast: false max-parallel: 2 matrix: scenario: [native, fallback, compatibility-only] runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['hermes-gpu-startup'] }} exclude: - scenario: fallback runtime_provider: podman - scenario: compatibility-only runtime_provider: podman env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" E2E_GATEWAY_RUNTIMES: docker,podman E2E_TARGET_ID: "hermes-gpu-startup" E2E_AGENT_RUNTIME: "hermes" E2E_OBSERVABLE_OUTCOME: "Hermes GPU startup reaches the stable Ready route" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local GPU inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/${{ matrix.runtime_provider }} E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} NEMOCLAW_E2E_SHARD: ${{ matrix.scenario }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_AGENT: hermes NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_RECREATE_SANDBOX: "1" NEMOCLAW_SANDBOX_GPU: "1" NEMOCLAW_SANDBOX_NAME: ${{ matrix.scenario }} NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60" steps: - name: Install native Podman host dependencies if: ${{ matrix.runtime_provider == 'podman' }} uses: NVIDIA/NemoClaw/.github/actions/host-dependency-setup@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: packages: conmon fuse-overlayfs golang-github-containers-common iptables nftables slirp4netns uidmap - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - name: Checkout trusted Hermes GPU runtime fixture if: ${{ matrix.scenario == 'fallback' }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: NVIDIA/NemoClaw ref: ${{ github.workflow_sha }} path: .trusted-hermes-gpu-fixture-${{ github.run_id }}-${{ github.run_attempt }} sparse-checkout: tools/e2e/hermes-gpu-docker-runtime-fixture.sh sparse-checkout-cone-mode: false persist-credentials: false - name: Install trusted Hermes GPU runtime fixture if: ${{ matrix.scenario == 'fallback' }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} ENV: /dev/null TRUSTED_DISPATCH_SHA: ${{ github.sha }} TRUSTED_FIXTURE_SHA256: e273c4baa7fe89546d64517cf56eafec30aeda7b355971263605ab1327fade02 TRUSTED_WORKFLOW_SHA: ${{ github.workflow_sha }} run: | set -euo pipefail trusted_checkout="$GITHUB_WORKSPACE/.trusted-hermes-gpu-fixture-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" trusted_source="$trusted_checkout/tools/e2e/hermes-gpu-docker-runtime-fixture.sh" trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}" trusted_state_root=/var/lib/nemoclaw-e2e run_trusted_fixture() { /usr/bin/sudo -n /usr/bin/env -i \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ /bin/bash "$trusted_fixture" "$@" } [[ "$TRUSTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ ]] \ || { echo "Trusted workflow SHA must be an immutable commit" >&2; exit 1; } [[ "$TRUSTED_DISPATCH_SHA" = "$TRUSTED_WORKFLOW_SHA" ]] \ || { echo "Trusted fixture must match the dispatched main workflow" >&2; exit 1; } [[ "$TRUSTED_FIXTURE_SHA256" =~ ^[a-f0-9]{64}$ ]] \ || { echo "Trusted fixture SHA-256 must be pinned" >&2; exit 1; } [ "$(/usr/bin/git -C "$trusted_checkout" rev-parse HEAD)" = "$TRUSTED_WORKFLOW_SHA" ] \ || { echo "Trusted fixture checkout does not match the workflow SHA" >&2; exit 1; } [ -f "$trusted_source" ] && [ ! -L "$trusted_source" ] \ || { echo "Trusted Docker fixture must be a regular non-symlink file" >&2; exit 1; } /usr/bin/sudo /usr/bin/install -d -o root -g root -m 0755 /usr/local/libexec/nemoclaw /usr/bin/sudo /usr/bin/install -o root -g root -m 0500 \ "$trusted_source" "$trusted_fixture" [ "$(/usr/bin/sudo /usr/bin/stat -c '%a %u %g' "$trusted_fixture")" = "500 0 0" ] \ || { echo "Trusted Docker fixture ownership or mode is invalid" >&2; exit 1; } printf '%s %s\n' "$TRUSTED_FIXTURE_SHA256" "$trusted_fixture" \ | /usr/bin/sudo /usr/bin/sha256sum -c - /usr/bin/sudo /usr/bin/cmp -s "$trusted_source" "$trusted_fixture" \ || { echo "Installed Docker fixture does not match trusted workflow code" >&2; exit 1; } /usr/bin/sudo /usr/bin/install -d -o root -g root -m 0700 "$trusted_state_root" # Recover any root-owned snapshot left by a hard-cancelled earlier run before # PR-controlled build or test code executes on this persistent GPU runner. if ! /usr/bin/sudo /usr/bin/find "$trusted_state_root" -mindepth 1 -maxdepth 1 \ -type d -name 'hermes-gpu-fallback-docker-runtime.*' -print0 \ | while IFS= read -r -d '' stale_state_dir; do if ! run_trusted_fixture restore \ "$stale_state_dir" /etc/docker/daemon.json >/dev/null; then exit 1 fi done; then echo "Could not recover stale Docker fallback state" >&2 exit 1 fi - *dockerhub-auth - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" install-dependencies: "false" - name: Install reviewed OpenShell SDK archive without package credentials uses: NVIDIA/NemoClaw/.github/actions/install-reviewed-openshell-sdk@342c8f0ab115febb905eb4729d32478f22207589 - name: Restore exact-commit CLI artifact uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@3c6d991640414a2d78f95f629d67194d44829554 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Reassert trusted Node runtime uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 env: BASH_ENV: /dev/null E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} ENV: /dev/null NODE_OPTIONS: "" with: node-version: "24.22.1" - name: Reinstall reviewed npm after Node reassertion uses: NVIDIA/NemoClaw/.github/actions/setup-reviewed-npm@98669f24d35f18e49b6b2769cd68709509ea24f2 - name: Recover Docker CLI before native Podman E2E if: ${{ matrix.runtime_provider == 'podman' }} uses: NVIDIA/NemoClaw/.github/actions/restore-native-podman-e2e@9650336899bf836db5844381a97cbc2b0fe4a2b8 with: enabled: "true" - name: Prepare native Podman E2E runtime uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} - name: Run Hermes GPU startup live Vitest test shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} ENV: /dev/null run: | set -euo pipefail if [ "$E2E_HERMES_GPU_STARTUP_SCENARIO" = fallback ]; then umask 077 daemon_json=/etc/docker/daemon.json trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}" trusted_state_root=/var/lib/nemoclaw-e2e run_trusted_fixture() { /usr/bin/sudo -n /usr/bin/env -i \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ /bin/bash "$trusted_fixture" "$@" } mkdir -p "$E2E_ARTIFACT_DIR" [ "$(/usr/bin/sudo /usr/bin/stat -c '%a %u %g' "$trusted_fixture")" = "500 0 0" ] \ || { echo "Trusted Docker fixture ownership or mode changed" >&2; exit 1; } /usr/bin/sudo /usr/bin/install -d -o root -g root -m 0700 "$trusted_state_root" state_dir="$(/usr/bin/sudo /usr/bin/mktemp -d \ "$trusted_state_root/hermes-gpu-fallback-docker-runtime.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.fallback.XXXXXX")" /usr/bin/sudo /usr/bin/chown root:root "$state_dir" /usr/bin/sudo /usr/bin/chmod 0700 "$state_dir" # SOURCE_OF_TRUTH_REVIEW (Hermes GPU fallback fixture; #6110): # invalidState: a cancelled fallback test leaves the runner-global Docker default # runtime changed, contaminating later OpenShell jobs on this self-hosted runner. # sourceBoundary: daemon mutation uses only immutable root-owned workflow_sha # code; the PR test cannot replace the helper or its root-owned snapshot state. # whyNotSourceFix: the compatibility retry must prove its explicit GPU envelope # without inheriting an NVIDIA daemon default from this managed runner. # regressionTest: hermes-gpu-startup-workflow-boundary requires this same-step trap. # removalCondition: remove the daemon fixture when the GPU runner supplies an # isolated runc-default Docker daemon for this compatibility qualification. restore_docker_default_runtime() { local command_status=$? local restore_status=0 local restored_runtime="" trap - EXIT INT TERM set +e restored_runtime="$(run_trusted_fixture restore "$state_dir" "$daemon_json")" restore_status=$? if [ "$restore_status" -eq 0 ] && [ -n "$restored_runtime" ]; then printf '%s\n' "$restored_runtime" \ >"$E2E_ARTIFACT_DIR/docker-default-runtime-restored.txt" \ || restore_status=1 fi if [ "$restore_status" -ne 0 ]; then exit 1 fi exit "$command_status" } # Force runc only within this trap-guarded process. The wrapper rejects native # create before progress, and the compatibility retry must add GPU access # explicitly instead of inheriting the managed runner's NVIDIA daemon default. trap restore_docker_default_runtime EXIT trap 'exit 130' INT trap 'exit 143' TERM original_runtime="$(run_trusted_fixture capture "$state_dir" "$daemon_json")" printf '%s\n' "$original_runtime" \ >"$E2E_ARTIFACT_DIR/docker-default-runtime-before.txt" selected_runtime="$(run_trusted_fixture select-runc "$state_dir" "$daemon_json")" printf '%s\n' "$selected_runtime" \ >"$E2E_ARTIFACT_DIR/docker-default-runtime-during.txt" fi npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/hermes-gpu-startup.test.ts - name: Recover Docker daemon after Hermes GPU fallback fixture if: always() shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} ENV: /dev/null run: | set -euo pipefail if [ "$E2E_HERMES_GPU_STARTUP_SCENARIO" != fallback ]; then exit 0 fi recovery_failed=0 restored_runtime="" trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}" trusted_state_root=/var/lib/nemoclaw-e2e state_prefix="hermes-gpu-fallback-docker-runtime.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.fallback." run_trusted_fixture() { /usr/bin/sudo -n /usr/bin/env -i \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ /bin/bash "$trusted_fixture" "$@" } if ! /usr/bin/sudo /usr/bin/find "$trusted_state_root" -mindepth 1 -maxdepth 1 \ -type d -name "${state_prefix}*" -print0 \ | while IFS= read -r -d '' state_dir; do if ! restored_runtime="$(run_trusted_fixture restore \ "$state_dir" /etc/docker/daemon.json)"; then exit 1 fi if [ -n "$restored_runtime" ]; then printf '%s\n' "$restored_runtime" \ >"$E2E_ARTIFACT_DIR/docker-default-runtime-restored.txt" fi done; then recovery_failed=1 fi if [ "$recovery_failed" -ne 0 ]; then echo "Independent Docker daemon recovery could not prove restoration" >&2 exit 1 fi - name: Remove trusted Hermes GPU runtime fixture if: ${{ always() && matrix.scenario == 'fallback' }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} ENV: /dev/null run: | set -euo pipefail trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}" /usr/bin/sudo /usr/bin/rm -f -- "$trusted_fixture" - name: Upload Hermes GPU startup artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-hermes-gpu-startup-${{ matrix.scenario }}-${{ matrix.runtime_provider }} path: e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/${{ matrix.runtime_provider }}/ - name: Restore Docker CLI after native Podman E2E if: ${{ always() && matrix.runtime_provider == 'podman' }} uses: NVIDIA/NemoClaw/.github/actions/restore-native-podman-e2e@9650336899bf836db5844381a97cbc2b0fe4a2b8 with: enabled: "true" - name: Clean up Docker auth if: always() shell: bash run: bash .github/scripts/docker-auth-cleanup.sh jetson-nvmap-gpu: needs: [base-image-publication, generate-matrix] if: ${{ always() && needs['base-image-publication'].result == 'success' && needs['base-image-publication'].outputs.managed_image_revision != '' && needs['generate-matrix'].result == 'success' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.allow_jetson_dispatch && (inputs.checkout_repository == '' || inputs.checkout_repository == github.repository) && ((inputs.jobs == '' && inputs.targets == '') || contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'jetson-nvmap-gpu')))) }} concurrency: group: jetson-nvmap-gpu-dispatch cancel-in-progress: false runs-on: ubuntu-latest timeout-minutes: 60 permissions: contents: read id-token: write steps: - name: Check out trusted Jetson controller uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: NVIDIA/NemoClaw ref: ${{ github.workflow_sha }} persist-credentials: false - name: Set up Node for Jetson controller uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24.22.1 - name: Install reviewed npm uses: NVIDIA/NemoClaw/.github/actions/setup-reviewed-npm@98669f24d35f18e49b6b2769cd68709509ea24f2 - name: Dispatch exact commit to Jetson through operator backend env: E2E_ARTIFACT_DIR: ${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu JETSON_DISPATCH_CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} JETSON_DISPATCH_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} JETSON_DISPATCH_URL: ${{ vars.JETSON_DISPATCH_URL }} run: node --no-warnings tools/e2e/jetson-dispatch-client.mts - name: Upload Jetson nvmap GPU artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-jetson-nvmap-gpu path: ${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu/ dgx-station-express: needs: [base-image-publication, generate-matrix] if: ${{ always() && needs['base-image-publication'].result == 'success' && needs['base-image-publication'].outputs.managed_image_revision != '' && needs['generate-matrix'].result == 'success' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || inputs.checkout_sha != inputs.base_sha) && (inputs.checkout_repository == '' || inputs.checkout_repository == github.repository) && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'dgx-station-express') && ((inputs.jobs == 'dgx-station-express' && inputs.targets == '') || (inputs.targets == 'dgx-station-express' && inputs.jobs == '')) }} concurrency: group: dgx-station-express-dispatch cancel-in-progress: false runs-on: ubuntu-latest timeout-minutes: 60 permissions: contents: read id-token: write env: E2E_JOB: "1" E2E_TARGET_ID: dgx-station-express E2E_TEST_FILE: test/e2e/live/dgx-express.test.ts E2E_DEFAULT_ENABLED: "0" E2E_GATEWAY_RUNTIMES: agnostic E2E_AGENT_RUNTIME: openclaw E2E_OBSERVABLE_OUTCOME: Station Express installs managed Super VL vLLM and NemoClaw, serves sandbox inference, and removes job runtime E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: Dedicated DGX Station GB300 with cached Nemotron 3.5 Super VL 120B steps: - name: Check out trusted Station controller uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: NVIDIA/NemoClaw ref: ${{ github.workflow_sha }} persist-credentials: false - name: Set up Node for Station controller uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24.22.1 - name: Install reviewed npm uses: NVIDIA/NemoClaw/.github/actions/setup-reviewed-npm@98669f24d35f18e49b6b2769cd68709509ea24f2 - name: Dispatch exact commit to Station through operator backend env: E2E_ARTIFACT_DIR: ${{ runner.temp }}/e2e-artifacts/live/dgx-station-express DGX_STATION_DISPATCH_CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} DGX_STATION_DISPATCH_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} DGX_STATION_DISPATCH_URL: ${{ vars.DGX_STATION_DISPATCH_URL }} run: node --no-warnings tools/e2e/dgx-station-dispatch-client.mts - name: Upload Station Express artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-dgx-station-express path: ${{ runner.temp }}/e2e-artifacts/live/dgx-station-express/ cloud-onboard: name: Cloud onboard (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix, package-openshell-sdk] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'cloud-onboard') }} runs-on: ubuntu-latest timeout-minutes: 70 strategy: fail-fast: false matrix: runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['cloud-onboard'] }} env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" E2E_GATEWAY_RUNTIMES: docker,podman E2E_TARGET_ID: "cloud-onboard" E2E_AGENT_RUNTIME: "openclaw" E2E_OBSERVABLE_OUTCOME: "Public install onboarding hosted inference and security checks succeed" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/cloud-onboard/${{ matrix.runtime_provider }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_SANDBOX_NAME: "e2e-cloud-onboard" NEMOCLAW_PUBLIC_INSTALL_REF: "${{ inputs.checkout_sha || github.sha }}" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - *dockerhub-auth - name: Configure cloud-onboard trace directory shell: bash run: | set -euo pipefail printf 'NEMOCLAW_TRACE_DIR=%s\n' "${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces" >> "${GITHUB_ENV}" # invalidState: the cloud-onboard DCode TUI check requires a PTY driver, # but the fixed GitHub-hosted runner image does not provide expect. # sourceBoundary: privileged host setup runs from the first-party # host-dependency-setup action pinned to an immutable full SHA, never the # repository target ref; the check only consumes expect without privilege. # whyNotSourceFix: GitHub-hosted jobs cannot use a repository-owned host # image, and caching privileged dpkg state between clean runners is not # supported. # regressionTest: workflow-boundary tests pin the ordering, action # provenance, and package mapping. # removalCondition: remove when the hosted runner supplies expect or the # cloud-onboard acceptance check no longer requires a PTY. - name: Install cloud-onboard DCode TUI host dependencies uses: NVIDIA/NemoClaw/.github/actions/host-dependency-setup@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: packages: expect - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" install-dependencies: "false" - name: Install reviewed OpenShell SDK archive without package credentials uses: NVIDIA/NemoClaw/.github/actions/install-reviewed-openshell-sdk@342c8f0ab115febb905eb4729d32478f22207589 - name: Restore exact-commit CLI artifact uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@3c6d991640414a2d78f95f629d67194d44829554 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Prepare native Podman E2E runtime uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} - name: Install OpenShell CLI run: bash scripts/install-openshell.sh - name: Hide Docker CLI from native Podman public install if: ${{ matrix.runtime_provider == 'podman' }} shell: bash run: | set -euo pipefail install -d -m 0700 "${E2E_ARTIFACT_DIR}" if command -v docker >/dev/null 2>&1; then echo "::error::Native Podman public install still resolves a Docker CLI" >&2 exit 1 fi jq -n \ '{schemaVersion: 1, dockerClientAvailable: false}' \ >"${E2E_ARTIFACT_DIR}/docker-client-absence.json" - name: Run cloud-onboard live Vitest test env: NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} run: | set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" if command -v openshell >/dev/null 2>&1; then OPENSHELL_BIN="$(command -v openshell)" elif [ -x "$HOME/.local/bin/openshell" ]; then OPENSHELL_BIN="$HOME/.local/bin/openshell" else echo "::error::OpenShell CLI not found after install" exit 1 fi export OPENSHELL_BIN "$OPENSHELL_BIN" --version npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/cloud-onboard.test.ts - name: Build trusted cloud-onboard timing summary if: always() shell: bash run: | set -euo pipefail NEMOCLAW_TRACE_DIR="${NEMOCLAW_TRACE_DIR:-}" expected_trace_dir="${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces" if [ -z "${RUNNER_TEMP}" ] || [ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]; then echo "::error title=E2E trace sanitization refused::NEMOCLAW_TRACE_DIR does not match its workflow-owned RUNNER_TEMP path. No raw traces were read or uploaded. Correct the trace path configuration before rerunning." >&2 printf 'Expected trace path: %s\n' "${expected_trace_dir}" >&2 exit 1 fi python3 scripts/e2e/sanitize-trace-timing.py \ "${NEMOCLAW_TRACE_DIR}" \ "${E2E_ARTIFACT_DIR}" # The target process must emit full local traces for diagnosis, but those # traces may contain prompts, environment data, and credential material. # Keep cleanup as a separate always() step so a sanitizer failure cannot # bypass it. A runner-level termination is contained by the ephemeral # GitHub-hosted runner.temp boundary. Remove this step only when the trace # producer itself emits the allowlisted timing-only schema. - name: Delete raw cloud-onboard traces if: always() shell: bash run: | set -euo pipefail NEMOCLAW_TRACE_DIR="${NEMOCLAW_TRACE_DIR:-}" expected_trace_dir="${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces" if [ -z "${RUNNER_TEMP}" ] || [ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]; then echo "::error title=E2E trace cleanup refused::NEMOCLAW_TRACE_DIR does not match its workflow-owned RUNNER_TEMP path. No files were deleted. Correct the trace path configuration before rerunning." >&2 printf 'Expected trace path: %s\n' "${expected_trace_dir}" >&2 exit 1 fi rm -rf -- "${NEMOCLAW_TRACE_DIR}" - name: Upload cloud-onboard artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-cloud-onboard-${{ matrix.runtime_provider }} path: e2e-artifacts/live/cloud-onboard/${{ matrix.runtime_provider }}/ - name: Restore Docker CLI after native Podman public install if: ${{ always() && matrix.runtime_provider == 'podman' }} uses: NVIDIA/NemoClaw/.github/actions/restore-native-podman-e2e@9650336899bf836db5844381a97cbc2b0fe4a2b8 with: enabled: "true" - name: Clean up Docker auth if: always() shell: bash run: bash .github/scripts/docker-auth-cleanup.sh messaging-providers: name: Messaging providers (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix, package-openshell-sdk] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'messaging-providers') }} runs-on: ubuntu-latest timeout-minutes: 90 strategy: fail-fast: false matrix: runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['messaging-providers'] }} env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" E2E_GATEWAY_RUNTIMES: docker,podman E2E_TARGET_ID: "messaging-providers" E2E_AGENT_RUNTIME: "openclaw" E2E_OBSERVABLE_OUTCOME: "Provider configuration redaction and optional real sends succeed" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference and messaging providers" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/messaging-providers/${{ matrix.runtime_provider }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" OPENSHELL_GATEWAY: "nemoclaw" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false - *dockerhub-auth - name: Download reviewed OpenShell SDK archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ needs.package-openshell-sdk.outputs.artifact_name }} path: ${{ runner.temp }}/openshell-sdk - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@e7e587da7c6c2c48aa112533c7eaeb549e40a351 with: build-cli: "false" install-dependencies: "false" - name: Install reviewed OpenShell SDK archive without package credentials uses: NVIDIA/NemoClaw/.github/actions/install-reviewed-openshell-sdk@342c8f0ab115febb905eb4729d32478f22207589 - name: Restore exact-commit CLI artifact uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@3c6d991640414a2d78f95f629d67194d44829554 with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} - name: Prepare native Podman E2E runtime uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@e85857e2a7bfb2af01241ecbce5947ee774c1280 with: enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} - name: Run messaging providers live Vitest test # The test keeps # the legacy fake-token defaults, optional _REAL secret overrides, # provider placeholder redaction checks, and WhatsApp QR-only parity. env: NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} TELEGRAM_BOT_TOKEN_REAL: ${{ secrets.TELEGRAM_BOT_TOKEN_REAL }} DISCORD_BOT_TOKEN_REAL: ${{ secrets.DISCORD_BOT_TOKEN_REAL }} SLACK_BOT_TOKEN_REAL: ${{ secrets.SLACK_BOT_TOKEN_REAL }} SLACK_APP_TOKEN_REAL: ${{ secrets.SLACK_APP_TOKEN_REAL }} TELEGRAM_ALLOWED_IDS: ${{ secrets.TELEGRAM_ALLOWED_IDS }} TELEGRAM_AUTHORIZED_CHAT_IDS: ${{ secrets.TELEGRAM_AUTHORIZED_CHAT_IDS }} TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }} TELEGRAM_CHAT_ID_E2E: ${{ secrets.TELEGRAM_CHAT_ID_E2E }} DISCORD_CHANNEL_ID_E2E: ${{ secrets.DISCORD_CHANNEL_ID_E2E }} SLACK_CHANNEL_ID_E2E: ${{ secrets.SLACK_CHANNEL_ID_E2E }} run: | set -euo pipefail npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/messaging-providers.test.ts - name: Upload messaging providers artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-messaging-providers-${{ matrix.runtime_provider }} path: e2e-artifacts/live/messaging-providers/${{ matrix.runtime_provider }}/ - name: Restore Docker CLI after native Podman E2E if: ${{ always() && matrix.runtime_provider == 'podman' }} uses: NVIDIA/NemoClaw/.github/actions/restore-native-podman-e2e@9650336899bf836db5844381a97cbc2b0fe4a2b8 with: enabled: "true" - name: Clean up Docker auth if: always() shell: bash run: bash .github/scripts/docker-auth-cleanup.sh report-to-pr: runs-on: ubuntu-latest timeout-minutes: 15 # This entire workflow is dispatch-only. Keeping selective jobs in `needs` # makes the report wait for and record any requested job without adding # skipped checks to the normal pull_request workflow. needs: &e2e-result-jobs [ base-image-publication, generate-matrix, llama-cpp-dgx-spark-plan, llama-cpp-dgx-spark-qualification, staging-brev-launchable, staging-brev-launchable-identity, live, shared-e2e, catalogue-standard, catalogue-nvidia-api, catalogue-nvidia-inference, catalogue-hosted-inference, catalogue-github-read, openshell-gateway-auth-contract, external-gateway-health, mcp-bridge, openshell-credential-generation-window, openshell-dev-artifact, mcp-bridge-dev, managed-image-multiarch-startup, managed-image-protected-runtime, hermes-e2e, hermes-gpu-startup, portable-hermes-finalization, messaging-providers, jetson-nvmap-gpu, dgx-station-express, cloud-onboard, native-runtime-qualification-producer, native-runtime-qualification-producer-aggregate, ] if: ${{ always() && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || (github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && needs.generate-matrix.result == 'success' && (contains(fromJSON(needs.generate-matrix.outputs.selected_workflow_jobs || '[]'), 'catalogue-hosted-inference') || contains(fromJSON(needs.generate-matrix.outputs.selected_workflow_jobs || '[]'), 'llama-cpp-dgx-spark-qualification')))) }} permissions: actions: read contents: read # The issue-comment endpoint accepts pull request write permission for PR comments. # Keep issues: write absent so this job cannot restore general issue routing. pull-requests: write steps: - name: Check out the trusted E2E reporting helper uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.workflow_sha }} persist-credentials: false sparse-checkout: | tools/e2e/report-e2e-results.mts tools/e2e/selector-aliases.mts sparse-checkout-cone-mode: false - name: Post E2E target results to PR uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }} TEST_MATRIX: ${{ needs.generate-matrix.outputs.test_matrix }} JOB_CHECKOUT_SHA: ${{ inputs.checkout_sha }} JOB_CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }} JOB_BASE_SHA: ${{ inputs.base_sha }} JOB_WORKFLOW_SHA: ${{ inputs.workflow_sha }} WORKFLOW_SHA: ${{ github.workflow_sha }} MATRIX_RESULT: ${{ needs.generate-matrix.result }} SELECTED_WORKFLOW_JOBS: ${{ needs.generate-matrix.outputs.selected_workflow_jobs }} JOB_PR_NUMBER: ${{ inputs.pr_number }} JOB_TARGETS: ${{ inputs.targets }} JOBS: ${{ inputs.jobs }} NEEDS_JSON: ${{ toJSON(needs) }} with: script: | const path = require('node:path'); const { pathToFileURL } = require('node:url'); const { resolveReportPr, loadReportJobs, renderE2eReport } = await import( pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'tools/e2e/report-e2e-results.mts')).href ); const needs = JSON.parse(process.env.NEEDS_JSON || '{}'); const prNumber = await resolveReportPr({ github, context, core, env: process.env }); if (prNumber === undefined) return; const { apiJobs, loaded } = await loadReportJobs({ github, context, core }); const report = renderE2eReport({ needs, env: process.env, apiJobs, apiJobsLoaded: loaded, context, }); if (report.fatal) { core.setFailed(report.fatal); return; } for (const warning of report.warnings) core.warning(warning); await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: prNumber, body: report.body, }); relevant-e2e: name: Relevant E2E runs-on: ubuntu-latest timeout-minutes: 5 needs: *e2e-result-jobs if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '')) }} permissions: contents: read steps: - name: Check out the E2E result evaluator uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.workflow_sha }} persist-credentials: false sparse-checkout: tools/e2e/release-qualification.mts sparse-checkout-cone-mode: false - name: Require every selected E2E result env: MANAGED_IMAGE_REQUIRED: ${{ needs.generate-matrix.outputs.managed_image_required }} NEEDS_JSON: ${{ toJSON(needs) }} RELEASE_REQUIRED_JOBS: ${{ needs.generate-matrix.outputs.selected_workflow_jobs }} E2E_RESULT_PATH: ${{ inputs.checkout_sha != '' && format('{0}/review-queue-e2e-result.json', runner.temp) || '' }} run: node --no-warnings tools/e2e/release-qualification.mts - name: Upload PR E2E results if: ${{ always() && inputs.checkout_sha != '' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: review-queue-e2e-result-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/review-queue-e2e-result.json if-no-files-found: error - name: Summarize selected E2E env: SELECTED_WORKFLOW_JOBS: ${{ needs.generate-matrix.outputs.selected_workflow_jobs }} run: | set -euo pipefail if [ "${SELECTED_WORKFLOW_JOBS}" = "[]" ]; then echo "No E2E target owns a file changed by this push." >> "${GITHUB_STEP_SUMMARY}" else { echo "The selected E2E workflow jobs passed:" echo jq -r '.[] | "- `\(.)`"' <<< "${SELECTED_WORKFLOW_JOBS}" } >> "${GITHUB_STEP_SUMMARY}" fi release-qualification: name: Release qualification runs-on: ubuntu-latest timeout-minutes: 5 needs: *e2e-result-jobs if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && inputs.jobs == '' && inputs.targets == '' && inputs.include_staging_brev_launchable && !inputs.allow_jetson_dispatch }} permissions: contents: read steps: - name: Check out the qualification evaluator uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.workflow_sha }} persist-credentials: false sparse-checkout: tools/e2e/release-qualification.mts sparse-checkout-cone-mode: false - name: Require every release E2E result env: NEEDS_JSON: ${{ toJSON(needs) }} RELEASE_REQUIRED_JOBS: ${{ needs.generate-matrix.outputs.release_required_jobs }} run: node --no-warnings tools/e2e/release-qualification.mts # ── Push/manual scorecard ───────────────────────────────────────────────── scorecard: runs-on: ubuntu-latest timeout-minutes: 15 needs: *e2e-result-jobs if: ${{ always() && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '')) }} permissions: actions: read contents: read steps: - name: Checkout scorecard builders uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} ref: ${{ inputs.checkout_sha || github.sha }} fetch-depth: 0 persist-credentials: false sparse-checkout: | ci/onboard-performance-budget.json scripts/audit-test-runtime.mts scripts/lib/read-artifact-zip.mts scripts/scorecard sparse-checkout-cone-mode: false - name: Download E2E progress artifacts continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: ${{ runner.temp }}/e2e-runtime-audit pattern: e2e-* - name: Generate E2E scorecard id: scorecard uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }} JOBS: ${{ inputs.jobs }} RUNTIME_ARTIFACTS: ${{ runner.temp }}/e2e-runtime-audit RUNTIME_SUMMARY_FILE: ${{ runner.temp }}/e2e-runtime-summary.json TARGETS: ${{ inputs.targets }} NEEDS_JSON: ${{ toJSON(needs) }} with: script: | const path = require('path'); const { buildScorecard } = require( path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/coordinate-scorecard.mts'), ); const scorecardJobs = require( path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/summarize-jobs.mts'), ); const traceTiming = require( path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-trace-timing.mts'), ); const runtimeAudit = require( path.join(process.env.GITHUB_WORKSPACE, 'scripts/audit-test-runtime.mts'), ); const runtimeHistory = require( path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-runtime-history.mts'), ); const firstTurnLatency = require( path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-first-turn-latency.mts'), ); const sandboxPhaseTail = require( path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-sandbox-phase-tail.mts'), ); const needs = JSON.parse(process.env.NEEDS_JSON || '{}'); // GitHub's jobs API is the canonical source because `needs.live` // collapses every matrix target into one result and has no job URL. // The typed helper owns and tests the degraded `needs` fallback. const apiJobs = await scorecardJobs.loadWorkflowRunJobs({ github, context, core }); let runtimeSummaryMarkdown; let runtimeHistoryRows = null; try { const runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]); runtimeSummaryMarkdown = runtimeAudit.formatRuntimeAuditSummary(runtimeRows); runtimeHistoryRows = runtimeAudit.collectRuntimeHistorySamples([ process.env.RUNTIME_ARTIFACTS, ]); } catch { core.warning('E2E test phase runtime summary unavailable: invalid progress artifact'); runtimeSummaryMarkdown = [ '## E2E Test Phase Runtime', '', 'The summary is unavailable because a `test-progress.json` artifact was invalid.', '', ].join('\n'); } const runtimeHistoryMarkdown = runtimeHistoryRows === null ? [ '## E2E Push Runtime Trend', '', 'The trend is unavailable because a `test-progress.json` artifact was invalid.', '', ].join('\n') : await runtimeHistory.buildRuntimeHistory( { github, context, core }, runtimeHistoryRows, process.env.RUNTIME_SUMMARY_FILE, { currentFirstTurnLatency: firstTurnLatency.readCurrentFirstTurnLatencySample( process.env.RUNTIME_ARTIFACTS, ), currentSandboxPhaseTail: sandboxPhaseTail.readCurrentSandboxPhaseTailSample( process.env.RUNTIME_ARTIFACTS, ), loadPriorPushHistory: runtimeHistory.loadPriorPushHistory, }, ); const trace = await traceTiming.buildTraceTimingResult({ github, context, core }); if (trace.budgetWarningMessage) core.warning(trace.budgetWarningMessage); const { summaryMarkdown, scorecardData, slackData } = buildScorecard({ eventName: context.eventName, actor: context.actor || '', serverUrl: context.serverUrl, repo: context.repo, runId: context.runId, rawJobs: process.env.JOBS || '', rawTargets: process.env.TARGETS || '', rawExplicitOnly: process.env.EXPLICIT_ONLY_JOBS || '', needs, apiJobs, trace, today: new Date().toLocaleDateString('en-US', { month: 'short', day: 'numeric' }), }); await core.summary .addRaw(`${summaryMarkdown}\n\n${runtimeSummaryMarkdown}\n${runtimeHistoryMarkdown}`) .write(); core.setOutput('scorecardData', JSON.stringify(scorecardData)); core.setOutput('slackData', JSON.stringify(slackData)); - name: Post scorecard to Slack # Webhook secrets never enter branch-dispatched runs. The payload is # computed in the preceding no-secret step; this fixed publisher does # not load code from the checked-out workflow ref. if: ${{ steps.scorecard.outputs.slackData != '' && github.ref == 'refs/heads/main' }} uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: SLACK_WEBHOOK_URL_DAILY: ${{ secrets.SLACK_WEBHOOK_URL_DAILY }} SLACK_WEBHOOK_URL_FULLRUN: ${{ secrets.SLACK_WEBHOOK_URL_FULLRUN }} SLACK_WEBHOOK_URL_PREVIEW: ${{ secrets.SLACK_WEBHOOK_URL_PREVIEW }} SLACK_DATA: ${{ steps.scorecard.outputs.slackData }} POST_TO_SLACK: ${{ inputs.post_to_slack }} with: script: | const data = JSON.parse(process.env.SLACK_DATA); const envByChannel = { daily: 'SLACK_WEBHOOK_URL_DAILY', fullrun: 'SLACK_WEBHOOK_URL_FULLRUN', preview: 'SLACK_WEBHOOK_URL_PREVIEW', }; const channel = data?.channel; const payload = data?.payload; const attachments = payload?.attachments; const validPayload = Object.hasOwn(envByChannel, channel) && payload !== null && typeof payload === 'object' && typeof payload.text === 'string' && Array.isArray(attachments) && attachments.every( (attachment) => attachment !== null && typeof attachment === 'object' && typeof attachment.color === 'string' && Array.isArray(attachment.blocks), ); if (!validPayload) { core.setFailed('Invalid precomputed Slack payload'); return; } if (channel === 'preview' && process.env.POST_TO_SLACK !== 'true') { core.info('Selective dispatch without post_to_slack — skipping'); return; } const webhookUrl = process.env[envByChannel[channel]]; if (!webhookUrl) { core.info(`Slack webhook for "${channel}" not configured — skipping`); return; } const response = await fetch(webhookUrl, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(data.payload), }); if (!response.ok) { core.setFailed(`Slack webhook returned ${response.status}`); } - name: Upload E2E runtime summary if: ${{ always() && github.event_name == 'push' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: name: e2e-runtime-summary path: ${{ runner.temp }}/e2e-runtime-summary.json