name: "zzz Smoke tests"

on:
  workflow_call:
    inputs:
      docker-image:
        required: true
        type: string
      harbor-prefix:
        required: false
        type: string
        default: ""
      cpu-docker-image:
        required: false
        type: string
        default: ""
        description: >-
          Canonical (non-Harbor-prefixed, publicly pullable) image ref used for the
          github_hosted_cpu SKU. GitHub-hosted runners have no Harbor access, so the
          Harbor-prefixed docker-image ref cannot be pulled there. Same idea as
          docs-latest-public.yaml, which pulls the raw GHCR tag on ubuntu-latest.
      package-artifact-name:
        required: true
        type: string
      prebuilt-image:
        required: false
        type: string
        default: ""
        description: >-
          Image with the packages already installed. When set it replaces
          docker-image and the download and install steps are skipped. Empty
          means the artifact path, which is what forks always get.
      enabled-skus:
        required: false
        type: string
        default: ALL_SKUS_IN_TESTS
        description: "Comma-separated SKUs to run, or ALL_SKUS_IN_TESTS to run every SKU in the tests yaml."
      product:
        required: true
        type: string # tt-metalium or tt-nn
      upload-coverage:
        required: false
        type: boolean
        default: false
      enable-watcher:
        description: "Enable watcher"
        default: false
        type: boolean
      enable-kernel-ccache:
        description: 'Enable kernel ccache for JIT compilation at runtime'
        default: false
        type: boolean
      test-category:
        required: false
        type: string
        default: merge_gate
        description: "Which tests yaml to load: the merge_gate or the pr_gate validation list."

jobs:
  load-test-matrix:
    runs-on: ubuntu-slim
    outputs:
      matrix: ${{ steps.build-matrix.outputs.matrix }}
    env:
      TESTS_YAML_PATH: >-
        ${{ inputs.product == 'tt-metalium'
            && (inputs.test-category == 'pr_gate'
                && './tests/pipeline_reorg/runtime_validation_pr_gate_tests.yaml'
                || './tests/pipeline_reorg/runtime_validation_merge_gate_tests.yaml')
            || (inputs.test-category == 'pr_gate'
                && './tests/pipeline_reorg/ttnn_validation_pr_gate_tests.yaml'
                || './tests/pipeline_reorg/ttnn_validation_merge_gate_tests.yaml') }}
    steps:
      - name: Checkout the repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        timeout-minutes: 20
        with:
          sparse-checkout: |
            .github
            tests/pipeline_reorg
          sparse-checkout-cone-mode: true

      - name: Install dependencies
        run: pip3 install PyYAML


      - name: Build test matrix based on enabled SKUs
        id: build-matrix
        uses: tenstorrent/tt-metal/.github/actions/prepare-test-matrix@main
        with:
          tests-yaml: "${{ env.TESTS_YAML_PATH }}"
          enabled-skus: "${{ inputs.enabled-skus }}"
          sku-config: ./.github/sku_config.yaml
          event: ${{ github.event_name }}

  smoke:
    needs: load-test-matrix
    if: ${{ needs.load-test-matrix.outputs.matrix != '[]' }}
    strategy:
      fail-fast: false
      matrix:
        test-group: ${{ fromJson(needs.load-test-matrix.outputs.matrix) }}
    name: ${{ matrix.test-group.name }}
    environment: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'mainline' || '' }}
    timeout-minutes: 30
    runs-on: ${{ matrix.test-group.runs_on }}
    container:
      # github_hosted_cpu runs on GitHub-hosted runners with no Harbor access, so it
      # needs the canonical (publicly pullable) cpu-docker-image ref instead of the
      # Harbor-prefixed one (same reason dedicated-merge-gate uses the raw ref).
      # A prebuilt image already carries the packages, and is a canonical ghcr
      # ref so it needs no Harbor prefix on any SKU.
      image: ${{ inputs.prebuilt-image || (matrix.test-group.sku == 'github_hosted_cpu' && (inputs.cpu-docker-image || inputs.docker-image)) || (inputs.docker-image && (contains(join(matrix.test-group.runs_on, ','), 'dedicated-merge-gate') && inputs.docker-image || format('{0}{1}', inputs.harbor-prefix, inputs.docker-image))) || 'docker-image-unresolved!' }}
      env:
        ASAN_OPTIONS: "color=always"
        TSAN_OPTIONS: "color=always"
        UBSAN_OPTIONS: "color=always:print_stacktrace=1:halt_on_error=1"
        LLVM_PROFILE_FILE: "/work/coverage/%p-%m.profraw"
      volumes:
        - /work
        # No hugepages on viommu runners or on GitHub-hosted CPU-only runners.
        - ${{ (!contains(join(matrix.test-group.runs_on, ','), 'viommu') && matrix.test-group.sku != 'github_hosted_cpu' && '/dev/hugepages-1G:/dev/hugepages-1G') || '/no_hugepages' }}
      # Memory reduction in WH LB coming up: https://github.com/tenstorrent/metal-internal-workflows/issues/786
      # github_hosted_cpu has no /dev/tenstorrent to pass through.
      options: ${{ matrix.test-group.sku != 'github_hosted_cpu' && '--device /dev/tenstorrent' || '' }} ${{ contains(matrix.test-group.sku, 'llmbox') && '--memory 256g' || '' }}
    defaults:
      run:
        shell: bash
        working-directory: /work # https://github.com/actions/runner/issues/878
    steps:
      - name: Checkout actions
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        timeout-minutes: 20
        with:
          sparse-checkout: |
            .github/actions/download-artifact-with-retry
            .github/scripts/ccache_log_filter.sh
          sparse-checkout-cone-mode: false

      - uses: ./.github/actions/download-artifact-with-retry
        if: ${{ inputs.prebuilt-image == '' }}
        timeout-minutes: 15
        with:
          name: ${{ inputs.package-artifact-name || 'packages artifact unresolved!' }}
          path: /work/pkgs/

      - name: Check available memory in container for WH LB 2
        if: ${{ contains(matrix.test-group.sku, 'llmbox') }}
        run: |
          # You never know with GH!!!
          set -e

          # Read memory limit for container cgroup from cgroup v2 or v1
          memory_bytes=$(cat /sys/fs/cgroup/memory.max 2>/dev/null || \
                         cat /sys/fs/cgroup/memory/memory.limit_in_bytes 2>/dev/null)

          if [ -z "$memory_bytes" ]; then
              echo "Unable to read memory limit from cgroup (v1 or v2)" >&2
              exit 1
          fi

          # 256GB = 274877906944 bytes
          max_bytes_wh_lb2=274877906944

          if [ "$memory_bytes" -gt "$max_bytes_wh_lb2" ]; then
              echo "Error: Container memory limit ${memory_bytes} upcoming max of 256GB (${max_bytes_wh_lb2} bytes)" >&2
              exit 1
          fi

          echo "Memory limit validated: ${memory_bytes} bytes"

      - name: Install packages
        if: ${{ inputs.prebuilt-image == '' }}
        timeout-minutes: 10
        run: |
          # Ideally only ${{ inputs.product }}-validation, but APT doesn't resolve dependencies from files on disk without being told about them.
          # tt-metalium-jit is optional: TT_USE_SYSTEM_SFPI builds don't produce it, since this
          # container already has /opt/tenstorrent/sfpi for the JIT-build runtime fallback
          # (see tt_metal/jit_build/build.cpp).
          shopt -s nullglob
          pkgs=(./pkgs/tt-metalium_*.deb ./pkgs/tt-metalium-jit_*.deb ./pkgs/tt-metalium-validation_*.deb ./pkgs/${{ inputs.product }}_*.deb ./pkgs/${{ inputs.product }}-validation_*.deb)
          apt install -y "${pkgs[@]}"

      - name: Prepare coverage directory
        run: |
          mkdir -p /work/coverage

      - name: Warmup caches
        if: false # No need to warmup currently because slow tests will never fail the job
        timeout-minutes: 2
        env:
          GTEST_COLOR: yes
          LSAN_OPTIONS: suppressions=/usr/share/tt-metalium/lsan.supp
          # TT_METAL_WATCHER: 5
          TT_METAL_WATCHER_TEST_MODE: 1
        run: |
          # Run a single test to warm up caches before the full test suite
          /usr/bin/${{ inputs.product }}-validation-smoke --gtest_filter="MeshDispatchFixture.TensixFailOnDuplicateKernelCreationDataflow" || true

      - name: Clear coverage from warmup
        if: ${{ inputs.upload-coverage }}
        run: |
          rm -f /work/coverage/*.profraw || true

      - name: Set up env vars for watcher
        if: ${{ inputs.enable-watcher == true }}
        run: |
          echo "Enabling TT Metal Watcher"
          echo "TT_METAL_WATCHER=2" >> $GITHUB_ENV
          echo "TT_METAL_WATCHER_APPEND=1" >> $GITHUB_ENV
          echo "TT_METAL_WATCHER_NOINLINE=1" >> $GITHUB_ENV
          echo "TT_METAL_WATCHER_DISABLE_ETH=1" >> $GITHUB_ENV

      - name: Set up kernel ccache
        if: ${{ inputs.enable-kernel-ccache == true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) }}
        run: |
          echo "Enabling TT Metal kernel ccache"
          echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
          echo "CCACHE_REMOTE_ONLY=true" >> $GITHUB_ENV
          echo "CCACHE_NOHASHDIR=true" >> $GITHUB_ENV
          echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV
          echo "CCACHE_COMPRESSLEVEL=3" >> $GITHUB_ENV
          echo "CCACHE_LOGFILE=/tmp/ccache.log" >> $GITHUB_ENV
          echo "CCACHE_BASEDIR=/work" >> $GITHUB_ENV
          echo "TT_METAL_CCACHE_KERNEL_SUPPORT=1" >> $GITHUB_ENV
          echo "CCACHE_TEMPDIR=/tmp/ccache" >> $GITHUB_ENV
          mkdir -p /tmp/ccache
          # Build Redis URL in bash so that the password is never embedded in a
          # workflow input string (GHA does not pass secrets through string inputs
          # to reusable workflows). This also ensures forks leave CCACHE_REMOTE_STORAGE
          # unset — the step is guarded by the fork check above.
          REDIS_USER="${{ vars.REDIS_CCACHE_USER }}"
          REDIS_PASSWORD="${{ secrets.REDIS_CCACHE_PASSWORD }}"
          REDIS_HOST="${{ vars.REDIS_CCACHE_HOST }}"
          REDIS_PORT="${{ vars.REDIS_CCACHE_PORT }}"
          REDIS_READONLY="${{ vars.REDIS_CCACHE_IS_READONLY }}"
          CCACHE_REMOTE_STORAGE="redis://${REDIS_USER}:${REDIS_PASSWORD}@${REDIS_HOST}:${REDIS_PORT}"
          # ccache 4.13 bug: the read-only optional trips the write gate whenever
          # it has *any* value — even false. Only append it when actually true.
          if [ "${REDIS_READONLY}" = "true" ]; then
            CCACHE_REMOTE_STORAGE="${CCACHE_REMOTE_STORAGE}|read-only=true"
          fi
          echo "CCACHE_REMOTE_STORAGE=${CCACHE_REMOTE_STORAGE}" >> $GITHUB_ENV

      - name: Run a test
        id: test
        timeout-minutes: ${{ matrix.test-group.timeout }}
        env:
          GTEST_COLOR: yes
          GTEST_OUTPUT: xml:/work/test-reports/
          LSAN_OPTIONS: suppressions=/usr/share/tt-metalium/lsan.supp
          TT_METAL_WATCHER_TEST_MODE: 1
          TT_LOGGER_LEVEL: warn
        run: ${{ matrix.test-group.cmd }}

      - name: Merge LLVM coverage
        if: ${{ inputs.upload-coverage }}
        run: |
          shopt -s nullglob || true
          files=(/work/coverage/*.profraw)
          if [ ${#files[@]} -gt 0 ]; then
            llvm-profdata-17 merge -sparse /work/coverage/*.profraw -o /work/coverage/coverage.profdata
          else
            echo "No .profraw files found; skipping merge"
          fi

      - name: Export LCOV (llvm-cov)
        if: ${{ inputs.upload-coverage }}
        run: |
          if [ -f /work/coverage/coverage.profdata ]; then
            objects=(
              /usr/lib/libtt_metal.so
              /usr/lib/libtt_stl.so
              /usr/lib/_ttnncpp.so
            )
            test_bin="/usr/bin/${{ inputs.product }}-validation-smoke"
            llvm-cov-17 export -format=lcov \
              -instr-profile=/work/coverage/coverage.profdata \
              "$test_bin" "${objects[@]}" > /work/coverage/coverage.info || {
                echo "llvm-cov export failed"; exit 1; }
          else
            echo "coverage.profdata not found; creating empty LCOV file"
            : > /work/coverage/coverage.info
          fi

      - name: Checkout repository for Codecov mapping
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        timeout-minutes: 20
        if: ${{ inputs.upload-coverage }}
        with:
          submodules: recursive
          fetch-depth: 1

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v5
        if: ${{ !cancelled() && inputs.upload-coverage && (
          (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false)
          || (github.event_name == 'push' && github.ref_name == 'main')
          ) }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          files: /work/coverage/coverage.info
          flags: ${{ inputs.product }}
          name: smoke-${{ inputs.product }}-${{ matrix.test-group.sku }}
          fail_ci_if_error: true
          verbose: true

      - name: Test Report
        # Because of https://github.com/tenstorrent/tt-metal/issues/19413, only run for our repo
        # for now. No forks!
        if: ${{ !cancelled() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) }}
        # annotate_only keeps this from creating a check run, which is what used to 404 the
        # merge gate: the queue's temporary commit could be discarded between the create and
        # the update. Annotations attach to this job, so there is no separate SHA to go stale.
        continue-on-error: true
        uses: mikepenz/action-junit-report@3585e9575db828022551b4231f165eb59a0e74e3 # v5.6.2
        with:
          report_paths: /work/test-reports/*.xml
          check_name: ${{ inputs.product }} ${{ matrix.test-group.sku }} smoke tests (${{ github.workflow }}, attempt ${{ github.run_attempt }})
          annotate_only: true
          detailed_summary: true
          include_passed: false
          fail_on_failure: false

      - uses: tenstorrent/tt-metal/.github/actions/upload-artifact-with-job-uuid@main
        if: ${{ !cancelled() }}
        timeout-minutes: 10
        with:
          path: /work/test-reports/
          prefix: "test_reports_"

      - name: Publish ccache summary
        if: ${{ !cancelled() && inputs.enable-kernel-ccache == true }}
        run: |
          echo '<details><summary>CCache Summary (kernel)</summary>' >> $GITHUB_STEP_SUMMARY
          echo '' >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          ccache -s >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          echo '' >> $GITHUB_STEP_SUMMARY
          echo '</details>' >> $GITHUB_STEP_SUMMARY

          # Surface remote storage stats and errors from the ccache log.
          # Uses a filter script to avoid overflowing the 1024k step summary limit.
          if [ -f /tmp/ccache.log ]; then
            echo '<details><summary>CCache Remote Storage (kernel)</summary>' >> $GITHUB_STEP_SUMMARY
            echo '' >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY
            "$GITHUB_WORKSPACE/.github/scripts/ccache_log_filter.sh" /tmp/ccache.log >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY
            echo '' >> $GITHUB_STEP_SUMMARY
            echo '</details>' >> $GITHUB_STEP_SUMMARY
          fi