name: eval

# DSH 插件回归评测门禁：跑 cases/real 全量用例（真实 LLM，需 DEEPSEEK_API_KEY），
# 与 baseline/report.json 对比出 OVERALL 判定。手动触发或 baseline 相关文件变更时触发；
# 用例/harness 代码变更时必须重跑 baseline（见 README「首轮评测结果人工复核后提交 baseline」）。

on:
  schedule:
    # 每天 02:00 Asia/Shanghai = 18:00 UTC
    - cron: '0 18 * * *'
  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - 'cases/**'
      - 'src/**'
      - 'baseline/**'
      - '.github/workflows/eval.yml'
  pull_request:
    paths:
      - 'cases/**'
      - 'src/**'
      - 'baseline/**'
      - '.github/workflows/eval.yml'

jobs:
  check-changes:
    runs-on: ubuntu-latest
    outputs:
      has_changes: ${{ steps.changes.outputs.has_changes }}
    steps:
      - name: Check recent commits
        id: changes
        uses: actions/github-script@v7
        with:
          script: |
            if (context.eventName !== 'schedule') {
              core.setOutput('has_changes', 'true')
              return
            }

            const since = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString()
            const { data: commits } = await github.rest.repos.listCommits({
              owner: context.repo.owner,
              repo: context.repo.repo,
              since,
              per_page: 1,
            })
            const hasChanges = commits.length > 0
            core.setOutput('has_changes', String(hasChanges))
            core.info(hasChanges ? 'Recent commits found in the last 24 hours.' : 'No commits found in the last 24 hours; skipping eval.')

  eval:
    needs: check-changes
    if: needs.check-changes.outputs.has_changes == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 45
    env:
      # secrets 不能直接出现在 step 级 if: 里，先落到 job 级 env 再判空
      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
    steps:
      - uses: actions/checkout@v4

      - uses: pnpm/action-setup@v4

      - uses: actions/setup-node@v4
        with:
          node-version: 22
          cache: pnpm

      - run: pnpm install --frozen-lockfile

      - run: pnpm build && pnpm test

      # dsh 依赖树大（531 包），冷装 ~6min；单独一步装好，评测用全局 dsh 二进制，
      # 避免 harness 探测 npx 时 60s 超时（npx 冷缓存解析必然超时）。
      - name: Cache npm (dsh install)
        uses: actions/cache@v4
        with:
          path: ~/.npm
          key: npm-dsh-0.1.1-rc.2

      - name: Install dsh
        run: npm install -g @deepseek-ai/dsh@0.1.1-rc.2

      - name: Run eval cases (real LLM)
        # fork / 未配置 DEEPSEEK_API_KEY 的环境跳过评测，不直接红（build+test 仍会跑）
        if: env.DEEPSEEK_API_KEY != ''
        run: |
          node --input-type=module -e "
          import { runEval } from './lib/runner.js'
          const report = await runEval({
            casesDir: 'cases/real',
            outputDir: '.eval/out',
            dshBin: 'dsh',
            concurrency: 3,
            // 偶发网络/模型抖动重跑一次；flaky 标记会留在报告里供排查
            retries: 1,
          })
          console.log(JSON.stringify(report.summary))
          "

      - name: Gate against baseline
        if: env.DEEPSEEK_API_KEY != ''
        run: |
          OUT=$(node --input-type=module -e "
          import { computeGate, loadReport, renderGateText } from './lib/gate.js'
          const after = await loadReport('.eval/out/report.json')
          const before = await loadReport('baseline/report.json', true)
          console.log(renderGateText(computeGate(before, after, false)))
          ")
          echo "$OUT"
          exit $(echo "$OUT" | sed -n 's/^EXIT_CODE=//p')

      # 只传两个报告文件，不传 .sessions/ 原始 trace（含真实工具输出，避免公开仓库暴露）
      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: eval-report
          path: |
            .eval/out/report.json
            .eval/out/report.md
          retention-days: 14