trigger:
  branches:
    include:
      - main
  paths:
    include:
      - .github/skills/**

pr: none

parameters:
  - name: runAll
    displayName: 'Run all skill evals (ignore change detection)'
    type: boolean
    default: false
  - name: areas
    displayName: 'Skill areas to eval (comma-separated, e.g. sensei,skill-authoring)'
    type: string
    default: ' '

variables:
  - group: 'AzSDK_Eval_Variable_group'

jobs:
  - job: Evaluate
    displayName: 'Vally Eval'
    pool:
      vmImage: 'ubuntu-latest'
    steps:
      - checkout: self
        fetchDepth: 2

      - task: NodeTool@0
        displayName: 'Use Node.js 22'
        inputs:
          versionSpec: '22.x'

      - script: npm install -g @microsoft/vally-cli@0.4.0
        displayName: 'Install Vally CLI'

      - script: npm install -g @github/copilot-sdk
        displayName: 'Install Copilot SDK'

      - script: |
          input_areas=$(echo "${{ parameters.areas }}" | xargs)
          if [ -n "$input_areas" ]; then
            areas="$input_areas"
          elif [ "${{ parameters.runAll }}" = "True" ]; then
            areas=""
          else
            areas=$(git diff --name-only HEAD~1 HEAD -- .github/skills/ \
              | grep -oP '\.github/skills/\K[^/]+(?=/)' \
              | sort -u \
              | grep -v '^\.' \
              | paste -sd, -)
          fi

          if [ -n "$areas" ]; then
            echo "Evaluating areas: $areas"
            echo "##vso[task.setvariable variable=EVAL_ARGS]--tag area=$areas"
          elif [ "${{ parameters.runAll }}" = "True" ]; then
            echo "Evaluating all skills"
            echo "##vso[task.setvariable variable=EVAL_ARGS] "
          else
            echo "No skill changes detected — skipping evals"
            echo "##vso[task.setvariable variable=EVAL_ARGS]SKIP"
          fi
        displayName: 'Detect changed skills'

      - script: |
          cd .github/skills
          vally eval --tag "type=ci-gate" $(EVAL_ARGS) --output-dir ".vally/results"
        displayName: 'Run evaluations'
        condition: and(succeeded(), ne(variables['EVAL_ARGS'], 'SKIP'))
        continueOnError: true
        env:
          GITHUB_TOKEN: $(azuresdk-copilot-github-pat)

      - task: PublishPipelineArtifact@1
        displayName: 'Upload eval results'
        condition: and(always(), ne(variables['EVAL_ARGS'], 'SKIP'))
        inputs:
          targetPath: '.github/skills/.vally/results'
          artifactName: 'eval-results-$(Build.BuildId)'