name: "LLM Models Check" description: "Check LLM model costs, status, and availability in CI/CD" author: "maxgfr" branding: icon: "cpu" color: "blue" inputs: command: description: "Command to run (find, cost, cheapest, compare, recommend, stats)" required: true args: description: "Arguments for the command" required: false default: "" max-budget: description: "Maximum total cost budget (USD). Fails the step if any model exceeds this." required: false fail-on-deprecated: description: "Fail if any matched model is deprecated (true/false)" required: false default: "false" node-version: description: "Node.js version to use (>= 22.12 required)" required: false default: "22" outputs: result: description: "JSON output of the command" value: ${{ steps.run.outputs.result }} model-count: description: "Number of models matched" value: ${{ steps.run.outputs.count }} runs: using: "composite" steps: - uses: actions/setup-node@v7 with: node-version: ${{ inputs.node-version }} - name: Run llm-models id: run shell: bash run: | # stderr (warnings, diagnostics) stays in the log: mixing it in would break the JSON. RESULT=$(npx -y llm-models@latest ${{ inputs.command }} ${{ inputs.args }} --json) echo "result<> $GITHUB_OUTPUT echo "$RESULT" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT # The checks below read this file. Pasting the JSON into their scripts broke # on the first apostrophe (model descriptions), and an env var caps its size. printf '%s' "$RESULT" > "$RUNNER_TEMP/llm-models-result.json" COUNT=$(echo "$RESULT" | node -e " const input = require('fs').readFileSync('/dev/stdin', 'utf-8'); try { const data = JSON.parse(input); console.log(Array.isArray(data) ? data.length : 1); } catch { console.log(0); } ") echo "count=$COUNT" >> $GITHUB_OUTPUT - name: Check budget if: inputs.max-budget != '' shell: bash env: MAX_BUDGET: ${{ inputs.max-budget }} run: | node -e " const fs = require('fs'); const data = JSON.parse(fs.readFileSync(process.env.RUNNER_TEMP + '/llm-models-result.json', 'utf-8')); const budget = Number(process.env.MAX_BUDGET); if (!Number.isFinite(budget)) { console.error('max-budget is not a number: ' + process.env.MAX_BUDGET); process.exit(1); } const items = Array.isArray(data) ? data : [data]; for (const item of items) { if (item.total_cost_usd && item.total_cost_usd > budget) { console.error('Budget exceeded: ' + item.model + ' costs \$' + item.total_cost_usd.toFixed(4) + ' (budget: \$' + budget + ')'); process.exit(1); } } console.log('All models within budget (\$' + budget + ')'); " - name: Check deprecated if: inputs.fail-on-deprecated == 'true' shell: bash run: | node -e " const fs = require('fs'); const data = JSON.parse(fs.readFileSync(process.env.RUNNER_TEMP + '/llm-models-result.json', 'utf-8')); const items = Array.isArray(data) ? data : [data]; const deprecated = items.filter(m => m.status === 'deprecated'); if (deprecated.length > 0) { console.error('Deprecated models found: ' + deprecated.map(m => m.id).join(', ')); process.exit(1); } console.log('No deprecated models found'); "