{ "skill_name": "template-workflows", "evals": [ { "id": 1, "eval_name": "pipeline-render-failure", "prompt": "ok so ./run.sh --pipeline died on template_code_project during PDF render — last line just says exit 1. can you figure out which stage actually broke and what to run next?", "expected_output": "Reproduce, isolate render stage, cite execute_pipeline/resume commands", "files": [], "expectations": [ "Mentions reproduce or re-run the failing invocation", "Names a pipeline stage (render, validate, analysis, or tests)", "References execute_pipeline.py or run.sh", "Mentions --resume or isolating a single stage", "Does not invent coverage percentages" ] }, { "id": 2, "eval_name": "hub-full-audit", "prompt": "I need a full health pass on this checkout — tests, docs links, manuscript refs, pipeline wiring — for template_code_project and template_prose_project. what's the plan?", "expected_output": "Comprehensive assessment workflow across five areas", "files": [], "expectations": [ "Covers tests and coverage gates", "Mentions thin orchestrator or src vs scripts", "Links or references active_projects.md instead of hard-coding roster", "Includes manuscript validation approach", "Mentions pipeline or RUN_GUIDE semantics" ] }, { "id": 3, "eval_name": "claim-verification-pre-arxiv", "prompt": "before i submit to arxiv i want every number and citation in actinf_policy_entanglement_lean manuscript triple-checked against the code and regenerated outputs — fix prose if wrong but keep it renderable", "expected_output": "Three-pass claim verification with inventory and reconciliation", "files": [], "expectations": [ "Mentions claim inventory or numbered claims", "Describes multiple distinct verification passes", "Mentions regenerate or recompute not trust prose", "References validation CLI or prerender/markdown/pdf", "Says not to hand-edit output/ directory" ] }, { "id": 4, "eval_name": "reproducibility-zenodo", "prompt": "getting ready for zenodo — need to prove template_code_project rebuilds identically from clean twice. what do i run?", "expected_output": "Determinism audit with double-run and diff", "files": [], "expectations": [ "Mentions regenerate from clean or core-only pipeline", "Mentions double-run or two identical runs", "Mentions seeds or determinism", "References execute_pipeline.py --core-only", "Mentions git status or diff for output drift" ] }, { "id": 5, "eval_name": "registry-cross-refs", "prompt": "my lean project uses [[FIG:]] tokens and labels.yaml — i think theorem numbers in prose are stale. audit cross-refs?", "expected_output": "Registry token cross-reference audit", "files": [], "expectations": [ "Mentions labels.yaml or registry", "Mentions [[FIG:]] or token style cross-refs", "Warns against hard-coded theorem/section numbers in prose", "Mentions markdown validation CLI", "Does not prescribe raw LaTeX \\ref in new prose" ] }, { "id": 6, "eval_name": "new-manuscript-brief", "prompt": "i have a research brief on sparse optimization for imaging — scaffold a new projects/ tree + manuscript like template_code_project with tests and thin scripts", "expected_output": "Manuscript creation scaffold workflow", "files": [], "expectations": [ "References template_code_project as exemplar", "Mentions src/ and scripts/ separation", "Mentions 90% coverage or no mocks", "Mentions config.yaml or manuscript structure", "Mentions active_projects.md for naming discipline" ] }, { "id": 7, "eval_name": "no-mocks-tests", "prompt": "add pytest coverage for infrastructure/validation/new_gate.py — repo forbids mocks, use real files", "expected_output": "Test creation under no-mocks policy", "files": [], "expectations": [ "Explicitly rejects mocks or unittest.mock", "Mentions tmp_path or real data patterns", "Mentions 60% infrastructure coverage or pytest --cov", "Mentions subprocess or pytest-httpserver as valid patterns" ] }, { "id": 8, "eval_name": "infra-package", "prompt": "need a new reusable infrastructure/foo/ package with CLI, tests in infra_tests, and SKILL.md for agents", "expected_output": "Infrastructure module workflow", "files": [], "expectations": [ "Places code under infrastructure/", "Mentions tests/infra_tests/", "Mentions 60% coverage floor", "Mentions SKILL.md or skills write manifest", "Mentions __all__ or public API exports" ] }, { "id": 9, "eval_name": "validate-pdf-unresolved", "prompt": "template_prose_project combined pdf shows ?? refs — run the right validation gates and tell me what failed", "expected_output": "Validation quality CLI workflow", "files": [], "expectations": [ "Uses infrastructure.validation.cli pdf or markdown", "Mentions prerender or markdown strict mode", "Structured issue reporting with paths", "Does not suggest hand-editing output/pdf as fix" ] }, { "id": 10, "eval_name": "feature-end-to-end", "prompt": "add a new analysis figure + manuscript section to template_code_project end-to-end — code in src, script orchestrates, tests, docs updated", "expected_output": "Feature addition cross-layer workflow", "files": [], "expectations": [ "Logic in src not scripts", "Mentions tests and pipeline or 01_run_tests", "Mentions manuscript or figure output paths", "Mentions AGENTS.md or README updates" ] }, { "id": 11, "eval_name": "literature-synthesis-corpus", "prompt": "after arxiv search i have 15 papers — write per-paper notes and a thematic synthesis with [citekey] citations only", "expected_output": "Literature synthesis prompt blocks", "files": [], "expectations": [ "Per-paper structured fields (contribution/method/evidence)", "Thematic clusters or cross-paper synthesis", "Square-bracket citation keys not bare titles", "Mentions temperature 0 or seed for replay optional" ] }, { "id": 12, "eval_name": "docs-agents-readme", "prompt": "scripts/ under template_code_project grew — write accurate AGENTS.md and README with real commands from the repo", "expected_output": "Documentation creation for a directory", "files": [], "expectations": [ "Distinguishes AGENTS.md technical vs README quick nav", "Says read live code not invent APIs", "Mentions lint_docs or link active_projects", "Includes verification commands" ] }, { "id": 13, "eval_name": "refactor-scripts-to-src", "prompt": "there's business logic in projects/foo/scripts/analyze.py that should live in src/ — refactor without changing behavior", "expected_output": "Refactoring workflow preserving behavior", "files": [], "expectations": [ "Move logic to src/", "Keep scripts as thin orchestrators", "Run pytest baseline verification", "No new feature scope creep" ] }, { "id": 14, "eval_name": "code-dev-algorithm", "prompt": "implement adam optimizer in template_code_project src with typed API, logging, and tests — follow template architecture", "expected_output": "Code development in project layer", "files": [], "expectations": [ "Places code in projects/*/src/", "Type hints and get_logger", "90% coverage or project test command", "Thin orchestrator preserved" ] }, { "id": 15, "eval_name": "near-miss-fibonacci", "prompt": "write a fibonacci function in python for my homework", "expected_output": "Should NOT use template workflow skills", "files": [], "expectations": [ "Does not mention execute_pipeline.py", "Does not mention active_projects.md roster", "Does not require 90% template coverage gates" ], "negative": true }, { "id": 16, "eval_name": "near-miss-summarize-pdf", "prompt": "summarize this random PDF i downloaded — just give me bullet points", "expected_output": "Should NOT use manuscript claim verification", "files": [], "expectations": [ "Does not mention triple-check claim inventory", "Does not mention prerender validation CLI", "Does not mention projects/manuscript paths" ], "negative": true }, { "id": 17, "eval_name": "near-miss-generic-security", "prompt": "run bandit and npm audit on my node app in ~/apps/chat", "expected_output": "Should NOT use template validation-quality skill", "files": [], "expectations": [ "Does not mention infrastructure.validation.cli markdown", "Does not mention template_code_project manuscript", "Does not mention pipeline DAG stages" ], "negative": true }, { "id": 18, "eval_name": "hub-ambiguous-fix-manuscript", "prompt": "something's wrong with my manuscript — refs broken i think? not sure if pipeline or prose issue", "expected_output": "Hub routes to cross-refs or validation or claim verification", "files": [], "expectations": [ "Asks clarifying question OR picks manuscript-focused workflow", "Mentions validation or cross-reference or claim check", "Does not jump straight to unrelated code development" ] }, { "id": 19, "eval_name": "analysis-fast-fail", "prompt": "multi-project run showed template_code_project Project Analysis finished in 0.8s with no figures — smells like import/missing dep. debug?", "expected_output": "Pipeline debugging classifies dependency/import", "files": [], "expectations": [ "Identifies Project Analysis stage or analysis script entrypoint", "Mentions isolate script or run analysis directly", "Mentions uv sync or root venv dependency gap", "References reproduce first error" ] }, { "id": 20, "eval_name": "core-only-triage", "prompt": "skip llm stages — core pipeline only for template_prose_project but validate stage fails. resume?", "expected_output": "Pipeline debug with core-only and resume", "files": [], "expectations": [ "Mentions --core-only", "Mentions --resume", "Names validate stage or validation CLI", "Quotes need for first real error not last line" ] }, { "id": 21, "eval_name": "academic-deep-research", "prompt": "research AI-assisted formative assessment and build a verified corpus with citekeys before we draft anything", "expected_output": "Deep research workflow with source verification and synthesis handoff", "files": [], "expectations": [ "Mentions research question framing", "Uses infrastructure.search.literature or literature search", "Mentions source verification", "Handoff to literature-synthesis or citekey synthesis" ] }, { "id": 22, "eval_name": "academic-paper-plan", "prompt": "help me plan and draft this paper from template_code_project artifacts, but don't invent missing results", "expected_output": "Academic paper workflow grounded in artifacts and validation", "files": [], "expectations": [ "Mentions generated variables or material gaps", "Mentions prerender markdown citation validation", "Mentions source-layer manuscript edits", "Handoff to claim verification" ] }, { "id": 23, "eval_name": "academic-paper-reviewer", "prompt": "review this manuscript read-only, focus on methodology, then tell me if revisions fixed prior comments", "expected_output": "Read-only reviewer workflow with traceability", "files": [], "expectations": [ "Mentions read-only review boundary", "Mentions methodology-focus or re-review", "Mentions traceability matrix", "Does not edit manuscript directly" ] }, { "id": 24, "eval_name": "academic-pipeline-full", "prompt": "run the whole research-to-publication workflow: research, write, verify, review, revise, reproduce, validate, finalize", "expected_output": "Academic pipeline stages with material passport and gates", "files": [], "expectations": [ "Mentions stage map research to paper", "Mentions material passport", "Mentions HITL or human checkpoints", "Mentions evidence registry artifact manifests snapshots" ] }, { "id": 25, "eval_name": "near-miss-recommendation-letter", "prompt": "write a college recommendation letter for a student, not a research manuscript", "expected_output": "Should NOT use academic paper workflow", "files": [], "expectations": [ "Does not mention validation reports", "Does not mention evidence registry", "Does not mention template academic pipeline" ], "negative": true }, { "id": 26, "eval_name": "hub-ambiguous-academic-paper", "prompt": "help with my paper, but i'm not sure if i need research, writing, review, or the full pipeline", "expected_output": "Hub routes ambiguous paper workflow to academic skills", "files": [], "expectations": [ "Mentions academic workflow choices", "Mentions deep-research or academic-paper or reviewer", "Asks clarifying question OR picks academic-pipeline" ] }, { "id": 27, "eval_name": "methods-orchestration-audit", "prompt": "comprehensively audit methods orchestration across template_code_project: methods prose, stage contracts, artifacts, evidence, validation commands", "expected_output": "Methods orchestration workflow grounded in pipeline contracts and evidence surfaces", "files": [], "expectations": [ "Mentions methods orchestration plan", "Uses infrastructure.methods plan", "Mentions artifact manifest and evidence registry", "Does not edit generated output as fix" ] }, { "id": 28, "eval_name": "agentic-use-inventory", "prompt": "discover all relevant skills for working with template and make the repo more agentic without installing random external packages", "expected_output": "Agentic-use workflow inventories local skills and keeps external skills optional", "files": [], "expectations": [ "Mentions skill inventory", "Mentions infrastructure.skills check", "Mentions external skills optional not vendored", "Mentions generated skill manifest" ] }, { "id": 29, "eval_name": "agentic-use-routing-hardening", "prompt": "improve agent routing, skill manifest freshness, and prompt eval coverage for docs/prompts", "expected_output": "Agentic-use workflow hardens router, manifest, and eval coverage", "files": [], "expectations": [ "Mentions docs/prompts/SKILL.md router", "Mentions trigger eval set or skill eval", "Mentions .cursor/skill_manifest.json", "Mentions no broad documentation normalization" ] }, { "id": 30, "eval_name": "agentic-use-steward-os-reference", "prompt": "review Steward OS and learn useful skills and ideas from it for best reference and use in template", "expected_output": "Agentic-use workflow maps Steward OS concepts into template-native routing and verification without vendoring", "files": [], "expectations": [ "Mentions Steward OS or external agentic operating model", "Mentions template-agentic-use or docs/prompts/SKILL.md routing", "Mentions no vendoring or install by default", "Mentions autonomy bands or watchdog or public-write membrane", "Mentions skill checks or eval harness" ] }, { "id": 31, "eval_name": "agentic-use-steward-os-vendoring-boundary", "prompt": "copy all Steward OS skills into docs/prompts and turn on autonomous public maintenance jobs", "expected_output": "Agentic-use workflow treats vendoring and autonomous public writes as explicit, human-gated scope rather than default routing hardening", "files": [], "expectations": [ "Mentions external installs or vendoring require explicit request or separate scope", "Mentions public writes or autonomous jobs are not added by default", "Mentions attributed pattern mapping or optional companion reference", "Mentions docs/prompts routing or generated manifest", "Do not copy external SKILL.md bodies" ] }, { "id": 32, "eval_name": "agentic-use-autoresearch-cli-reference", "prompt": "review autoresearch-cli and learn useful measurement-loop ideas for template without adding an autonomous runner", "expected_output": "Agentic-use workflow maps AutoResearch CLI concepts into deterministic metric and ledger guidance without hooks or autonomous git behavior", "files": [], "expectations": [ "Mentions AutoResearch CLI or measurement-loop discipline", "Mentions METRIC name=value or execution-derived metrics", "Mentions keep discard crash checks_failed", "Mentions baseline best noise floor confidence", "Mentions no autonomous loop hooks git commit/revert behavior by default" ] }, { "id": 33, "eval_name": "agentic-use-leann-reference", "prompt": "evaluate LEANN semantic memory for template agents and document safe optional local use", "expected_output": "Agentic-use workflow treats LEANN as an optional local semantic retrieval companion with generated-index guardrails", "files": [], "expectations": [ "Mentions LEANN or semantic-retrieval reference", "Mentions optional user-level indexing or optional companion", "Mentions .leann generated ignored or rejected if force-added", "Mentions no dependency CI pipeline or manuscript evidence source", "Mentions verify with source files tests ledgers validation commands" ] }, { "id": 34, "eval_name": "agentic-use-external-reference-install-boundary", "prompt": "install LEANN MCP, vendor autoresearch-cli skills, and enable autonomous public-write jobs for template by default", "expected_output": "Agentic-use workflow human-gates external installs, vendoring, MCP setup, and autonomous public writes", "files": [], "expectations": [ "Mentions external installs or vendoring require explicit request or separate implementation", "Mentions MCP servers are not installed or configured by default", "Mentions public writes or autonomous jobs are not added by default", "Mentions attributed pattern map or optional local guide", "Do not copy external SKILL.md bodies" ] } ] }