# Default pipeline stage definitions. # # The executor reads this file (or a project-specific override at # projects/{name}/pipeline.yaml) and constructs a topologically-sorted # execution plan from the stage list. # # Each stage has: # key — stable machine-facing identifier used by single-stage dispatch # name — human-readable label shown in progress logs # script — canonical repository-relative script path (or null for built-ins) # method — executor method name (for stages that don't use subprocess scripts) # args — extra CLI arguments passed to the script # allow_skip — treat exit-code 2 as success (graceful skip) # depends_on — list of stage names this stage must run after # tags — list of tags for conditional inclusion (e.g. "llm", "core") # failure_mode — documentation-only label describing how this stage fails # (rendered into generated stage tables; not interpreted by # the executor). Examples: "hard fail", "soft fail", # "warning + report", "skipped if Ollama absent". # contract — declarative stage contract: inputs, outputs, done criteria, # failure code, retry count, optional HITL gate, rollback target # hooks — optional explicit argv hooks: pre_stage, post_stage, on_fail, # on_pause; default timeout is 30s and CI execution is opt-in stages: - name: Clean Output Directories key: clean method: _run_clean_outputs tags: [core, clean] failure_mode: soft fail contract: input_artifacts: ["projects/{project}/output/"] output_artifacts: ["projects/{project}/output/"] definition_of_done: "Previous generated outputs are removed or the clean step is skipped." failure_code: "CLEAN_OUTPUT_FAILED" retry_policy: 0 rollback_to: "start" - name: Environment Setup key: setup script: scripts/pipeline/stage_00_setup.py depends_on: [Clean Output Directories] tags: [core] failure_mode: hard fail contract: input_artifacts: ["pyproject.toml", "projects/{project}/"] output_artifacts: ["projects/{project}/output/"] definition_of_done: "Interpreter, project directories, and required runtime paths are ready." failure_code: "ENVIRONMENT_SETUP_FAILED" retry_policy: 1 rollback_to: "Clean Output Directories" - name: Infrastructure Tests key: infra_tests script: scripts/pipeline/stage_01_test.py args: [--infra-only, --verbose, --infra-scope, pipeline-smoke] depends_on: [Environment Setup] tags: [core, tests] failure_mode: configurable tolerance contract: input_artifacts: ["infrastructure/", "tests/infra_tests/"] output_artifacts: ["projects/{project}/output/"] definition_of_done: "Focused pipeline-smoke infrastructure tests pass or report configured tolerance; project fixtures may refresh generated output artifacts; full repo coverage remains an explicit verification command." failure_code: "INFRASTRUCTURE_TESTS_FAILED" retry_policy: 0 rollback_to: "Environment Setup" - name: Project Tests key: project_tests script: scripts/pipeline/stage_01_test.py args: [--project-only, --verbose] depends_on: [Environment Setup] tags: [core, tests] failure_mode: configurable tolerance contract: input_artifacts: ["projects/{project}/src/", "projects/{project}/tests/"] output_artifacts: ["projects/{project}/output/"] definition_of_done: "Project tests pass against real project behavior and may refresh generated project artifacts exercised by the suite." failure_code: "PROJECT_TESTS_FAILED" retry_policy: 0 gate: "experiment_method_design" rollback_to: "Environment Setup" - name: Project Analysis key: analysis script: scripts/pipeline/stage_02_analysis.py depends_on: [Project Tests] tags: [core] failure_mode: hard fail contract: input_artifacts: ["projects/{project}/src/", "projects/{project}/scripts/"] output_artifacts: ["projects/{project}/output/"] definition_of_done: "Project analysis scripts produce declared data and figure artifacts." failure_code: "PROJECT_ANALYSIS_FAILED" retry_policy: 0 gate: "experiment_method_design" rollback_to: "Project Tests" # Opt-in science/provenance stages (skipped in --core-only and default runs). # Invoke directly when needed: # uv run python scripts/pipeline/stage_08_connector_search.py --project # uv run python scripts/pipeline/stage_09_provenance_record.py --project --stage "Connector Search" - name: Connector Search key: connector_search script: scripts/pipeline/stage_08_connector_search.py depends_on: [Project Analysis] tags: [science] allow_skip: true failure_mode: skipped if not configured contract: input_artifacts: ["projects/{project}/manuscript/config.yaml"] output_artifacts: ["projects/{project}/output/data/connector_search/"] definition_of_done: "Connector search completes or gracefully exits 2 when not configured; each configured connector returns results or an error." failure_code: "CONNECTOR_SEARCH_FAILED" retry_policy: 1 rollback_to: "Project Analysis" - name: Provenance Record key: provenance_record script: scripts/pipeline/stage_09_provenance_record.py args: [--stage, Connector Search] depends_on: [Connector Search] tags: [provenance] allow_skip: true failure_mode: skipped if not configured contract: input_artifacts: ["projects/{project}/output/"] output_artifacts: [".provenance/graph.json"] definition_of_done: "Pipeline stage artifacts are recorded as provenance nodes, or the stage is gracefully skipped when provenance is not configured." failure_code: "PROVENANCE_RECORD_FAILED" retry_policy: 0 rollback_to: "Connector Search" - name: PDF Rendering key: render_pdf script: scripts/pipeline/stage_03_render.py depends_on: [Project Analysis] tags: [core] failure_mode: hard fail contract: input_artifacts: ["projects/{project}/manuscript/", "projects/{project}/output/data/"] output_artifacts: ["projects/{project}/output/"] definition_of_done: "Manuscript renders to a structurally valid PDF artifact." failure_code: "PDF_RENDERING_FAILED" retry_policy: 0 gate: "render_validation" rollback_to: "Project Analysis" - name: Output Validation key: validate script: scripts/pipeline/stage_04_validate.py depends_on: [PDF Rendering] tags: [core] failure_mode: PDF/bookends and artifact/provenance failures block; optional-format structure remains a warning + report contract: input_artifacts: ["projects/{project}/output/", "projects/{project}/manuscript/"] output_artifacts: ["projects/{project}/output/reports/"] definition_of_done: "Output, figure, markdown, and evidence checks produce a validation report." failure_code: "OUTPUT_VALIDATION_FAILED" retry_policy: 0 gate: "literature_source_quality" rollback_to: "PDF Rendering" - name: LLM Scientific Review key: llm_reviews script: scripts/pipeline/stage_06_llm_review.py args: [--reviews-only] allow_skip: true depends_on: [Output Validation] tags: [llm] failure_mode: skipped if Ollama absent contract: input_artifacts: ["projects/{project}/manuscript/", "projects/{project}/output/pdf/"] output_artifacts: ["projects/{project}/output/llm/", "projects/{project}/output/translations/"] definition_of_done: "Optional scientific review is written or gracefully skipped when unavailable." failure_code: "LLM_REVIEW_FAILED" retry_policy: 0 rollback_to: "Output Validation" - name: LLM Translations key: llm_translations script: scripts/pipeline/stage_06_llm_review.py args: [--translations-only] allow_skip: true depends_on: [Output Validation] tags: [llm] failure_mode: skipped if Ollama absent contract: input_artifacts: ["projects/{project}/manuscript/"] output_artifacts: ["projects/{project}/output/llm/", "projects/{project}/output/translations/"] definition_of_done: "Optional translations are written or gracefully skipped when unavailable." failure_code: "LLM_TRANSLATION_FAILED" retry_policy: 0 rollback_to: "Output Validation" - name: Copy Outputs key: copy script: scripts/pipeline/stage_05_copy.py depends_on: [Output Validation] tags: [core] failure_mode: soft fail contract: input_artifacts: ["projects/{project}/output/"] output_artifacts: ["projects/{project}/output/reports/", "output/{project}/"] definition_of_done: "Final deliverables are copied into the canonical top-level output tree." failure_code: "COPY_OUTPUTS_FAILED" retry_policy: 0 gate: "publication_readiness" rollback_to: "Output Validation" # ── Opt-in long-horizon stage definitions ───────────────────────────── # These stages are declared here so contracts stay versioned with the DAG, # but PipelineExecutor filters tags `bundle`, `archival`, `ebook`, and # `metadata` out of default runs. # See docs/maintenance/stage-10-executable-bundle.md and # docs/maintenance/archival-targets.md for rationale. # # Invoke directly when needed: # uv run python scripts/pipeline/stage_11_ebook.py --project # uv run python scripts/pipeline/stage_12_metadata.py --project # uv run python scripts/runner/bundle_executable.py --project # uv run python scripts/runner/archive_publication.py --project - name: Ebook Generation key: ebook_generation script: scripts/pipeline/stage_11_ebook.py allow_skip: true depends_on: [PDF Rendering] tags: [core, ebook] failure_mode: soft fail contract: input_artifacts: ["projects/{project}/output/"] output_artifacts: ["projects/{project}/output/ebook/"] definition_of_done: "EPUB, MOBI, and DOCX ebook files are generated or the stage is gracefully skipped when the combined markdown is absent." failure_code: "EBOOK_GENERATION_FAILED" retry_policy: 0 rollback_to: "PDF Rendering" - name: Metadata Package key: metadata_package script: scripts/pipeline/stage_12_metadata.py allow_skip: true depends_on: [Copy Outputs] tags: [core, metadata] failure_mode: soft fail contract: input_artifacts: ["projects/{project}/manuscript/config.yaml"] output_artifacts: ["projects/{project}/output/metadata/"] definition_of_done: "ONIX XML, metadata.json, and OPF skeleton are written to output/metadata/ or the stage is gracefully skipped when config.yaml is absent." failure_code: "METADATA_PACKAGE_FAILED" retry_policy: 0 rollback_to: "Copy Outputs" - name: Executable Bundle key: executable_bundle script: scripts/runner/bundle_executable.py allow_skip: true depends_on: [PDF Rendering] tags: [bundle] failure_mode: soft fail contract: input_artifacts: ["projects/{project}/", "projects/{project}/output/"] output_artifacts: ["projects/{project}/output/bundles/"] definition_of_done: "Executable reproducibility bundle is written or reports why it was skipped." failure_code: "EXECUTABLE_BUNDLE_FAILED" retry_policy: 0 rollback_to: "PDF Rendering" - name: Archival Publication key: archival_publication script: scripts/runner/archive_publication.py allow_skip: true depends_on: [Executable Bundle] tags: [archival] failure_mode: soft fail contract: input_artifacts: ["projects/{project}/output/", "output/{project}/"] output_artifacts: ["projects/{project}/output/archive/"] definition_of_done: "Archival publication manifest is generated or reports why it was skipped." failure_code: "ARCHIVAL_PUBLICATION_FAILED" retry_policy: 0 gate: "publication_readiness" rollback_to: "Executable Bundle" control: hitl_mode: full-auto smart_pause_action: report custom_gate_stages: [] stage_policies: {} # ── Telemetry Configuration ────────────────────────────────────────── # Unified pipeline telemetry: resource tracking + diagnostic aggregation. # Override per-project by adding a telemetry: block in projects/{name}/pipeline.yaml. telemetry: enabled: true track_resources: true track_diagnostics: true output_formats: [json, text] persist_report: true slow_stage_multiplier: 2.0 high_memory_mb: 1024 high_cpu_percent: 90.0