name: Validate skills (PII + structure) # Server-side enforcement of validate_skills.sh. # This catches commits that bypassed the local pre-commit hook # (--no-verify, different machine, different user) before they reach main. on: push: branches: [main] pull_request: branches: [main] jobs: validate: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: # check_release_cadence.py compares CITATION.cff against the newest TAG. A shallow # checkout has no tags, and the gate would pass by seeing nothing — the worst kind of # green. fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.11" - name: Install Python test dependencies run: pip install pyyaml pandas numpy python-pptx python-docx - name: Install exiftool + poppler (EXIF scan + check_asset_anonymization PDF text/metadata) run: sudo apt-get update && sudo apt-get install -y libimage-exiftool-perl poppler-utils - name: "Workflow files parse (a broken one runs ZERO jobs and reports no checks)" run: python3 scripts/check_workflow_yaml.py --strict - name: "Workflow-yaml gate self-test (a runless step and an unquoted ': ' must both fail)" run: bash tests/test_workflow_yaml.sh - name: Run run-ci-mirror self-test (the local mirror must enumerate the real gates) run: bash tests/test_run_ci_mirror.sh - name: Run validate_skills.sh (PII + structure) run: bash scripts/validate_skills.sh # --- self-update foundation (PR-1a): manifests, version drift, transactional installer --- - name: Distribution manifests in sync (gen_distribution_manifest.py --check) run: python3 scripts/gen_distribution_manifest.py --check - name: Version consistency (CITATION == package.json == distribution_manifest) run: python3 scripts/check_version_consistency.py - name: Distribution manifest + scope-pinning test run: python3 installers/tests/test_distribution_manifest.py - name: Transactional installer crash-recovery + legacy-migration test run: python3 installers/tests/test_txn.py - name: Updater verify / safe-extract / check-update test (offline) run: python3 installers/tests/test_update.py - name: install.py transactional self-test (no host/state dir touched) run: python3 installers/install.py --self-test - name: Opt-in SessionStart update-notify hook test (settings merge + hook logic, offline) run: python3 installers/tests/test_session_hook.py - name: Release-ZIP provenance + updater-consumability round-trip (build -> safe-extract) run: bash installers/tests/test_release_zip.sh - name: Run validate_routing_assets.py (SKILL.md asset references must exist) run: python3 scripts/validate_routing_assets.py --strict - name: Run validate_routing_assets.py self-test (dangling references/ pointer fails) run: bash tests/test_routing_assets.sh - name: "Run check_phase_budget.py (a SKILL.md phase over 80 lines is a bill paid on every invocation)" run: python3 scripts/check_phase_budget.py --strict - name: "Run phase-budget self-test (the real 209-line phase must still fail the gate)" run: bash tests/test_phase_budget.sh - name: "Run check_named_skills_exist.py (a skill may not send you to a skill that does not exist)" run: python3 scripts/check_named_skills_exist.py --strict - name: Run check_domain_probe_sync.py (vendored domain probes must be byte-identical) run: python3 scripts/check_domain_probe_sync.py --strict - name: "Run vendoring-gate self-test (drifted checklist, drifted probe, undeclared pair all fail)" run: bash tests/test_vendoring_sync.sh - name: "Run check_script_reachability.py (a script no skill runs never runs)" run: python3 scripts/check_script_reachability.py --strict - name: "Run script-reachability self-test (de-wired script and undocumented allowlist entry fail)" run: bash tests/test_script_reachability.sh - name: "Run check_bundled_media_license.py (an MIT package may only ship what it holds — including inside a .pptx)" run: python3 scripts/check_bundled_media_license.py --strict - name: "Run check_third_party_index.py (the LICENSE must describe the package you downloaded)" run: python3 scripts/check_third_party_index.py --strict - name: "Run bundled-media + third-party-index self-test (restore each defect; the gate must FAIL)" run: bash tests/test_bundled_media_license.sh - name: Run validate_capabilities.py (skill-registry consistency, capabilities.yml vs skill.yml) run: python3 scripts/validate_capabilities.py --strict - name: Run capabilities-validator self-test (drift fixtures fail, live repo clean) run: bash tests/test_capabilities_validator.sh - name: "Run check_detector_reachability.py (a detector nothing calls never runs)" run: python3 scripts/check_detector_reachability.py --strict - name: Run check_orchestrate_reachability.py (every skill routable from /orchestrate) run: python3 scripts/check_orchestrate_reachability.py --strict - name: Run orchestrate-reachability self-test (missing-row + ghost-route fail) run: bash tests/test_orchestrate_reachability.sh - name: Run check_detector_envelopes.py (every detector's JSON names the detector that wrote it) run: python3 scripts/check_detector_envelopes.py --strict - name: Run detector-envelope gate self-test (unlabelled + mislabelled envelopes fail) run: bash tests/test_detector_envelopes.sh - name: Run check_detector_crossfire.py (a fixture clean for one detector must be clean for all) run: python3 scripts/check_detector_crossfire.py - name: Run detector-crossfire self-test (restores the real 18pt vs 20pt bug; the gate must catch it) run: bash tests/test_detector_crossfire.sh - name: Run check_python_floor.py (everything a user runs must parse on the Python we promise) run: python3 scripts/check_python_floor.py --strict - name: Run installer Python-guard test (old / missing Python must be refused, not crash) run: bash installers/tests/test_installer_python_guard.sh - name: Run setup-check test (hide pandoc; the report must NOTICE, and must install nothing) run: bash installers/tests/test_doctor.sh - name: Run manifest-drift message test (a contributor's red X must be actionable) run: bash tests/test_manifest_drift_message.sh - name: Run check_release_cadence.py (a release is an event, not a commit) run: python3 scripts/check_release_cadence.py --strict - name: Run release-cadence gate self-test run: bash tests/test_release_cadence.sh - name: Run check_locale_inventory.py (Korean-bearing files must be inventory-justified) run: python3 scripts/check_locale_inventory.py - name: Run check_locale_inventory.py self-test run: bash tests/test_locale_inventory.sh - name: "Run check_hardcoded_locale.py (no skill picks its user's language)" run: python3 scripts/check_hardcoded_locale.py --strict - name: Run hardcoded-locale gate self-test (the rule /publish-skill already wrote down) run: bash tests/test_hardcoded_locale.sh - name: Run self-review panel-mode structural + PII test run: bash skills/self-review/tests/test_panel_mode.sh - name: Run self-review reference-adequacy gate test run: bash skills/self-review/tests/test_reference_adequacy.sh - name: Run self-review editorial-impression gate test (defensiveness counterweight) run: bash skills/self-review/tests/test_editorial_impression.sh - name: Run self-review rounded-delta gate test (stated diff vs displayed components) run: bash skills/self-review/tests/test_rounded_delta.sh - name: Run self-review orphan figure/table citation gate test run: bash skills/self-review/tests/test_figure_citation.sh - name: Run self-review inline-emphasis over-use gate test run: bash skills/self-review/tests/test_emphasis_density.sh - name: Run humanize rewrite-fidelity gate test (a de-AI pass may not alter a number or drop a citation) run: bash skills/humanize/tests/test_rewrite_fidelity.sh - name: Run humanize sentence-variety gate test (Fix rule 7 is now measured, not just prescribed) run: bash skills/humanize/tests/test_sentence_variety.sh - name: Run self-review feature-selection-outside-CV leakage gate test run: bash skills/self-review/tests/test_cv_leakage.sh - name: Run validate_catalog_consistency.py (doc counts must match disk SSOT) run: python3 scripts/validate_catalog_consistency.py - name: Run check_frontmatter_schema.py (Agent Skills frontmatter spec — valid YAML, name, description) run: python3 scripts/check_frontmatter_schema.py - name: Run frontmatter-schema gate self-test run: bash tests/test_frontmatter_schema.sh - name: Run gen_skill_docs.py --check (per-skill docs must match SKILL.md) run: python3 scripts/gen_skill_docs.py --check - name: Run gen_skill_docs _ignorable regression test (worktree dotted-ancestor counts) run: bash tests/test_gen_skill_docs_ignorable.sh - name: Run gen_skills_catalog_json.py --check (storefront catalog SSOT) run: python3 scripts/gen_skills_catalog_json.py --check - name: Run skills_catalog generator self-test run: bash tests/test_skills_catalog_json.sh - name: Run gen_marketplace_json.py --check (plugin marketplace SSOT) run: python3 scripts/gen_marketplace_json.py --check - name: Run marketplace generator self-test run: bash tests/test_marketplace_json.sh - name: Run gen_detectors_catalog_json.py --check (MedSci-Audit detector SSOT) run: python3 scripts/gen_detectors_catalog_json.py --check - name: Run detectors_catalog generator self-test run: bash tests/test_detectors_catalog_json.sh - name: Run sync_hero_skill.py generator self-test (standalone mirror build) run: bash tests/test_sync_hero_skill.sh - name: Run peer-review self-improvement-claim gate test (self-confirming / ungrounded loop) run: bash skills/peer-review/tests/test_self_improvement_claims.sh - name: "Run peer-review PDF-scan XMP regression (an int xref must not crash the injection scan)" run: bash skills/peer-review/tests/test_scan_pdf_layers_xmp.sh - name: "Run peer-review draft gates (length per comment; recommendation must stay in the editor box)" run: bash skills/peer-review/tests/test_review_draft_gates.sh - name: Run contribute safety gate test (PHI/identifier scan before anything leaves a laptop) run: bash skills/contribute/tests/test_contribution_safety.sh - name: Run analyze-stats separation gate test (complete / quasi-complete separation) run: bash skills/analyze-stats/tests/test_separation.sh - name: Run analyze-stats survival template test (A1) run: bash skills/analyze-stats/tests/test_survival_template.sh - name: Run sync-submission asset-anonymization gate test (A2) run: bash skills/sync-submission/tests/test_asset_anonymization.sh - name: Run sync-submission disclosure/availability detector test run: bash skills/sync-submission/tests/test_disclosure_availability.sh - name: Run academic-aio summary-box detector test run: bash skills/academic-aio/tests/test_summary_box.sh - name: Run write-paper title-page affiliation-order test run: bash skills/write-paper/tests/test_title_page_affiliations.sh - name: Run write-paper backbone full-text readiness gate test run: bash skills/write-paper/tests/test_backbone_fulltext.sh - name: Run sync-submission cross-artifact staleness gate test (A3) run: bash skills/sync-submission/tests/test_cross_artifact_stale.sh - name: Run check-reporting checklist-version gate test (A4) run: bash skills/check-reporting/tests/test_checklist_version.sh - name: Run write-paper placeholder gate test (A5) run: bash skills/write-paper/tests/test_placeholders.sh - name: Run sync-submission preflight gate test (A6) run: bash skills/sync-submission/tests/test_preflight_gate.sh - name: "Run sync-submission portal-field-residue challenge (paste-verbatim .txt with markdown -> fires; clean -> clean)" run: bash skills/sync-submission/scripts/check_portal_field_residue_challenge/verify.sh - name: Run sync-submission word-count-cap gate test (A7) run: bash skills/sync-submission/tests/test_wordcount_cap.sh - name: Run verify-refs corporate-author gate test (A8) run: bash skills/verify-refs/tests/test_corporate_author.sh - name: Run verify-refs OpenAlex tertiary-index test (A8b) run: bash skills/verify-refs/tests/test_openalex_tier.sh - name: Run verify-refs fabricated-author cross-check test run: bash skills/verify-refs/tests/test_fabricated_author.sh - name: Run verify-refs author-normalization test (Unicode dash + BBT brace-protected surname) run: bash skills/verify-refs/tests/test_author_normalization.sh - name: Run manage-refs bib-title publisher-markup gate test run: bash skills/manage-refs/tests/test_bib_title_markup.sh - name: Run revise response-claim verification gate test run: bash skills/revise/tests/test_response_claims.sh - name: "Run revise density-complaint challenge (point-by-point revision that got longer must fail)" run: bash skills/revise/scripts/density_complaint_challenge/verify.sh - name: Run sync-submission marked-manuscript round-trip gate test (move-aware) run: bash skills/sync-submission/tests/test_marked_manuscript.sh - name: Run find-cohort-gap local-codebook input adapter test run: bash skills/find-cohort-gap/tests/test_cohort_profile.sh - name: Run self-review parenthesis-span corruption gate test (A9) run: bash skills/self-review/tests/test_paren_spans.sh - name: Run self-review classical-style gate test (A10) run: bash skills/self-review/tests/test_classical_style.sh - name: Run self-review confounding-completeness gate test (A11) run: bash skills/self-review/tests/test_confounding_completeness.sh - name: Run self-review supplement-hygiene gate test run: bash skills/self-review/tests/test_supplement_hygiene.sh - name: Run self-review artifact-coverage gate test run: bash skills/self-review/tests/test_artifact_coverage.sh - name: Run self-review null-calibration gate test run: bash skills/self-review/tests/test_null_calibration.sh - name: Run sync-submission supplement-assembler test (A12) run: bash skills/sync-submission/tests/test_assemble_supplement.sh - name: Run render-pdf-doc glyph-coverage scan test (A13) run: bash skills/render-pdf-doc/tests/test_glyph_coverage.sh - name: Run author-strategy archetype-classifier test (A14) run: bash skills/author-strategy/tests/test_archetype_classifier.sh # --- Tier 0/1 skill regression tests (existing + new, now CI-wired) --- - name: Run make-figures legend-reconcile test run: bash skills/make-figures/tests/test_legend_reconcile.sh - name: Run make-figures pptx Mac-compatibility validator test run: python3 skills/make-figures/tests/test_pptx_mac_compat.py - name: Run clean-data structural-zero test run: bash skills/clean-data/tests/test_structural_zero.sh - name: Run clean-data reverse-coding test run: bash skills/clean-data/tests/test_reverse_coding.sh - name: Run lit-sync poll-logic test run: bash skills/lit-sync/tests/test_poll_logic.sh - name: Run meta-analysis pool-consistency test run: bash skills/meta-analysis/tests/test_pool_consistency.sh - name: "Run meta-analysis exclusion-code-validity challenge (code excludes an eligible design -> fires; correct exclusion -> clean)" run: bash skills/meta-analysis/scripts/check_exclusion_code_validity_challenge/verify.sh - name: Run generate-codebook test run: bash skills/generate-codebook/tests/test_generate_codebook.sh - name: Run slide-tells challenge (AI-made deck -> all 6 marks; human-made deck -> CLEAN) run: bash skills/present-paper/scripts/check_slide_tells_challenge/verify.sh - name: "Run deck-budget challenge (same deck: fits an oral, too dense for a keynote)" run: bash skills/present-paper/scripts/check_deck_budget_challenge/verify.sh - name: Run present-paper builder chrome test (the old eyebrow-everywhere default must be CAUGHT) run: python3 skills/present-paper/tests/test_builder_no_chrome.py - name: Run present-paper speaker-notes markdown test run: python3 skills/present-paper/tests/test_speaker_notes_markdown.py - name: Run version-dataset manifest/verify test run: bash skills/version-dataset/tests/test_version_dataset.sh - name: Run manage-refs vN-docx cross-reference test run: bash skills/manage-refs/tests/test_vN_docx_check.sh - name: Run manage-refs CSL-render hardening test run: bash skills/manage-refs/tests/test_csl_render.sh - name: Run polish-language consistency-linter challenge run: bash skills/polish-language/scripts/lint_challenge/verify.sh - name: Run deidentify PHI-scan contract test run: bash skills/deidentify/tests/test_deidentify_scan.sh - name: Run fill-icmje-coi form-clone contract test run: bash skills/fill-icmje-coi/tests/test_fill_icmje_coi.sh - name: Run academic-aio schema-validator test run: bash skills/academic-aio/tests/test_validate_schema.sh - name: Run academic-aio metadata-audit test run: bash skills/academic-aio/tests/test_batch_metadata_audit.sh - name: Run fill-protocol form-fill test run: bash skills/fill-protocol/tests/test_fill_form.sh - name: Run fulltext-retrieval pdf_to_md helper test run: python3 skills/fulltext-retrieval/tests/test_pdf_to_md.py - name: Run fulltext-retrieval report-builder challenge run: bash skills/fulltext-retrieval/fetch_oa_report_challenge/verify.sh - name: Run self-review scope-coherence gate test run: bash skills/self-review/tests/test_scope_coherence.sh - name: Run self-review cohort-arithmetic gate test run: bash skills/self-review/tests/test_cohort_arithmetic.sh - name: Run model-validation split-leakage challenge run: bash skills/model-validation/scripts/check_split_leakage_challenge/verify.sh - name: Run model-validation split-leakage gate test run: bash skills/model-validation/tests/test_split_leakage.sh - name: Run preprocess-imaging leakage challenge run: bash skills/preprocess-imaging/scripts/check_preprocessing_leakage_challenge/verify.sh - name: Run preprocess-imaging leakage gate test run: bash skills/preprocess-imaging/tests/test_preprocessing_leakage.sh - name: Run explainability report challenge run: bash skills/explainability/scripts/check_explainability_report_challenge/verify.sh - name: Run explainability report gate test run: bash skills/explainability/tests/test_explainability_report.sh - name: Run radiomics-ml pipeline challenge run: bash skills/radiomics-ml/scripts/check_radiomics_ml_challenge/verify.sh - name: Run radiomics-ml pipeline gate test run: bash skills/radiomics-ml/tests/test_radiomics_ml.sh - name: Run model-scaffold build→validate challenge run: bash skills/model-scaffold/scripts/scaffold_challenge/verify.sh - name: Run model-scaffold training-hygiene gate test run: bash skills/model-scaffold/tests/test_training_hygiene.sh - name: Run uncertainty-imaging reporting challenge run: bash skills/uncertainty-imaging/scripts/check_uncertainty_reporting_challenge/verify.sh - name: Run uncertainty-imaging reporting gate test run: bash skills/uncertainty-imaging/tests/test_uncertainty_reporting.sh - name: Run model-card completeness challenge run: bash skills/model-card/scripts/check_model_card_complete_challenge/verify.sh - name: Run model-card completeness gate test run: bash skills/model-card/tests/test_model_card_complete.sh - name: Run model-evaluation metric-reporting challenge run: bash skills/model-evaluation/scripts/metric_reporting_challenge/verify.sh - name: Run model-evaluation metric-reporting gate test run: bash skills/model-evaluation/tests/test_metric_reporting.sh - name: Run mllm-eval completeness challenge run: bash skills/mllm-eval/scripts/mllm_eval_completeness_challenge/verify.sh - name: Run mllm-eval completeness gate test run: bash skills/mllm-eval/tests/test_mllm_eval_completeness.sh - name: Run peer-review pdf-injection challenge (hidden-text guard, PyMuPDF-free) run: bash skills/peer-review/scripts/check_pdf_injection_challenge/verify.sh - name: Run self-review table-percentage challenge (n (%) recompute vs denominator) run: bash skills/self-review/scripts/check_table_percentages_challenge/verify.sh - name: Run self-review nested-group-comparison challenge (subset vs parent cohort) run: bash skills/self-review/scripts/check_nested_group_comparison_challenge/verify.sh - name: Run self-review reported-P-from-counts challenge (recompute row P, stdlib) run: bash skills/self-review/scripts/check_reported_p_from_counts_challenge/verify.sh - name: Run self-review DTA-denominator challenge (sens/spec vs reference-standard) run: bash skills/self-review/scripts/check_dta_denominators_challenge/verify.sh - name: "Run self-review figure-citation panel-suffix challenge (Figure 3a cites Figure 3 -> no orphan; uncited -> orphan)" run: bash skills/self-review/scripts/check_figure_citation_challenge/verify.sh - name: Run self-review paired-difference-estimator challenge (median parity / degenerate CI) run: bash skills/self-review/scripts/check_paired_difference_estimator_challenge/verify.sh - name: Run self-review effect-stability challenge (76-fold CI + low EPV vs tight CI + caveat) run: bash skills/self-review/scripts/check_effect_stability_challenge/verify.sh - name: Run self-review incorporation-bias challenge (trajectory standard + trajectory predictor) run: bash skills/self-review/scripts/check_incorporation_bias_challenge/verify.sh - name: Run analyze-stats generated-code gate test run: bash skills/analyze-stats/tests/test_generated_code.sh # --- previously-dormant skill regression tests, now CI-wired (v5 coverage-gap closure) --- # These tests shipped with their detectors but were never added to this workflow, so CI # gave false coverage. All pass on the same toolchain CI installs (stdlib + python-docx; # no pandoc/R). Re-run gen_distribution_manifest.py after adding any new fixture. - name: Run check-reporting checklist fail-fast test run: bash skills/check-reporting/tests/test_checklist_fail_fast.sh - name: Run check-reporting framework-naming gate test run: bash skills/check-reporting/tests/test_framework_naming.sh - name: "Run check-reporting checklist-fidelity gate test (a checklist must match the official inventory, issue #352)" run: bash skills/check-reporting/tests/test_checklist_fidelity.sh - name: Run check-reporting PRISMA-cascade test run: bash skills/check-reporting/tests/test_prisma_cascade.sh - name: Run check-reporting PRISMA Figure 1 audit test (Step 4d) run: bash skills/check-reporting/tests/test_prisma_figure.sh - name: Run manage-refs duplicate-bibliography gate test run: bash skills/manage-refs/tests/test_reference_duplication.sh - name: Run self-review binning-consistency gate test run: bash skills/self-review/tests/test_binning_consistency.sh - name: Run self-review float citation-order gate test run: bash skills/self-review/tests/test_citation_order.sh - name: Run self-review claim-artifact (estimand provenance) gate test run: bash skills/self-review/tests/test_claim_artifact.sh - name: Run self-review panel-diversity gate test run: bash skills/self-review/tests/test_panel_diversity.sh - name: Run self-review reviewer-team-consistency gate test run: bash skills/self-review/tests/test_reviewer_team_consistency.sh - name: Run sync-submission audit-dump leak gate test run: bash skills/sync-submission/tests/test_checklist_dump_leak.sh - name: Run sync-submission copy-divergence gate test run: bash skills/sync-submission/tests/test_copy_divergence.sh - name: Run sync-submission cross-document-N gate test run: bash skills/sync-submission/tests/test_cross_document_n.sh - name: Run sync-submission scope-drift gate test run: bash skills/sync-submission/tests/test_scope_drift.sh - name: Run sync-submission cover-letter title-drift gate test run: bash skills/sync-submission/tests/test_cover_letter_title_drift.sh - name: Run sync-submission vN-docx assertion gate test run: bash skills/sync-submission/tests/test_vN_docx_assertion.sh - name: Run verify-refs pagination-placeholder gate test run: bash skills/verify-refs/tests/test_pagination_placeholder.sh - name: Verify demo manifest.lock files (reproducibility lock) run: | for d in demo/01_wisconsin_bc demo/02_metafor_bcg demo/03_nhanes_obesity; do python3 skills/version-dataset/scripts/version_dataset.py verify \ --manifest "$d/manifest.lock.json" --base "$d" --strict done # --- npm/npx distribution baseline (runs last so packaging failures are easy to isolate) --- - name: Set up Node uses: actions/setup-node@v4 with: node-version: "20" - name: npm package self-test (CLI shim + exec bit + version sync + pack audit) run: bash tests/test_npm_package.sh - name: npm pack content audit (real pack, package/ prefix normalized) run: python3 scripts/check_npm_package_contents.py --real # Cross-platform safety for the transactional installer + updater (PR-1a/PR-1b). Ubuntu-only CI # cannot assert macOS/Windows path/journal/os.replace/extraction behavior; this matrix runs the # crash-recovery + legacy-migration tests, the updater verify/safe-extract tests, and the # transactional self-test on macOS + Windows (Ubuntu is covered by the `validate` job). Hash-based # manifest --check stays Ubuntu-only — it is sensitive to checkout line endings, not OS behavior. foundation-os: strategy: fail-fast: false matrix: os: [macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.11" - name: Transactional installer crash-recovery + legacy-migration test run: python installers/tests/test_txn.py - name: Updater verify / safe-extract / check-update test (offline) run: python installers/tests/test_update.py - name: Opt-in SessionStart update-notify hook test (settings merge + hook logic) run: python installers/tests/test_session_hook.py - name: install.py transactional self-test run: python installers/install.py --self-test