#!/usr/bin/env python3 """ Plugin Collection Validator =========================== Validates all plugins in the collection against the Claude Code plugin spec: - plugin.json manifest: required fields, author attribution, keywords - Skills: YAML frontmatter (name must match directory, description required) - Commands: YAML frontmatter (description and argument-hint required) - Cross-references: commands referencing skills that exist in the same plugin - README: exists and has expected sections Based on: - Anthropic plugin-dev README (https://github.com/anthropics/claude-code/tree/main/plugins/plugin-dev) - agentskills.io specification - Claude Code plugins reference (https://code.claude.com/docs/en/plugins-reference) Author: Paweł Huryn — The Product Compass Newsletter (https://www.productcompass.pm) """ import json import os import re import sys from pathlib import Path from typing import Optional # ─── Configuration ─────────────────────────────────────────────────────────── # Required plugin.json fields per spec REQUIRED_MANIFEST_FIELDS = ["name", "version", "description"] RECOMMENDED_MANIFEST_FIELDS = ["author", "keywords", "homepage", "license"] REQUIRED_AUTHOR_FIELDS = ["name", "email"] RECOMMENDED_AUTHOR_FIELDS = ["url"] # Required skill frontmatter fields REQUIRED_SKILL_FIELDS = ["name", "description"] # Required command frontmatter fields REQUIRED_COMMAND_FIELDS = ["description"] RECOMMENDED_COMMAND_FIELDS = ["argument-hint"] # Expected README sections (case-insensitive substring match) EXPECTED_README_SECTIONS = ["overview", "install", "skill", "command"] # ─── ANSI Colors ───────────────────────────────────────────────────────────── class C: RED = "\033[91m" GREEN = "\033[92m" YELLOW = "\033[93m" BLUE = "\033[94m" CYAN = "\033[96m" BOLD = "\033[1m" DIM = "\033[2m" RESET = "\033[0m" # ─── Helpers ───────────────────────────────────────────────────────────────── def parse_yaml_frontmatter(content: str) -> Optional[dict]: """Extract YAML frontmatter from a markdown file (between --- markers).""" if not content.startswith("---"): return None end = content.find("---", 3) if end == -1: return None fm_text = content[3:end].strip() # Simple YAML parser for flat key-value pairs result = {} for line in fm_text.split("\n"): line = line.strip() if not line or line.startswith("#"): continue match = re.match(r'^(\S+):\s*(.+)$', line) if match: key = match.group(1) value = match.group(2).strip().strip('"').strip("'") result[key] = value return result def count_words(content: str) -> int: """Count words in markdown content (excluding frontmatter).""" # Strip frontmatter if content.startswith("---"): end = content.find("---", 3) if end != -1: content = content[end + 3:] return len(content.split()) # ─── Validators ────────────────────────────────────────────────────────────── class ValidationResult: def __init__(self): self.errors: list[str] = [] # Must fix self.warnings: list[str] = [] # Should fix self.info: list[str] = [] # FYI def error(self, msg: str): self.errors.append(msg) def warn(self, msg: str): self.warnings.append(msg) def note(self, msg: str): self.info.append(msg) @property def ok(self) -> bool: return len(self.errors) == 0 def validate_manifest(plugin_dir: str) -> ValidationResult: """Validate plugin.json manifest.""" result = ValidationResult() pj_path = os.path.join(plugin_dir, ".claude-plugin", "plugin.json") if not os.path.isfile(pj_path): result.error("Missing .claude-plugin/plugin.json") return result try: with open(pj_path, "r") as f: data = json.load(f) except json.JSONDecodeError as e: result.error(f"Invalid JSON in plugin.json: {e}") return result # Required fields for field in REQUIRED_MANIFEST_FIELDS: if field not in data or not data[field]: result.error(f"Missing required field: {field}") # Name must match directory name dir_name = os.path.basename(plugin_dir) if data.get("name") and data["name"] != dir_name: result.error(f"Name mismatch: plugin.json says '{data['name']}' but directory is '{dir_name}'") # Version format version = data.get("version", "") if version and not re.match(r'^\d+\.\d+\.\d+$', version): result.warn(f"Version '{version}' doesn't follow semver (x.y.z)") # Recommended fields for field in RECOMMENDED_MANIFEST_FIELDS: if field not in data: result.warn(f"Missing recommended field: {field}") # Author validation author = data.get("author") if isinstance(author, dict): for field in REQUIRED_AUTHOR_FIELDS: if field not in author or not author[field]: result.warn(f"Missing author.{field}") for field in RECOMMENDED_AUTHOR_FIELDS: if field not in author: result.note(f"Missing author.{field}") elif author is not None: result.warn("Author should be an object with name, email, url fields") # Keywords validation keywords = data.get("keywords", []) if not keywords: result.warn("No keywords defined") elif not isinstance(keywords, list): result.error("Keywords must be an array") # Description length check desc = data.get("description", "") if desc and len(desc) < 20: result.warn(f"Description is very short ({len(desc)} chars)") result.note(f"Version: {version}") result.note(f"Keywords: {len(keywords) if isinstance(keywords, list) else 0}") return result def validate_skill(skill_dir: str) -> ValidationResult: """Validate a single skill directory.""" result = ValidationResult() skill_name = os.path.basename(skill_dir) skill_md = os.path.join(skill_dir, "SKILL.md") if not os.path.isfile(skill_md): result.error("Missing SKILL.md") return result with open(skill_md, "r", encoding="utf-8") as f: content = f.read() # Frontmatter check fm = parse_yaml_frontmatter(content) if fm is None: result.error("Missing YAML frontmatter (must start with ---)") return result # Required fields for field in REQUIRED_SKILL_FIELDS: if field not in fm or not fm[field]: result.error(f"Missing required frontmatter field: {field}") # Name must match directory name (agentskills.io spec) if fm.get("name") and fm["name"] != skill_name: result.error(f"Name mismatch: frontmatter says '{fm['name']}' but directory is '{skill_name}'") # Description quality desc = fm.get("description", "") if desc: if len(desc) < 30: result.warn(f"Description is very short ({len(desc)} chars)") # Check for trigger phrases (recommended) trigger_keywords = ["trigger", "use when", "use for"] has_triggers = any(kw in desc.lower() for kw in trigger_keywords) if not has_triggers: result.note("Description lacks explicit trigger phrases (e.g., 'Triggers: ...')") # Word count word_count = count_words(content) result.note(f"Word count: {word_count}") if word_count > 3000: result.warn(f"Skill is quite long ({word_count} words). Consider progressive disclosure with references/") elif word_count < 50: result.warn(f"Skill is very short ({word_count} words). May need more content.") return result def validate_command(cmd_path: str) -> ValidationResult: """Validate a single command file.""" result = ValidationResult() with open(cmd_path, "r", encoding="utf-8") as f: content = f.read() # Frontmatter check fm = parse_yaml_frontmatter(content) if fm is None: result.error("Missing YAML frontmatter (must start with ---)") return result # Required fields for field in REQUIRED_COMMAND_FIELDS: if field not in fm or not fm[field]: result.error(f"Missing required frontmatter field: {field}") # Recommended fields for field in RECOMMENDED_COMMAND_FIELDS: if field not in fm or not fm[field]: result.warn(f"Missing recommended frontmatter field: {field}") # Description quality desc = fm.get("description", "") if desc and len(desc) < 10: result.warn(f"Description is very short ({len(desc)} chars)") # Check if command references skills (informational) skill_refs = re.findall(r'\*\*(\w[\w-]+)\*\*\s+skill', content) if skill_refs: result.note(f"References skills: {', '.join(skill_refs)}") return result def validate_readme(plugin_dir: str) -> ValidationResult: """Validate plugin README.md.""" result = ValidationResult() readme_path = os.path.join(plugin_dir, "README.md") if not os.path.isfile(readme_path): result.warn("Missing README.md") return result with open(readme_path, "r", encoding="utf-8") as f: content = f.read().lower() # Check for expected sections for section in EXPECTED_README_SECTIONS: if section not in content: result.note(f"README may be missing '{section}' section") result.note(f"README: {count_words(content)} words") return result def validate_cross_references(plugin_dir: str, skill_names: list[str]) -> ValidationResult: """Check that commands reference skills that actually exist in this plugin.""" result = ValidationResult() cmds_dir = os.path.join(plugin_dir, "commands") if not os.path.isdir(cmds_dir): return result for cmd_file in sorted(os.listdir(cmds_dir)): if not cmd_file.endswith(".md"): continue cmd_path = os.path.join(cmds_dir, cmd_file) with open(cmd_path, "r", encoding="utf-8") as f: content = f.read() # Find skill references like **skill-name** skill refs = re.findall(r'\*\*(\w[\w-]+)\*\*\s+skill', content) for ref in refs: if ref not in skill_names: result.warn(f"Command {cmd_file} references skill '{ref}' not found in this plugin") return result # ─── Main Validator ────────────────────────────────────────────────────────── def validate_plugin(plugin_dir: str) -> dict: """Run all validations on a single plugin.""" plugin_name = os.path.basename(plugin_dir) results = {"name": plugin_name, "sections": {}} # 1. Manifest results["sections"]["manifest"] = validate_manifest(plugin_dir) # 2. Skills skills_dir = os.path.join(plugin_dir, "skills") skill_names = [] skill_results = {} if os.path.isdir(skills_dir): for skill_name in sorted(os.listdir(skills_dir)): skill_path = os.path.join(skills_dir, skill_name) if os.path.isdir(skill_path): skill_names.append(skill_name) skill_results[skill_name] = validate_skill(skill_path) results["sections"]["skills"] = skill_results results["skill_count"] = len(skill_names) # 3. Commands cmds_dir = os.path.join(plugin_dir, "commands") cmd_results = {} if os.path.isdir(cmds_dir): for cmd_file in sorted(os.listdir(cmds_dir)): if cmd_file.endswith(".md"): cmd_path = os.path.join(cmds_dir, cmd_file) cmd_results[cmd_file] = validate_command(cmd_path) results["sections"]["commands"] = cmd_results results["command_count"] = len(cmd_results) # 4. README results["sections"]["readme"] = validate_readme(plugin_dir) # 5. Cross-references results["sections"]["cross-refs"] = validate_cross_references(plugin_dir, skill_names) return results def print_validation_result(label: str, vr: ValidationResult, indent: int = 4): """Print a single validation result.""" prefix = " " * indent if vr.errors: for e in vr.errors: print(f"{prefix}{C.RED}✗ ERROR:{C.RESET} {e}") if vr.warnings: for w in vr.warnings: print(f"{prefix}{C.YELLOW}⚠ WARN:{C.RESET} {w}") if vr.info: for i in vr.info: print(f"{prefix}{C.DIM}ℹ {i}{C.RESET}") def print_report(all_results: list[dict]): """Print the full validation report.""" total_errors = 0 total_warnings = 0 total_skills = 0 total_commands = 0 print(f"\n{C.BOLD}{'='*70}") print(f" Plugin Collection Validator — Report") print(f"{'='*70}{C.RESET}\n") for plugin in all_results: name = plugin["name"] sc = plugin["skill_count"] cc = plugin["command_count"] total_skills += sc total_commands += cc # Count errors/warnings for this plugin p_errors = 0 p_warnings = 0 for key, section in plugin["sections"].items(): if isinstance(section, ValidationResult): p_errors += len(section.errors) p_warnings += len(section.warnings) elif isinstance(section, dict): for vr in section.values(): p_errors += len(vr.errors) p_warnings += len(vr.warnings) total_errors += p_errors total_warnings += p_warnings # Plugin header status = f"{C.GREEN}✓ PASS{C.RESET}" if p_errors == 0 else f"{C.RED}✗ FAIL{C.RESET}" warn_str = f" {C.YELLOW}({p_warnings} warnings){C.RESET}" if p_warnings > 0 else "" print(f"{C.BOLD}{C.CYAN}┌─ {name}{C.RESET} [{sc} skills, {cc} commands] {status}{warn_str}") # Manifest manifest = plugin["sections"]["manifest"] if manifest.errors or manifest.warnings: print(f" {C.BOLD}Manifest:{C.RESET}") print_validation_result("manifest", manifest) # Skills with issues skill_results = plugin["sections"]["skills"] skills_with_issues = {k: v for k, v in skill_results.items() if v.errors or v.warnings} if skills_with_issues: print(f" {C.BOLD}Skills with issues:{C.RESET}") for sname, vr in skills_with_issues.items(): print(f" {sname}:") print_validation_result(sname, vr, indent=6) # Commands with issues cmd_results = plugin["sections"]["commands"] cmds_with_issues = {k: v for k, v in cmd_results.items() if v.errors or v.warnings} if cmds_with_issues: print(f" {C.BOLD}Commands with issues:{C.RESET}") for cname, vr in cmds_with_issues.items(): print(f" {cname}:") print_validation_result(cname, vr, indent=6) # README readme = plugin["sections"]["readme"] if readme.errors or readme.warnings: print(f" {C.BOLD}README:{C.RESET}") print_validation_result("readme", readme) # Cross-references xrefs = plugin["sections"]["cross-refs"] if xrefs.errors or xrefs.warnings: print(f" {C.BOLD}Cross-references:{C.RESET}") print_validation_result("cross-refs", xrefs) print(f"{C.CYAN}└{'─'*69}{C.RESET}\n") # Summary print(f"{C.BOLD}{'='*70}") print(f" Summary") print(f"{'='*70}{C.RESET}") print(f" Plugins: {len(all_results)}") print(f" Skills: {total_skills}") print(f" Commands: {total_commands}") print(f" Total: {total_skills + total_commands} components") print() if total_errors == 0: print(f" {C.GREEN}{C.BOLD}✓ ALL CHECKS PASSED{C.RESET} ({total_warnings} warnings)") else: print(f" {C.RED}{C.BOLD}✗ {total_errors} ERRORS{C.RESET}, {total_warnings} warnings") print() return total_errors def main(): """Find and validate all plugins in the collection.""" # Ensure UTF-8 output on Windows if sys.platform == 'win32': import io sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') # Determine base path if len(sys.argv) > 1: base_path = sys.argv[1] else: base_path = os.path.dirname(os.path.abspath(__file__)) if not os.path.isdir(base_path): print(f"Error: {base_path} is not a directory") sys.exit(1) # Find all plugin directories (those containing .claude-plugin/) plugin_dirs = [] for entry in sorted(os.listdir(base_path)): full_path = os.path.join(base_path, entry) if os.path.isdir(full_path) and os.path.isdir(os.path.join(full_path, ".claude-plugin")): plugin_dirs.append(full_path) if not plugin_dirs: print(f"No plugins found in {base_path}") print("(Looking for directories containing .claude-plugin/)") sys.exit(1) print(f"Found {len(plugin_dirs)} plugins in {base_path}\n") # Validate each plugin all_results = [] for pd in plugin_dirs: all_results.append(validate_plugin(pd)) # Print report errors = print_report(all_results) sys.exit(1 if errors > 0 else 0) if __name__ == "__main__": main()