""" Generate llms.txt and llms-full.txt from documentation files. llms.txt: Generated from mkdocs nav structure with descriptions from page frontmatter llms-full.txt: Full concatenation of all markdown content """ import os from pathlib import Path from typing import Any, Dict, List, Optional import yaml # Configuration for which sections to include/exclude INCLUDE_SECTIONS = ["Getting started", "Concepts", "Guides", "Examples"] EXCLUDE_SECTIONS = ["Reference"] def read_frontmatter(file_path: Path) -> Dict[str, Any]: """Read YAML frontmatter from markdown file.""" try: content = file_path.read_text(encoding="utf-8") if content.startswith("---"): parts = content.split("---", 2) if len(parts) >= 3: frontmatter = yaml.safe_load(parts[1]) if isinstance(frontmatter, dict): return frontmatter except Exception as e: print(f"Warning: Failed to read frontmatter from {file_path}: {e}") return {} def get_page_info(page_path: str, docs_dir: Path) -> Optional[Dict[str, str]]: """Get title and description for a page from its frontmatter.""" # page_path is relative to docs_dir full_path = docs_dir / page_path if not full_path.exists(): return None frontmatter = read_frontmatter(full_path) # Get title from frontmatter or filename title = frontmatter.get("title") if not title: # Use filename as fallback title = full_path.stem.replace("-", " ").title() # Get description from frontmatter description = frontmatter.get("description", "") return {"title": title, "description": description} def parse_mkdocs_nav(mkdocs_config: Dict[str, Any], repo_root: str) -> List[Dict[str, Any]]: """Parse mkdocs nav structure and extract relevant sections.""" nav = mkdocs_config.get("nav", []) sections = [] # Get docs_dir from config docs_dir = Path(repo_root) / mkdocs_config.get("docs_dir", "docs") def extract_pages(content_list): """Recursively extract all pages from a section's content, including nested subsections.""" items = [] for item in content_list: if isinstance(item, str): # Plain string path like "examples.md" page_info = get_page_info(item, docs_dir) if page_info: items.append( { "type": "page", "title": page_info["title"], "path": item, "description": page_info["description"], } ) elif isinstance(item, dict): for title, path in item.items(): if isinstance(path, str): # Page with title page_info = get_page_info(path, docs_dir) if page_info: items.append( { "type": "page", "title": title, # Use title from nav "path": path, "description": page_info["description"], } ) elif isinstance(path, list): # Nested subsection - create subsection with its pages subsection_items = extract_pages(path) if subsection_items: items.append( { "type": "subsection", "title": title, "items": subsection_items, } ) return items def process_nav_items(nav_items): """Recursively process nav items to find matching sections.""" for item in nav_items: if isinstance(item, dict): for section_name, section_content in item.items(): # Check if this section should be included if section_name in INCLUDE_SECTIONS and section_name not in EXCLUDE_SECTIONS: # Extract all pages from this section, including nested subsections items = [] if isinstance(section_content, list): items = extract_pages(section_content) if items: sections.append( { "title": section_name, "items": items, } ) # Recursively process nested sections elif isinstance(section_content, list): process_nav_items(section_content) process_nav_items(nav) return sections def generate_llms_txt(repo_root: str, mkdocs_config: Dict[str, Any], output_path: str) -> None: """Generate llms.txt from mkdocs nav structure.""" # Get title, description, and base_url from mkdocs config title = mkdocs_config.get("site_name", "") description = mkdocs_config.get("site_description", "") base_url = mkdocs_config.get("site_url", "").rstrip("/") lines = [] # Title and description lines.append(f"# {title}\n") lines.append(f"> {description}\n") # Parse sections from mkdocs nav sections = parse_mkdocs_nav(mkdocs_config, repo_root) # Generate sections def render_items(items, indent_level=0): """Render items (pages and subsections) with proper formatting.""" rendered = [] for item in items: if item["type"] == "page": # Use .md paths as-is since hooks.py copies them to site url = f"{base_url}/{item['path']}" if item["description"]: rendered.append(f"- [{item['title']}]({url}): {item['description']}") else: rendered.append(f"- [{item['title']}]({url})") elif item["type"] == "subsection": # Render subsection header rendered.append(f"\n### {item['title']}\n") # Render subsection items rendered.extend(render_items(item["items"], indent_level + 1)) return rendered for section in sections: lines.append(f"## {section['title']}\n") lines.extend(render_items(section["items"])) lines.append("") # Write to file with open(output_path, "w", encoding="utf-8") as f: f.write("\n".join(lines)) def generate_llms_full_txt( repo_root: str, mkdocs_config: Dict[str, Any], output_path: str ) -> None: """Generate llms-full.txt by concatenating all pages from llms.txt sections.""" content_parts = [] # Get docs_dir from config docs_dir = Path(repo_root) / mkdocs_config.get("docs_dir", "docs") # Parse sections from mkdocs nav (same as llms.txt) sections = parse_mkdocs_nav(mkdocs_config, repo_root) def extract_page_paths(items): """Recursively extract all page paths from items (including nested subsections).""" paths = [] for item in items: if item["type"] == "page": paths.append(item["path"]) elif item["type"] == "subsection": paths.extend(extract_page_paths(item["items"])) return paths # Concatenate all pages from all sections for section in sections: for page_path in extract_page_paths(section["items"]): full_path = docs_dir / page_path if full_path.is_file(): try: content = full_path.read_text(encoding="utf-8") content_parts.append(f"# {page_path}\n\n{content}\n\n") except Exception as e: print(f"Warning: Failed to read {page_path}: {e}") else: print(f"Warning: File not found: {page_path}") # Write to file if content_parts: with open(output_path, "w", encoding="utf-8") as f: f.write("".join(content_parts)) else: print("Warning: No content found for llms-full.txt") def generate_llms_files(repo_root: str, site_dir: str, mkdocs_config: Dict[str, Any]) -> None: """Generate both llms.txt and llms-full.txt.""" llms_txt_path = os.path.join(site_dir, "llms.txt") llms_full_txt_path = os.path.join(site_dir, "llms-full.txt") print("Generating llms.txt from mkdocs nav...") generate_llms_txt(repo_root, mkdocs_config, llms_txt_path) print("Generated llms.txt") print("Generating llms-full.txt...") generate_llms_full_txt(repo_root, mkdocs_config, llms_full_txt_path) print("Generated llms-full.txt")