//! Python-Markdown anchor generation (used by MkDocs) //! //! This module implements the exact slugify algorithm from Python-Markdown's //! `toc` extension, which is the default used by MkDocs. //! //! Algorithm (from markdown/extensions/toc.py): //! 1. NFKD Unicode normalization, then encode to ASCII (ignore errors) //! 2. Remove all characters except `\w`, `\s`, and `-` //! 3. Lowercase and strip leading/trailing whitespace //! 4. Collapse consecutive hyphens and whitespace into a single separator //! //! Verified against Python-Markdown 3.x via: //! ```bash //! uv run --with markdown python3 -c " //! from markdown.extensions.toc import slugify //! print(slugify('Your Heading', '-')) //! " //! ``` use regex::Regex; use std::sync::LazyLock; use unicode_normalization::UnicodeNormalization; use super::common::MAX_INPUT_LENGTH; // HTML/JSX tag stripping - strip entire HTML tags from heading text before anchor generation static HTML_TAG_PATTERN: LazyLock = LazyLock::new(|| Regex::new(r"(?i)]*>").unwrap()); // HTML comment stripping. Matches what Python-Markdown's TOC treeprocessor does // effectively: HTML comments in headings become Comment nodes in the element // tree, so `itertext()` never sees the inner text and the surrounding // whitespace collapses via the final `[-\s]+` separator pass. // Non-greedy `.*?` keeps this ReDoS-safe on any single-line heading. static HTML_COMMENT_PATTERN: LazyLock = LazyLock::new(|| Regex::new(r"").unwrap()); // Code span pattern for extracting content from backtick-delimited spans static CODE_PATTERN: LazyLock = LazyLock::new(|| Regex::new(r"`+([^`]*?)`+").unwrap()); static STRIP_NON_WORD: LazyLock = LazyLock::new(|| { // Matches anything that is NOT a word character, whitespace, or hyphen. // `\w` in Rust's regex (with Unicode enabled) matches [a-zA-Z0-9_] plus Unicode letters/digits. // Python-Markdown operates on ASCII-only text at this point, but we replicate the // intent: keep alphanumeric, underscore, whitespace, and hyphen. Regex::new(r"[^\w\s-]").unwrap() }); static COLLAPSE_SEPARATORS: LazyLock = LazyLock::new(|| { // Collapse one or more hyphens/whitespace into the separator Regex::new(r"[-\s]+").unwrap() }); /// Generate a Python-Markdown style anchor fragment from heading text. /// /// This matches the behavior of `markdown.extensions.toc.slugify(value, '-')`. /// /// # Examples /// ``` /// use rumdl_lib::utils::anchor_styles::python_markdown; /// /// assert_eq!(python_markdown::heading_to_fragment("Hello World"), "hello-world"); /// assert_eq!( /// python_markdown::heading_to_fragment("Cross-references to other projects / inventories"), /// "cross-references-to-other-projects-inventories" /// ); /// assert_eq!(python_markdown::heading_to_fragment("test_with_underscores"), "test_with_underscores"); /// ``` pub fn heading_to_fragment(heading: &str) -> String { if heading.is_empty() { return String::new(); } // Enforce input size limit let input = if heading.len() > MAX_INPUT_LENGTH { let mut end = MAX_INPUT_LENGTH; while end < heading.len() && !heading.is_char_boundary(end) { end += 1; } &heading[..end.min(heading.len())] } else { heading }; // Extract code span content to protect from HTML tag stripping let mut code_extracts: Vec = Vec::new(); let input = CODE_PATTERN .replace_all(input, |caps: ®ex::Captures| { let idx = code_extracts.len(); let content = caps.get(1).map_or("", |m| m.as_str()); code_extracts.push(content.to_string()); format!("\x00CODE{idx}\x00") }) .to_string(); // Strip HTML comments before tag stripping. The later separator-collapse // step folds the surrounding whitespace, matching Python-Markdown's TOC // behavior where comments never enter the anchor text. let input = if input.contains("") // HTML comments never contribute to the anchor, and the later `[-\s]+` // collapse folds any surrounding whitespace. assert_eq!(heading_to_fragment("Hello "), "hello"); assert_eq!(heading_to_fragment("A B"), "a-b"); assert_eq!(heading_to_fragment(" Title"), "title"); assert_eq!(heading_to_fragment("Title"), "title"); assert_eq!(heading_to_fragment("Hello"), "hello"); assert_eq!(heading_to_fragment("Hello World"), "hello-world"); assert_eq!(heading_to_fragment("Arrow Test"), "arrow-test"); assert_eq!(heading_to_fragment("Has inside"), "has-inside"); // Real `-->` outside the comment remains literal text; Python-Markdown // filters the `>` out and the hyphens collapse with surrounding spaces. assert_eq!(heading_to_fragment("Has and --> outside"), "has-and-outside"); // Unclosed comments are not stripped - left as literal text assert_eq!(heading_to_fragment("Unclosed B"), "a-b"); assert_eq!(heading_to_fragment("A -> B"), "a-b"); } #[test] fn test_leading_trailing_stripped() { assert_eq!(heading_to_fragment(" Hello "), "hello"); } #[test] fn test_mixed_separators() { // Spaces, hyphens all collapsed into single `-` assert_eq!(heading_to_fragment("a - b - c"), "a-b-c"); assert_eq!(heading_to_fragment("a - b"), "a-b"); } }