const fs = require('fs');
const path = require('path');
const glob = require('glob');
const DOCS_DIR = path.join(__dirname, '../app/ui/docs');
const OUTPUT_DIR = path.join(__dirname, '../public');
const LLMS_TXT_PATH = path.join(OUTPUT_DIR, 'llms.txt');
const LLMS_FULL_TXT_PATH = path.join(OUTPUT_DIR, 'llms-full.txt');
// ---------------------------------------------------------------------------
// 1. MDX → Clean Markdown
// ---------------------------------------------------------------------------
/**
* Strip the opening tag (including its
* mdxContent prop with nested template literals) and
* closing tag. The body content after the opening tag IS the actual documentation.
*
* We can't use a simple regex for the opening tag because the mdxContent prop
* contains nested template literals (backticks inside backticks), so we find
* the opening tag start and then walk forward to find its closing `>`.
*/
function stripAdvancedComponentLLMGenerator(raw) {
const tagStart = raw.indexOf('' of the opening tag.
// We need to handle nested braces {} because the mdxContent prop uses {`...`}.
let depth = 0;
let pos = tagStart + '' && depth === 0) {
// Found the closing '>' of the opening tag
break;
}
pos++;
}
if (pos >= raw.length) return raw;
// Strip: from tagStart to pos+1 (inclusive of the '>')
let content = raw.substring(0, tagStart) + raw.substring(pos + 1);
// Strip the closing tag
content = content.replace(/<\/AdvancedComponentLLMGenerator>/g, '');
return content;
}
/**
* Main MDX → clean markdown converter.
* Handles all the common MDX patterns used in gluestack-ui docs:
* - import statements → removed
* - → fenced code blocks
* - → fenced code blocks
* - // → markdown tables
* - ... → `code`
* - / → content extracted per tab
* - Visual-only components → stripped
*/
function cleanMDX(raw) {
let content = raw;
// --- Strip AdvancedComponentLLMGenerator wrapper ---
// This component wraps the documentation for LLM generation at runtime.
// The body content after the opening tag IS the documentation — we just
// need to strip the tag itself (its mdxContent prop has nested template
// literals that can't be parsed by regex).
content = stripAdvancedComponentLLMGenerator(content);
// --- Remove all import lines ---
content = content.replace(/^import\s+[\s\S]*?from\s+['"][^'"]*['"];?\s*$/gm, '');
// --- Process → fenced code block ---
content = replaceCodePreviewer(content);
// --- Process → fenced code block ---
content = replaceCodeBlock(content);
// --- Protect fenced code blocks from subsequent transformations ---
const codeBlocks = [];
content = content.replace(/```[\s\S]*?```/g, (match) => {
codeBlocks.push(match);
return `%%CODE_BLOCK_${codeBlocks.length - 1}%%`;
});
// --- Process ... → `code` ---
content = content.replace(/([\s\S]*?)<\/InlineCode>/g, '`$1`');
// --- Process → markdown table ---
content = replaceTables(content);
// --- Process / → extract content per tab ---
content = replaceTabs(content);
// --- Strip remaining known visual/interactive components ---
content = stripVisualComponents(content);
// --- Strip remaining self-closing JSX components ---
content = content.replace(/<[A-Z][a-zA-Z]*\s*[^>]*\/>/g, '');
// --- Strip remaining paired JSX components but keep their text children ---
content = content.replace(/<([A-Z][a-zA-Z]*)[^>]*>([\s\S]*?)<\/\1>/g, '$2');
// --- Restore code blocks ---
content = content.replace(/%%CODE_BLOCK_(\d+)%%/g, (match, idx) => {
return codeBlocks[parseInt(idx)];
});
// --- Strip
and
---
content = content.replace(/
/gi, '');
// --- Strip empty <> ... > fragments ---
content = content.replace(/<>\s*<\/>/g, '');
// --- Resolve template literals with process.env ---
content = content.replace(
/\$\{process\.env\.NEXT_PUBLIC_GLUESTACK_COMMAND\s*\|\|\s*'([^']*)'\}/g,
'$1'
);
// --- Clean up excessive blank lines ---
content = content.replace(/\n{3,}/g, '\n\n');
// --- Remove trailing whitespace on each line ---
content = content
.split('\n')
.map((line) => line.trimEnd())
.join('\n');
return content.trim();
}
/**
* Replace with fenced code blocks.
*
* The code prop uses template literals that may contain JSX with curly braces,
* self-closing tags, and other syntax that makes regex parsing unreliable.
* Strategy: find the tag, find the code prop, then search for the known props
* (argTypes, reactLive, importMap) that always follow the code prop to locate
* where the code template literal ends.
*/
function replaceCodePreviewer(content) {
let result = content;
let searchStart = 0;
while (true) {
const tagStart = result.indexOf(' tagStart + 200) {
// No backtick-delimited code prop found near the tag start
searchStart = tagStart + 1;
continue;
}
const codeContentStart = codePropIdx + 'code={`'.length;
// Find the end of the entire tag
// We search for '/>' but must skip any '/>' inside the code template literal.
// The trick: the code prop is always followed by argTypes= or reactLive= or importMap=.
// Find the first occurrence of these after the code prop start.
const afterCodeStart = result.substring(codeContentStart);
const nextPropMatch = afterCodeStart.match(/[`\}]\s*\n\s*(?:argTypes|reactLive|importMap)\s*=/);
if (!nextPropMatch) {
// Can't find the end of the code prop — skip this tag
searchStart = tagStart + 1;
continue;
}
// The code content ends just before the closing backtick
const codeContentEnd = nextPropMatch.index;
// Find the self-closing /> of the entire tag
// After the nextProp, search for />
const afterCodeProp = codeContentStart + nextPropMatch.index + nextPropMatch[0].length;
const tagEndIdx = result.indexOf('/>', afterCodeProp);
if (tagEndIdx === -1) {
searchStart = tagStart + 1;
continue;
}
const code = result.substring(codeContentStart, codeContentStart + codeContentEnd);
const replacement = '\n```jsx\n' + code.trim() + '\n```\n';
const tagEnd = tagEndIdx + 2; // past '/>'
result = result.substring(0, tagStart) + replacement + result.substring(tagEnd);
searchStart = tagStart + replacement.length;
}
return result;
}
/**
* Replace with fenced code blocks.
*/
function replaceCodeBlock(content) {
//
const codeBlockBacktickRegex = //g;
content = content.replace(codeBlockBacktickRegex, (match, code, lang) => {
return '\n```' + lang + '\n' + code.trim() + '\n```\n';
});
//
const codeBlockQuoteRegex = //g;
content = content.replace(codeBlockQuoteRegex, (match, code, lang) => {
return '\n```' + lang + '\n' + code.trim() + '\n```\n';
});
// (no language)
const codeBlockNoLangRegex = //g;
content = content.replace(codeBlockNoLangRegex, (match, code) => {
return '\n```\n' + code.trim() + '\n```\n';
});
// (no language)
const codeBlockNoLangQuoteRegex = //g;
content = content.replace(codeBlockNoLangQuoteRegex, (match, code) => {
return '\n```\n' + code.trim() + '\n```\n';
});
return content;
}
/**
* Convert with TableRow/TableCell to markdown tables.
*/
function replaceTables(content) {
// Find each block
const tableRegex = /([\s\S]*?)<\/Table>/g;
content = content.replace(tableRegex, (match, inner) => {
const rows = [];
// Extract header rows
const headerRegex = /([\s\S]*?)<\/TableHeader>/g;
let headerMatch;
while ((headerMatch = headerRegex.exec(inner)) !== null) {
const rowRegex = /([\s\S]*?)<\/TableRow>/g;
let rowMatch;
while ((rowMatch = rowRegex.exec(headerMatch[1])) !== null) {
const cells = extractCells(rowMatch[1], 'TableHeaderCell');
if (cells.length > 0) rows.push(cells);
}
}
// Extract body rows
const bodyRegex = /([\s\S]*?)<\/TableBody>/g;
let bodyMatch;
while ((bodyMatch = bodyRegex.exec(inner)) !== null) {
const rowRegex = /([\s\S]*?)<\/TableRow>/g;
let rowMatch;
while ((rowMatch = rowRegex.exec(bodyMatch[1])) !== null) {
const cells = extractCells(rowMatch[1], 'TableCell');
if (cells.length > 0) rows.push(cells);
}
}
if (rows.length === 0) return '';
// Build markdown table
const colCount = rows[0].length;
let md = '\n';
// Header row
md += '| ' + rows[0].join(' | ') + ' |\n';
// Separator
md += '| ' + rows[0].map(() => '---').join(' | ') + ' |\n';
// Data rows
for (let i = 1; i < rows.length; i++) {
// Pad row to match header column count
while (rows[i].length < colCount) rows[i].push('');
md += '| ' + rows[i].join(' | ') + ' |\n';
}
return md;
});
return content;
}
/**
* Extract cell text from a table row, handling both TableCell and TableHeaderCell.
*/
function extractCells(rowContent, cellTag) {
const cells = [];
const cellRegex = new RegExp(`<${cellTag}>([\\s\\S]*?)<\\/${cellTag}>`, 'g');
let cellMatch;
while ((cellMatch = cellRegex.exec(rowContent)) !== null) {
let cellText = cellMatch[1].trim();
// Convert text within cells to `text`
cellText = cellText.replace(/([\s\S]*?)<\/InlineCode>/g, '`$1`');
// Clean up whitespace
cellText = cellText.replace(/\n/g, ' ').replace(/\s+/g, ' ').trim();
cells.push(cellText);
}
return cells;
}
/**
* Replace / blocks — extract content per tab with a heading.
*/
function replaceTabs(content) {
// Process each ... block
const tabsRegex = /([\s\S]*?)<\/Tabs>/g;
content = content.replace(tabsRegex, (match, inner) => {
let result = '';
const tabItemRegex = /([\s\S]*?)<\/TabItem>/g;
let tabMatch;
while ((tabMatch = tabItemRegex.exec(inner)) !== null) {
const label = tabMatch[1];
const tabContent = tabMatch[2].trim();
result += `\n**${label}:**\n\n${tabContent}\n\n`;
}
return result;
});
return content;
}
/**
* Strip visual-only / interactive components that don't contribute to LLM understanding.
*/
function stripVisualComponents(content) {
const stripPaired = [
'AnatomyImage',
'Responsiveness',
'ManualInstallationNote',
'GithubLink',
'StarterKitLink',
'KitchenSinkLink',
];
for (const tag of stripPaired) {
// Paired component: ... → removed entirely
const pairRegex = new RegExp(`<${tag}[^>]*>[\\s\\S]*?<\\/${tag}>`, 'g');
content = content.replace(pairRegex, '');
// Self-closing:
const selfClosingRegex = new RegExp(`<${tag}[^>]*/>`, 'g');
content = content.replace(selfClosingRegex, '');
}
// Strip (self-closing variant)
content = content.replace(/]*>/g, '');
// Strip and (embedded content, not useful for LLMs)
content = content.replace(/