/** * Copies every core package / app README into the Fumadocs content tree so the * whole monorepo is documented at `/docs` instead of on an external docs host. * * READMEs are GitHub-flavoured markdown, which is not valid MDX: they carry raw * HTML badge blocks with unclosed `

` wrappers and non-self-closing `` / * `
` tags. `convertReadmeToMdx` normalises those so the pages compile, and * leaves the prose untouched. * * Every `packages/*` directory with a README gets a page. The curated * `PACKAGES` list below sets the sidebar title, icon and category; a package * missing from it is still synced, under "Other" with a default icon, so a new * package never silently drops out of the docs. * * Run with `bun run docs:sync` after editing a package README. CI runs it on * every docs publish (`.github/workflows/docs-sync.yml`). */ import { existsSync } from 'node:fs' import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises' import * as path from 'node:path' const DOCS_ROOT = path.resolve(import.meta.dirname, '..') const REPO_ROOT = path.resolve(DOCS_ROOT, '../..') const CONTENT_ROOT = path.join(DOCS_ROOT, 'content/docs/(index)') const REPO_URL = 'https://github.com/OpenSourceAGI/dev-tools-starter-agent' const DEFAULT_BRANCH = 'master' interface Entry { /** Directory relative to the repo root. */ dir: string /** Sidebar label. */ title: string /** Lucide icon name, resolved by the docs `lucideIconsPlugin`. */ icon: string /** Falls back to the package.json description when omitted. */ description?: string /** Category separator rendered above this page in the sidebar. */ category?: string } const PACKAGES: Entry[] = [ { category: 'CLI Tools', dir: 'packages/about-system', title: 'about-system', icon: 'Cpu', }, { dir: 'packages/api2ai-mcp-generator', title: 'api2ai', icon: 'WandSparkles', }, { dir: 'packages/create-cloud-db', title: 'create-cloud-db', icon: 'Database', }, { dir: 'packages/create-starter-app', title: 'create-starter-app', icon: 'Rocket', }, { dir: 'packages/git0-repo-downloader', title: 'git0', icon: 'GitBranch', }, { dir: 'packages/open-when-ready', title: 'open-ready', icon: 'Globe', }, { dir: 'packages/server-shell-setup', title: 'server-shell-setup', icon: 'HardDrive', description: 'One-command bootstrap for a modern dev environment: fish, nvim, nushell, bun, node, helix, starship, docker, and more.', }, { category: 'Libraries', dir: 'packages/code-tree-graph', title: 'code-tree-graph', icon: 'Workflow', }, { dir: 'packages/export-svg-icons-typescript', title: 'export-svg-typescript', icon: 'Image', }, { dir: 'packages/legal-terms-privacy-policy', title: 'legal-terms', icon: 'Scale', }, { dir: 'packages/manage-storage', title: 'manage-storage', icon: 'HardDrive', }, { dir: 'packages/ask-ai-button', title: 'ask-ai-button', icon: 'Sparkles', }, { dir: 'packages/react-app-store-buttons', title: 'react-app-store-buttons', icon: 'Smartphone', }, { dir: 'packages/setup-git-repo', title: 'setup-git-repo', icon: 'GitBranch', }, { dir: 'packages/template-git-repo', title: 'template-git-repo', icon: 'GitFork', }, { category: 'Infrastructure', dir: 'packages/cloudflare-to-claude-fix', title: 'cloudflare-to-claude-fix', icon: 'Zap', description: 'Cloudflare Workers queue consumer that fires a Claude Code routine whenever a Workers build fails.', }, { dir: 'packages/native-app-wrapper', title: 'native-app-wrapper', icon: 'MonitorSmartphone', }, { dir: 'packages/test-google-login', title: 'test-google-login', icon: 'LogIn', }, { dir: 'packages/verify-phone-sms', title: 'verify-phone-sms', icon: 'Shield', }, { dir: 'packages/web2mobile-wrapper', title: 'create-mobile-wrapper', icon: 'Smartphone', description: 'Turn any website URL into a native iOS and Android app wrapper with push notifications.', }, ] const APPS: Entry[] = [ { dir: 'apps/Cloud-Computer-Control-Panel', title: 'Cloud Computer Control Panel', icon: 'Server', }, { dir: 'apps/cccp-vscode-ext', title: 'CCCP for VS Code', icon: 'Cloud', }, { dir: 'apps/vscode-cloud', title: 'vscode-cloud', icon: 'Code', description: 'Per-user VS Code (code-server) instances on Cloudflare Containers, with SSO, isolated storage, and traffic routing.', }, ] /** Icon for packages not yet in the curated `PACKAGES` list. */ const DEFAULT_PACKAGE_ICON = 'Package' /** * The curated list plus any `packages/*` directory it does not mention, so a * package added without touching this file still gets a page. */ async function discoverPackages(): Promise { const known = new Set(PACKAGES.map((entry) => entry.dir)) const dirents = await readdir(path.join(REPO_ROOT, 'packages'), { withFileTypes: true, }) const extra: Entry[] = [] for (const dirent of dirents.sort((a, b) => a.name.localeCompare(b.name))) { const dir = `packages/${dirent.name}` if (!dirent.isDirectory() || known.has(dir) || !findReadme(dir)) continue extra.push({ ...(extra.length === 0 ? { category: 'Other' } : {}), dir, title: (await readPackageName(dir)) ?? dirent.name, icon: DEFAULT_PACKAGE_ICON, }) } if (extra.length) { console.warn( `[sync-readme-docs] not in the curated list, synced under "Other": ${extra .map((entry) => entry.dir) .join(', ')}` ) } return [...PACKAGES, ...extra] } const SECTIONS = [ { slug: 'packages', title: 'Packages', icon: 'Package', description: 'Every published CLI, library, and service in the monorepo.', entries: discoverPackages, }, { slug: 'apps', title: 'Apps', icon: 'AppWindow', description: 'Full applications shipped from this repository.', entries: async () => APPS, }, ] as const /** Void elements must be self-closed to parse as JSX. */ const VOID_TAGS = [ 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr', ] /** HTML attributes React spells differently. */ const ATTRIBUTE_RENAMES: Record = { allowfullscreen: 'allowFullScreen', autoplay: 'autoPlay', class: 'className', colspan: 'colSpan', crossorigin: 'crossOrigin', for: 'htmlFor', frameborder: 'frameBorder', maxlength: 'maxLength', readonly: 'readOnly', rowspan: 'rowSpan', srcset: 'srcSet', tabindex: 'tabIndex', } /** Fence languages Shiki does not know, mapped to the closest one it does. */ const LANGUAGE_ALIASES: Record = { env: 'dotenv', } const KNOWN_TAGS = new Set([ ...VOID_TAGS, 'a', 'b', 'blockquote', 'center', 'code', 'details', 'div', 'em', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'i', 'kbd', 'li', 'ol', 'p', 'picture', 'pre', 'span', 'strong', 'sub', 'summary', 'sup', 'table', 'tbody', 'td', 'th', 'thead', 'tr', 'ul', 'video', ]) const README_NAMES = ['README.md', 'readme.md', 'Readme.md'] const OPEN_TOKEN = '\u0000CODE' const CLOSE_TOKEN = '\u0000' function findReadme(dir: string): string | undefined { for (const name of README_NAMES) { const candidate = path.join(REPO_ROOT, dir, name) if (existsSync(candidate)) return candidate } return undefined } async function readPackageJson( dir: string ): Promise<{ name?: unknown; description?: unknown } | undefined> { const pkgPath = path.join(REPO_ROOT, dir, 'package.json') if (!existsSync(pkgPath)) return undefined try { return JSON.parse(await readFile(pkgPath, 'utf8')) } catch { return undefined } } async function readPackageName(dir: string): Promise { const name = (await readPackageJson(dir))?.name return typeof name === 'string' && name.trim() ? name.trim() : undefined } async function readDescription(dir: string): Promise { const raw = (await readPackageJson(dir))?.description const description = typeof raw === 'string' ? raw.trim() : undefined if (!description) return undefined // package.json descriptions occasionally embed markdown links. return description.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1') } /** Replaces fenced and inline code with placeholders so transforms skip them. */ function protectCode(input: string): { text: string restore: (value: string) => string } { const blocks: string[] = [] const stash = (value: string) => { blocks.push(value) return `${OPEN_TOKEN}${blocks.length - 1}${CLOSE_TOKEN}` } let text = input.replace( /^([ \t]*)(`{3,}|~{3,})([^\n]*)\n([\s\S]*?)^[ \t]*\2[ \t]*$/gm, (_match, indent: string, fence: string, info: string, body: string) => { const [lang = '', ...rest] = info.trim().split(/\s+/) const alias = LANGUAGE_ALIASES[lang.toLowerCase()] const meta = [alias ?? lang, ...rest].filter(Boolean).join(' ') return stash(`${indent}${fence}${meta}\n${body}${indent}${fence}`) } ) // An unterminated fence would otherwise swallow the rest of the transforms. text = text.replace(/^([ \t]*)(`{3,}|~{3,})([^\n]*)\n[\s\S]*$/m, (match) => stash(match.trimEnd()) ) text = text.replace(/(`+)(?:(?!\1)[\s\S])*?\1/g, (match) => stash(match)) const placeholder = new RegExp(`${OPEN_TOKEN}(\\d+)${CLOSE_TOKEN}`, 'g') return { text, // A stashed block can itself contain placeholders (an unterminated fence // swallows earlier ones), so keep expanding until none are left. restore: (value) => { let result = value while (placeholder.test(result)) { placeholder.lastIndex = 0 result = result.replace( placeholder, (_m, index: string) => blocks[Number(index)] ) } return result }, } } function rewriteRelativeLinks(text: string, dir: string): string { const toAbsolute = (target: string) => { const [pathPart, hash = ''] = target.split('#') const resolved = path.posix .normalize(path.posix.join(dir, pathPart)) .replace(/^\/+/, '') const kind = pathPart.endsWith('/') ? 'tree' : 'blob' const suffix = hash ? `#${hash}` : '' return `${REPO_URL}/${kind}/${DEFAULT_BRANCH}/${resolved}${suffix}` } return text .replace( /(\]\()(?!https?:|#|mailto:|\/\/|data:)([^)\s]+)(\))/g, (_m, open: string, target: string, close: string) => `${open}${toAbsolute(target)}${close}` ) .replace( /(<(?:a|img)\b[^>]*?\b(?:href|src)=")(?!https?:|#|mailto:|\/\/|data:)([^"]+)(")/g, (_m, open: string, target: string, close: string) => `${open}${toAbsolute(target)}${close}` ) } /** * Rewrites an HTML attribute list as JSX. Values keep their quoting so query * strings such as `?style=flat-square` are not mistaken for new attributes. */ function normaliseAttributes(raw: string): string { const pattern = /([A-Za-z_:][-A-Za-z0-9_:.]*)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g const attributes: string[] = [] for (const match of raw.matchAll(pattern)) { const name = match[1] // `style="a: b"` is a string in HTML but must be an object in JSX. if (name.toLowerCase() === 'style') continue const jsxName = ATTRIBUTE_RENAMES[name.toLowerCase()] ?? name const value = match[2] ?? match[3] ?? match[4] attributes.push( value === undefined ? jsxName : `${jsxName}="${value.replace(/"/g, '"')}"` ) } return attributes.length ? ` ${attributes.join(' ')}` : '' } /** Rewrites a single raw HTML tag into JSX-safe form. */ function normaliseTag(tag: string): string { const match = tag.match( /^<\s*(\/?)\s*([A-Za-z][A-Za-z0-9-]*)([\s\S]*?)(\/?)>$/ ) if (!match) return tag const [, closing, rawName, rawAttrs, selfClosing] = match const name = rawName.toLowerCase() if (!KNOWN_TAGS.has(name)) return tag if (closing) return `` const attrs = normaliseAttributes(rawAttrs) if (VOID_TAGS.includes(name)) return `<${name}${attrs} />` return `<${name}${attrs}${selfClosing ? ' /' : ''}>` } /** * Closes tags a README block left open and drops orphan closing tags, so each * markdown HTML block is balanced JSX. */ function balanceBlock(block: string): string { const stack: string[] = [] let output = '' let index = 0 for (const match of block.matchAll( /<\/?([A-Za-z][A-Za-z0-9-]*)[^>]*?\/?>/g )) { const tag = match[0] const name = match[1].toLowerCase() output += block.slice(index, match.index) index = match.index + tag.length if ( !KNOWN_TAGS.has(name) || VOID_TAGS.includes(name) || tag.endsWith('/>') ) { output += tag continue } if (tag.startsWith('` open = stack.pop() } output += tag continue } stack.push(name) output += tag } output += block.slice(index) while (stack.length) output += `\n` return output } function convertReadmeToMdx(raw: string, dir: string): string { const { text: protectedText, restore } = protectCode( raw.replace(/\r\n/g, '\n') ) let text = protectedText // HTML comments are not valid MDX. .replace(//g, '') // The page title comes from frontmatter; drop the README's own H1. .replace(/^#\s+[^\n]*\n?/m, '') text = rewriteRelativeLinks(text, dir) // `

` badge wrappers routinely contain headings, which React refuses to // nest inside a paragraph. They are always layout wrappers here. text = text .replace(/]*)?>/gi, (_m, attrs) => ``) .replace(/<\/p>/gi, '') text = text.replace(/<\/?[A-Za-z][A-Za-z0-9-]*[^>]*?>/g, normaliseTag) // Anything left that looks like a tag but is not one (``, `< 5`), plus // braces, which MDX would otherwise read as expressions. text = text .replace(/<(?!\/?[A-Za-z][A-Za-z0-9-]*[\s/>])/g, '<') .replace(/<\/?([A-Za-z][A-Za-z0-9-]*)/g, (match, name: string) => KNOWN_TAGS.has(name.toLowerCase()) ? match : `<${match.slice(1)}` ) .replace(/[{}]/g, (brace) => (brace === '{' ? '{' : '}')) text = text .split(/\n{2,}/) .map((block) => (block.includes('<') ? balanceBlock(block) : block)) .join('\n\n') return restore(text).trim() } function yamlString(value: string): string { return `'${value.replace(/'/g, "''")}'` } async function buildPage(entry: Entry): Promise { const readmePath = findReadme(entry.dir) if (!readmePath) { console.warn(`[sync-readme-docs] no README found in ${entry.dir}`) return undefined } const raw = await readFile(readmePath, 'utf8') const description = entry.description ?? (await readDescription(entry.dir)) const relative = path.relative(REPO_ROOT, readmePath) const source = `${REPO_URL}/blob/${DEFAULT_BRANCH}/${relative}` const frontmatter = [ '---', `title: ${yamlString(entry.title)}`, ...(description ? [`description: ${yamlString(description)}`] : []), `icon: ${entry.icon}`, '---', ].join('\n') const banner = `{/* Generated from ${relative} ${GENERATED_MARKER} Edit the README, then run \`bun run docs:sync\`. */}` const footer = `---\n\nSource: [\`${relative}\`](${source})` return `${frontmatter}\n\n${banner}\n\n${convertReadmeToMdx(raw, entry.dir)}\n\n${footer}\n` } const GENERATED_MARKER = 'by scripts/sync-readme-docs.ts.' /** * Deletes pages a previous sync wrote, so a package that was removed or renamed * does not leave a stale page. Hand-written pages in the same folder (no * generated banner) are kept. */ async function removeGeneratedPages(outDir: string) { for (const name of await readdir(outDir)) { if (!name.endsWith('.mdx')) continue const file = path.join(outDir, name) if ((await readFile(file, 'utf8')).includes(GENERATED_MARKER)) { await rm(file) } } } export async function syncReadmeDocs() { for (const section of SECTIONS) { const outDir = path.join(CONTENT_ROOT, section.slug) await mkdir(outDir, { recursive: true }) await removeGeneratedPages(outDir) const pages: string[] = [] let written = 0 for (const entry of await section.entries()) { const page = await buildPage(entry) if (!page) continue const slug = path.basename(entry.dir) await writeFile(path.join(outDir, `${slug}.mdx`), page, 'utf8') if (entry.category) pages.push(`---${entry.category}---`) pages.push(slug) written += 1 } await writeFile( path.join(outDir, 'meta.json'), `${JSON.stringify( { title: section.title, icon: section.icon, description: section.description, pages, }, null, 2 )}\n`, 'utf8' ) console.log( `[sync-readme-docs] wrote ${written} pages to content/docs/(index)/${section.slug}/` ) } } if (import.meta.main) { await syncReadmeDocs().catch((error) => { console.error('[sync-readme-docs] failed', error) process.exit(1) }) }