import { isHighValueDocsSlug } from './sitemap-docs' import { getStartExampleSitemapEntries } from './start-example-pages' import { getBranch, libraries } from '~/libraries' import type { LibrarySlim } from '~/libraries/types' import { getPublishedPosts } from '~/utils/blog' import { getDocsManifest } from '~/utils/docs' import { getPartnerSitemapEntries } from '~/utils/partner-pages' import { SITE_URL } from '~/utils/site' import { getChartsCatalogSitemapEntries } from './charts-catalog' export type SitemapEntry = { path: string lastModified?: string } const HIGH_VALUE_NON_DOC_PAGES = [ '/', '/blog', '/libraries', '/learn', '/showcase', '/support', '/workshops', '/paid-support', ] as const satisfies ReadonlyArray function trimTrailingSlash(url: string) { return url.replace(/\/$/, '') } function escapeXml(value: string) { return value .replace(/&/g, '&') .replace(//g, '>') .replace(/"/g, '"') .replace(/'/g, ''') } function asLastModified(value: string) { return new Date(`${value}T12:00:00.000Z`).toISOString() } function getLibraryEntries(): Array { return libraries.flatMap((library) => { if ( library.visible === false || !library.latestVersion || library.sitemap?.includeLandingPage !== true ) { return [] } const basePath = `/${library.id}/latest` return [{ path: basePath }] }) } async function getLibraryDocsEntries( library: LibrarySlim, ): Promise> { if ( library.visible === false || !library.latestVersion || library.sitemap?.includeDocsPages !== true ) { return [] } const docsRoot = library.docsRoot || 'docs' const branch = getBranch(library, 'latest') const manifest = await getDocsManifest({ repo: library.repo, branch, docsRoot, }).catch(() => ({ paths: [], redirects: {} })) return manifest.paths .filter(Boolean) .filter(isHighValueDocsSlug) .map((slug) => ({ path: `/${library.id}/latest/docs/${slug}`, })) } function getBlogEntries(): Array { return getPublishedPosts().map((post) => ({ path: `/blog/${post.slug}`, lastModified: asLastModified(post.published), })) } export function getSiteOrigin() { return trimTrailingSlash(SITE_URL) } export async function getSitemapEntries(): Promise> { const [docsEntries, chartsCatalogEntries, showcaseEntries] = await Promise.all([ Promise.all(libraries.map((library) => getLibraryDocsEntries(library))), getPublishedChartsCatalogEntries(), import('./showcase-sitemap.server').then( ({ getShowcaseSitemapEntries }) => getShowcaseSitemapEntries(), ), ]) const entries = [ ...HIGH_VALUE_NON_DOC_PAGES.map((path) => ({ path })), ...getLibraryEntries(), ...getStartExampleSitemapEntries(), ...docsEntries.flat(), ...getBlogEntries(), ...getPartnerSitemapEntries(), ...chartsCatalogEntries, ...showcaseEntries, ].filter( (entry) => entry.path !== '/intent/registry' && !entry.path.startsWith('/intent/registry/'), ) return Array.from( new Map(entries.map((entry) => [entry.path, entry])).values(), ) } async function getPublishedChartsCatalogEntries(): Promise< Array > { try { const { getChartsCatalogIndexPublication } = await import('./charts-catalog-index.server') const publication = await getChartsCatalogIndexPublication() return getChartsCatalogSitemapEntries(publication.index) } catch (error) { console.error('[sitemap] Charts catalog unavailable', error) return [{ path: '/charts/catalog' }] } } export async function generateSitemapXml(origin: string) { const urls = (await getSitemapEntries()) .map((entry) => { const loc = `${origin}${entry.path}` return [ ' ', ` ${escapeXml(loc)}`, entry.lastModified ? ` ${entry.lastModified}` : '', ' ', ] .filter(Boolean) .join('\n') }) .join('\n') return ` ${urls} ` } export function generateRobotsTxt(origin: string) { return [ 'User-agent: *', 'Allow: /', '', `Sitemap: ${origin}/sitemap.xml`, ].join('\n') }