import type { Asset, ContentEntry, Menu, NodeRef, Page, Redirect, Site, Snapshot, Taxonomy, Term, } from "@cloudhal/core-contracts"; import type { RedirectMatch, SearchKind, SearchResults, SiteGraphStore } from "@cloudhal/core-ts"; import { matchRedirect, redirectLocale } from "@cloudhal/core-ts"; import { hasGates, inheritedGate, isAvailable, nextInstant } from "@cloudhal/core-ts/availability"; import { type ConversionEntries, type DerivedGate, derivedGates, type ResolvedFields, } from "@cloudhal/core-ts/fields"; import { matches } from "@cloudhal/core-ts/filter"; import { assetRefsOf } from "@cloudhal/core-ts/media"; import { evaluateEntryQuery, type FieldEntryQuery, type FieldEntryResult, rememberList, } from "./entry-query.ts"; import { gateIndex, splitEntries } from "./entry-views.ts"; import { type EntryFieldsOptions, entryFields, type RenderedFields } from "./fields.ts"; import { type Navigation, type NavigationOptions, navigationFor } from "./navigation.ts"; import { cachedConversionMaps, conversionMapsOf, siteConverts } from "./reference-conversions.ts"; import { membersIn, type ReferencedByQuery, ReferenceIndex, referrersIn } from "./references.ts"; import { byNewest, byPosition, compile, shown, termFilter, visible } from "./source-helpers.ts"; export { LiveSource } from "./live-source.ts"; export type { ReferencedByQuery } from "./references.ts"; /** * What a renderer needs from a site, independent of whether it reads live nodes (dev) or a * published snapshot (build/production). */ export interface GraphSource { readonly mode: "live" | "snapshot"; site(): Promise; pages(): Promise; pageByPath(path: string, locale?: string): Promise; /** Listed entries of a collection, newest first (RFC-0042: unlisted ones left out). */ entries(collection: string, locale?: string): Promise; /** An entry with a public URL by slug: listed, or unlisted past its collection's expiry. */ entryBySlug(collection: string, slug: string, locale?: string): Promise; /** Every listed entry of a locale, newest first, across collections (feeds, sitemaps). */ allEntries(locale?: string): Promise; /** * RFC-0042 §3: every entry of a locale with a public URL, newest first — `allEntries` plus the * unlisted ones (routes, static builds, language alternates). Optional for older sources. */ routableEntries?(locale?: string): Promise; /** * RFC-0042 §4: a collection filtered and ordered by its list fields, paged (`queryEntries`). * Optional for older sources; the standalone `queryEntries(source, q)` works with any. */ queryEntries?(q: FieldEntryQuery): Promise; /** * A visible entry by id, any locale (reader comments name the entry by id, RFC-0007 §6.2). * Optional for sources that predate comments. */ entryById?(id: string): Promise; /** The site's taxonomies (RFC-0002). */ taxonomies(): Promise; /** * Terms of a taxonomy that may be shown, by `position`, then name. Snapshot mode (and live mode * without drafts): active and public only. Live mode with drafts: every non-retired term. */ terms(key: string): Promise; /** A shown term by its current slug (old slugs answer with a redirect, not a term). */ termBySlug(key: string, slug: string): Promise; /** * Every term of a taxonomy (or of the site, without `key`) in any status: proposed, internal and * retired terms included. Routes need them to tell a 404 from a 410; never list them publicly. */ vocabulary(key?: string): Promise; /** * Visible entries matching a filter expression (RFC-0002 §5), newest first. Unknown terms match * nothing (a published snapshot must keep rendering) and are logged. */ entriesByFilter( filter: string, options?: { locale?: string; limit?: number }, ): Promise; /** Visible entries assigned to a term (with `descendants`, also to its sub-terms), newest first. */ entriesByTerm( key: string, slug: string, options?: { locale?: string; limit?: number; descendants?: boolean }, ): Promise; /** The redirect for a request path in a locale (default: the site's): exact, then longest prefix. */ redirectFor(path: string, locale?: string): Promise; /** Every redirect of the site (static builds write them out; see `redirectsFile`). */ redirects(): Promise; /** The menu stored under `key`, if the site has one (raw node, RFC-0003). */ menu(key: string): Promise; /** * The menu under `key` resolved for a locale (default: the site's): only shown items, with labels * and hrefs. Undefined: no menu under `key` (themes fall back to `derivedNavigation`). */ navigation( key: string, locale?: string, options?: NavigationOptions, ): Promise; /** * RFC-0004: every asset this source shows — a snapshot's captured assets, or the site's current * ones (any status: retired assets keep rendering where they are used). Optional for sources * that predate media. */ assets?(): Promise; /** The asset with this id, if this source shows it. */ asset?(id: string): Promise; /** The asset behind a media URL's 16-hex prefix (the media route). */ assetByPrefix?(prefix16: string): Promise; /** * RFC-0008 §7.4: the next instant (ms since the epoch) at which a time gate changes what this * source shows; undefined when none lies ahead or the source ignores gates (previews, drafts). * Optional for sources that predate gates. */ nextGate?(): Promise; /** * RFC-0019 §3: articles matching a reader's query, best first. Snapshot mode reads the search * index (it matches the live snapshot; `stale` says when it is behind); live mode searches the * entries it shows, drafts included. Optional for sources that predate search. */ search?(query: string, options?: SourceSearchOptions): Promise; /** * RFC-0041 §9: an entry's typed fields as a theme draws them: resolved (a translation reads the * original's values where §5 says so), labelled in `options.locale` (default: the entry's), rich * text as sanitized HTML, URLs through `safeHref`, images as `` attributes. `{}` when the * collection has no definitions. Optional for sources that predate fields; any source works * with the standalone `entryFields(source, entry)`. */ entryFields?(entry: ContentEntry, options?: EntryFieldsOptions): Promise; /** * RFC-0045 §4: the members of these translation groups that this view shows, in this site, by * group (one call per resolution level; the live source reads the store once, a snapshot never). * Optional for sources that predate references: their references resolve to nothing. */ groupMembers?(groupIds: readonly string[]): Promise>; /** * RFC-0045 §4 "who references me": entries this view lists whose resolved values name the * group (a translation inheriting its original's value included), newest first, at most 100. */ referencedBy?(groupId: string, q?: ReferencedByQuery): Promise; /** RFC-0045 §7: the entry maps of a running text ↔ reference conversion in a collection. */ conversionEntries?(collection: string): Promise; } export interface SourceSearchOptions { page?: number; perPage?: number; /** Default: the site's default locale. */ locale?: string; /** RFC-0029: only these kinds (default: all). */ kinds?: readonly SearchKind[]; } /** Everything a snapshot shows before gates are applied; built once per snapshot and isolate. */ interface Loaded { site: Site; /** Published pages in `Site.pageIds` order and published entries, gates not yet applied. */ pages: Page[]; entries: ContentEntry[]; /** The same published entries (a view keeps them for originals; RFC-0042). */ published: ContentEntry[]; /** Published entries with a public URL (unlisted ones too); `entries` holds the listed. */ reachable: ContentEntry[]; /** * RFC-0045 §7: every entry, drafts and archived ones included (`conversionEntries` filters what * matches text itself); kept only while a conversion runs. */ current: ContentEntry[]; /** Every published entry's derived gate (RFC-0042 §3), computed once per snapshot. */ derived: ReadonlyMap; /** Resolved field values by entry id, filled as queries need them (the snapshot is immutable). */ resolved: Map; taxonomies: Taxonomy[]; terms: Term[]; redirects: Redirect[]; menus: Menu[]; assets: Asset[]; /** Assets by id and by 16-hex URL prefix, built once per snapshot (RFC-0004 §7.1). */ assetIndex: Map; /** * The gate index (RFC-0008 §7.4): every instant at which a published page's or entry's gate * changes — derived gates and list-field instants included (RFC-0042) — sorted. Empty for a * site that schedules nothing — then nothing is filtered at all. */ gates: number[]; } /** What a snapshot shows at one instant (or, for previews, regardless of gates). */ interface View { site: Site; pages: Page[]; /** Listed entries. */ entries: ContentEntry[]; /** Listed and unlisted entries (public URLs). */ reachable: ContentEntry[]; /** Every published entry of the snapshot, gates aside (originals of translations). */ published: ContentEntry[]; /** The snapshot's cache of resolved field values (shared by every view). */ resolved: Map; taxonomies: Taxonomy[]; terms: Term[]; redirects: Redirect[]; menus: Menu[]; assets: Asset[]; assetIndex: ReadonlyMap; } /** State shared by every view of one snapshot: the loaded nodes and the asset gates. */ interface Shared { loaded?: Promise | undefined; assetGates?: Promise>; /** RFC-0010: whether the snapshot's bundle was there when it loaded, or is being written. */ bundle?: "present" | "missing" | "persisting"; /** Bodies read node by node (no bundle), as stored JSON text, kept to write the bundle. */ fetched?: Map | undefined; /** RFC-0045 §4: members and referrers by group over the published entries, gates aside. */ references?: ReferenceIndex; /** RFC-0045 §7: conversion maps by collection, while a text ↔ reference conversion runs. */ conversions?: Map>; } /** Who may see the nodes of a snapshot: the clock gates are checked against, or nobody (previews). */ type Clock = (() => number) | null; /** * The embargo instant each asset inherits (RFC-0008 §7.2): an asset whose every live referrer — * published pages and entries, shown terms — has an embargo is hidden until the earliest one * opens. Only assets used by a gated node can inherit anything, so only their referrers are read. */ function assetGates(loaded: Loaded): Map { const out = new Map(); const gated = [...loaded.pages, ...loaded.entries].filter(hasGates); const kindOf = (n: Page | ContentEntry) => ("collection" in n ? "contentEntry" : "page"); const candidates = new Set(gated.flatMap((n) => assetRefsOf(kindOf(n), n).map((r) => r.assetId))); if (candidates.size === 0) return out; const referrers = new Map>(); const add = (kind: string, node: Page | ContentEntry | Term, gate: Page["availability"]) => { for (const ref of assetRefsOf(kind, node)) { if (!candidates.has(ref.assetId)) continue; const list = referrers.get(ref.assetId) ?? []; list.push(gate === undefined ? {} : { availability: gate }); referrers.set(ref.assetId, list); } }; for (const p of loaded.pages) add("page", p, p.availability); for (const e of loaded.entries) add("contentEntry", e, e.availability); for (const t of loaded.terms) if (shown(t, false)) add("term", t, undefined); for (const [id, list] of referrers) { const from = inheritedGate(list)?.from; if (from !== undefined) out.set(id, Date.parse(from)); } return out; } /** * Reads exactly the node bodies captured in a snapshot (immutable; only published nodes are * rendered). Time gates (RFC-0008 §7) apply on top: by default against the wall clock at each * call; `at(now)` fixes the instant for one request (the Site Worker does, so a page and the * lists on it agree); `ungated()` is the preview view, which shows gated nodes. */ export class SnapshotSource implements GraphSource { readonly mode = "snapshot"; readonly #store: SiteGraphStore; readonly #snapshot: Snapshot; readonly #shared: Shared; readonly #clock: Clock; #view: { now: number | null; view: Promise } | undefined; constructor( store: SiteGraphStore, snapshot: Snapshot, options: { now?: () => number; shared?: Shared; ungated?: boolean } = {}, ) { this.#store = store; this.#snapshot = snapshot; this.#shared = options.shared ?? {}; this.#clock = options.ungated ? null : (options.now ?? Date.now); } static async open(store: SiteGraphStore, snapshotId: string): Promise { const snapshot = await store.getSnapshot(snapshotId); if (!snapshot) throw new Error(`snapshot ${snapshotId} not found`); return new SnapshotSource(store, snapshot); } get snapshot(): Snapshot { return this.#snapshot; } /** This snapshot as the public sees it at `now` (one instant for a whole request). */ at(now: number): SnapshotSource { return new SnapshotSource(this.#store, this.#snapshot, { now: () => now, shared: this.#shared, }); } /** This snapshot with gates ignored: previews show embargoed and expired nodes (§7.3). */ ungated(): SnapshotSource { return new SnapshotSource(this.#store, this.#snapshot, { shared: this.#shared, ungated: true, }); } /** True when this view ignores gates (a preview). */ get gatesIgnored(): boolean { return this.#clock === null; } /** * The next instant (ms) at which anything this snapshot shows changes because of a gate, after * this view's `now`; undefined when no gate lies ahead (or gates are ignored). The Site Worker * caps the edge cache with it (§7.4). */ async nextGate(): Promise { if (this.#clock === null) return undefined; const { gates } = await this.#load(); return gates.length === 0 ? undefined : nextInstant(gates, this.#clock()); } /** Reads the snapshot's nodes once; kinds this renderer does not know are ignored. */ #load(): Promise { this.#shared.loaded ??= (async () => { // RFC-0010: one round of shard reads when the snapshot has a bundle; node by node otherwise. const bundle = await this.#store.readSnapshotBundle(this.#snapshot).catch(() => undefined); this.#shared.bundle = bundle ? "present" : "missing"; const fetched = bundle ? undefined : new Map(); const read = (refs: NodeRef[]) => Promise.all( refs.map(async (r) => { const hit = bundle?.get(r.hash); if (hit !== undefined) return hit as T & Site; const body = await this.#store.readSnapshotNode(r); // as stored, before anything downstream could touch the object fetched?.set(r.hash, JSON.stringify(body)); return body; }), ); const siteRef = this.#snapshot.nodes.find((n) => n.kind === "site"); if (!siteRef) throw new Error(`snapshot ${this.#snapshot.id} has no site node`); const [site] = await read([siteRef]); const pages = (await read( this.#snapshot.nodes.filter((n) => n.kind === "page"), )) as unknown as Page[]; const entries = (await read( this.#snapshot.nodes.filter((n) => n.kind === "contentEntry"), )) as unknown as ContentEntry[]; const byKind = async (kind: string) => (await read( this.#snapshot.nodes.filter((n) => (n.kind as string) === kind), )) as unknown as T[]; const [taxonomies, terms, redirects, menus, assets] = await Promise.all([ byKind("taxonomy"), byKind("term"), byKind("redirect"), byKind("menu"), byKind("asset"), ]); const assetIndex = new Map(); for (const a of assets) { assetIndex.set(a.id, a); assetIndex.set(a.sha256.slice(0, 16), a); } const order = new Map((site as Site).pageIds.map((id, i) => [id, i])); const shownPages = visible(pages, false).sort( (a, b) => (order.get(a.id) ?? 1e9) - (order.get(b.id) ?? 1e9), ); const shownEntries = visible(entries, false); const derived = derivedGates(site as Site, shownEntries); if (fetched) this.#shared.fetched = fetched; return { site: site as Site, pages: shownPages, entries: shownEntries, published: shownEntries, reachable: shownEntries, current: siteConverts(site as Site) ? entries : [], derived, resolved: new Map(), taxonomies, terms, redirects, menus, assets, assetIndex, gates: gateIndex(site as Site, shownPages, shownEntries, derived), }; })(); this.#shared.loaded.catch(() => { this.#shared.loaded = undefined; }); return this.#shared.loaded; } /** * RFC-0010 read-through: when this snapshot loaded without a bundle, write one in the background * (`defer` keeps the task alive, e.g. the Worker's `waitUntil`). The resolver calls it for the * published snapshot only: previews and history never get a bundle. Never throws; a failed write * leaves the snapshot loading node by node, and the next request tries again. */ persistBundle(defer: (task: Promise) => void): void { if (this.#shared.bundle === "present" || this.#shared.bundle === "persisting") return; const task = this.#load() .then(async () => { if (this.#shared.bundle !== "missing") return; this.#shared.bundle = "persisting"; await this.#store.writeSnapshotBundle(this.#snapshot, this.#shared.fetched ?? new Map()); this.#shared.bundle = "present"; this.#shared.fetched = undefined; }) .catch(() => { if (this.#shared.bundle === "persisting") this.#shared.bundle = "missing"; }); defer(task); } #assetGates(loaded: Loaded): Promise> { this.#shared.assetGates ??= Promise.resolve().then(() => assetGates(loaded)); return this.#shared.assetGates; } /** * What this view shows: every published node when gates are ignored or the snapshot has none; * otherwise the nodes (and inherited assets) whose gates are open at this view's instant. */ #current(): Promise { const now = this.#clock === null ? null : this.#clock(); if (this.#view?.now === now) return this.#view.view; const view = (async (): Promise => { const loaded = await this.#load(); if (now === null || loaded.gates.length === 0) return loaded; const gates = await this.#assetGates(loaded); const hidden = new Set([...gates].filter(([, from]) => !(from <= now)).map(([id]) => id)); const assets = hidden.size === 0 ? loaded.assets : loaded.assets.filter((a) => !hidden.has(a.id)); const assetIndex = hidden.size === 0 ? loaded.assetIndex : new Map([...loaded.assetIndex].filter(([, a]) => !hidden.has(a.id))); const split = splitEntries(loaded.entries, now, loaded.derived); return { ...loaded, pages: loaded.pages.filter((p) => isAvailable(p, now)), entries: split.listed, reachable: split.reachable, assets, assetIndex, }; })(); this.#view = { now, view }; return view; } async site(): Promise { return (await this.#current()).site; } async pages(): Promise { return (await this.#current()).pages; } async pageByPath(path: string, locale?: string): Promise { const { site, pages } = await this.#current(); const loc = locale ?? site.defaultLocale; return pages.find((p) => p.path === path && p.locale === loc); } async entries(collection: string, locale?: string): Promise { const { site, entries } = await this.#current(); const loc = locale ?? site.defaultLocale; const out = entries.filter((e) => e.collection === collection && e.locale === loc); return rememberList(out.sort(byNewest), { source: this, locale: loc }); } async entryBySlug( collection: string, slug: string, locale?: string, ): Promise { const { site, reachable } = await this.#current(); const loc = locale ?? site.defaultLocale; return reachable.find( (e) => e.collection === collection && e.slug === slug && e.locale === loc, ); } async allEntries(locale?: string): Promise { const { site, entries } = await this.#current(); const loc = locale ?? site.defaultLocale; return entries.filter((e) => e.locale === loc).sort(byNewest); } /** Every entry of a locale with a public URL, unlisted ones included (RFC-0042 §3). */ async routableEntries(locale?: string): Promise { const { site, reachable } = await this.#current(); const loc = locale ?? site.defaultLocale; return reachable.filter((e) => e.locale === loc).sort(byNewest); } /** RFC-0042 §4: a collection by its list fields, `now` at this view's instant. */ async queryEntries(q: FieldEntryQuery): Promise { const view = await this.#current(); const loc = q.locale ?? view.site.defaultLocale; const shownHere = q.includeExpired ? view.reachable : view.entries; return evaluateEntryQuery({ site: view.site, candidates: shownHere.filter((e) => e.collection === q.collection && e.locale === loc), published: view.published, taxonomies: view.taxonomies, terms: view.terms, query: q, now: this.#clock?.() ?? Date.now(), cache: view.resolved, }); } async search(query: string, options: SourceSearchOptions = {}): Promise { const locale = options.locale ?? (await this.site()).defaultLocale; return this.#store.search(this.#snapshot.siteId, query, { ...options, locale, now: this.#clock?.() ?? Date.now(), snapshotId: this.#snapshot.id, }); } /** A published entry of this snapshot by id (any locale), if its gates are open. */ async entryById(id: string): Promise { return (await this.#current()).reachable.find((e) => e.id === id); } entryFields(entry: ContentEntry, options?: EntryFieldsOptions): Promise { return entryFields(this, entry, options); } async #references(): Promise { const loaded = await this.#load(); if (this.#shared.references) return this.#shared.references; const { site } = loaded; const maps = await conversionMapsOf(site, Object.keys(site.collections ?? {}), (c) => this.conversionEntries(c), ); this.#shared.references ??= new ReferenceIndex(site, loaded.published, maps); return this.#shared.references; } /** RFC-0045 §4: in memory, through this view's gates (`reachable`: unlisted targets resolve). */ async groupMembers(groupIds: readonly string[]): Promise> { const [index, view] = [await this.#references(), await this.#current()]; return membersIn(index.members(), groupIds, { listed: view.entries, reachable: view.reachable, }); } async referencedBy(groupId: string, q: ReferencedByQuery = {}): Promise { const [index, view] = [await this.#references(), await this.#current()]; return referrersIn(index.referrers(groupId), q, { listed: view.entries, reachable: view.reachable, }); } async conversionEntries(collection: string): Promise { const loaded = await this.#load(); this.#shared.conversions ??= new Map(); return cachedConversionMaps( this.#shared.conversions, loaded.site, collection, () => loaded.current, ); } async taxonomies(): Promise { return (await this.#current()).taxonomies; } async terms(key: string): Promise { const { site, taxonomies, terms } = await this.#current(); const tax = taxonomies.find((t) => t.key === key); if (!tax) return []; return terms .filter((t) => t.taxonomyId === tax.id && shown(t, false)) .sort(byPosition(site.defaultLocale)); } async termBySlug(key: string, slug: string): Promise { return (await this.terms(key)).find((t) => t.slug === slug); } async vocabulary(key?: string): Promise { const { site, taxonomies, terms } = await this.#current(); if (key === undefined) return [...terms].sort(byPosition(site.defaultLocale)); const tax = taxonomies.find((t) => t.key === key); if (!tax) return []; return terms.filter((t) => t.taxonomyId === tax.id).sort(byPosition(site.defaultLocale)); } async entriesByFilter( filter: string, options: { locale?: string; limit?: number } = {}, ): Promise { const { site, entries, taxonomies, terms } = await this.#current(); const loc = options.locale ?? site.defaultLocale; const f = compile(filter, taxonomies, terms); const out = entries.filter((e) => e.locale === loc && matches(f, e)).sort(byNewest); return rememberList(options.limit === undefined ? out : out.slice(0, options.limit), { source: this, locale: loc, }); } entriesByTerm( key: string, slug: string, options: { locale?: string; limit?: number; descendants?: boolean } = {}, ): Promise { return this.entriesByFilter(termFilter(key, slug, options.descendants === true), options); } async redirectFor(path: string, locale?: string): Promise { const { site, redirects } = await this.#current(); return matchRedirect(redirects, path, locale && redirectLocale(locale, site.defaultLocale)); } async redirects(): Promise { return (await this.#current()).redirects; } async menu(key: string): Promise { return (await this.#current()).menus.find((m) => m.key === key); } async navigation( key: string, locale?: string, options: NavigationOptions = {}, ): Promise { const view = await this.#current(); const menu = view.menus.find((m) => m.key === key); if (!menu) return undefined; // Only published pages and entries whose gates are open are here: anything else resolves as // missing, so its menu item is hidden (RFC-0003 §7.1, RFC-0008 §7.2). return navigationFor( menu, { site: view.site, pages: view.pages, entries: view.reachable, taxonomies: view.taxonomies, terms: view.terms, ...(options.collectionIndex ? { collectionIndex: options.collectionIndex } : {}), }, locale ?? view.site.defaultLocale, ); } async assets(): Promise { return (await this.#current()).assets; } async asset(id: string): Promise { return id.startsWith("asset_") ? (await this.#current()).assetIndex.get(id) : undefined; } async assetByPrefix(prefix16: string): Promise { return /^[0-9a-f]{16}$/.test(prefix16) ? (await this.#current()).assetIndex.get(prefix16) : undefined; } }