import type { Attribute, Match, Option, Variant, MessageImport, VariantImport, Bundle, Pattern, Declaration, Expression, VariableReference, Message, } from "@inlang/sdk"; import type { plugin } from "../plugin.js"; import { flattenMessageKeys, matchEntries } from "../utils/messageKeys.js"; import { orderSelectors } from "../utils/orderSelectors.js"; import { orderVariants } from "../utils/orderVariants.js"; import type { ComplexMessage, ComplexMessageObject, SimpleMessage, } from "../fileSchema.js"; export const importFiles: NonNullable<(typeof plugin)["importFiles"]> = async ({ files, settings, sdkCapabilities, }) => { const bundles: Bundle[] = []; const bundlesById = new Map(); const messages: MessageImport[] = []; const variants: VariantImport[] = []; // inputs that only plain strings declare, by bundle const plainInputs = new Map>(); const declaredInputs = new Map>(); // the variants of each message, `variants.slice(start, end)` const ranges: Array<{ message: MessageImport; start: number; end: number }> = []; for (const file of files) { const json = JSON.parse(new TextDecoder().decode(file.content)); // A flat string catalog already has exactly the flattened iteration order. // Keep nested/legacy shapes on the full flattener. const entries = json !== null && typeof json === "object" && !Array.isArray(json) ? Object.entries(json) : undefined; const flattened = entries?.every(([, value]) => typeof value === "string") ? entries : flattenMessageKeys(json); for (const [key, value] of flattened) { if (key === "$schema") { continue; } const result = parseBundle( key, file.locale, value as SimpleMessage | ComplexMessage ); if ( sdkCapabilities?.includes("bundle-description") && file.locale === settings.baseLocale ) { const descriptions = typeof value === "string" ? [] : (value as ComplexMessage).flatMap((entry) => { if (entry.description === undefined) return []; if (typeof entry.description !== "string") throw new Error(`Invalid description for "${key}"`); return [entry.description]; }); if (new Set(descriptions).size > 1) throw new Error(`Conflicting descriptions for "${key}"`); result.bundle.description = descriptions[0] ?? null; } messages.push(result.message); const start = variants.length; variants.push(...result.variants); if (result.variants.length > 1) { ranges.push({ message: result.message, start, end: variants.length }); } const inputs = typeof value === "string" ? plainInputs : declaredInputs; for (const declaration of result.bundle.declarations) { if (declaration.type !== "input-variable") continue; let names = inputs.get(key); if (!names) inputs.set(key, (names = new Set())); names.add(declaration.name); } const existingBundle = bundlesById.get(result.bundle.id); if (existingBundle === undefined) { bundles.push(result.bundle); bundlesById.set(result.bundle.id, result.bundle); } else { if (result.bundle.description !== undefined) { if ( existingBundle.description !== undefined && existingBundle.description !== result.bundle.description ) throw new Error(`Conflicting descriptions for "${key}"`); existingBundle.description = result.bundle.description; } // merge declarations without duplicates existingBundle.declarations = mergeDeclarations( existingBundle.declarations, result.bundle.declarations ); } } } // A plain string reads its placeholders as inputs. If another locale // declares a local of that name, e.g. `local formattedAmount = amount: // number`, the placeholder is that local, not an input. for (const bundle of bundles) { if ( !bundle.declarations.some( (declaration) => declaration.type === "local-variable" ) ) continue; const locals = new Set( bundle.declarations .filter((declaration) => declaration.type === "local-variable") .map((declaration) => declaration.name) ); bundle.declarations = bundle.declarations.filter( (declaration) => declaration.type !== "input-variable" || locals.has(declaration.name) === false || plainInputs.get(bundle.id)?.has(declaration.name) !== true || declaredInputs.get(bundle.id)?.has(declaration.name) === true ); } // The variants in the order export writes them, which is the order in // which runtimes like Paraglide JS 2.26 try them. Files that earlier // versions wrote with a variant that is never selected, e.g. sorted with // the catch-all first, select as the message says. Each message keeps its // place in `variants`. for (const { message, start, end } of ranges) { if (end - start < 2) continue; const declarations = bundlesById.get(message.bundleId)!.declarations; const ordered = orderVariants( variants.slice(start, end) as Array< VariantImport & Pick >, orderSelectors( (message.selectors ?? []).map((selector) => selector.name).sort(), declarations ), declarations, message.locale ); variants.splice(start, ordered.length, ...ordered); } return { bundles, messages, variants }; }; function parseBundle( key: string, locale: string, value: SimpleMessage | ComplexMessage ): { bundle: Bundle; message: MessageImport; variants: VariantImport[]; } { if (typeof value === "string") { const parsed = parsePattern(value); // Plain patterns only infer input declarations. Preserve first occurrence // order without serializing fresh parser nodes through JSON. const names = parsed.declarations.length < 2 ? undefined : new Set(); const declarations = parsed.declarations.length < 2 ? parsed.declarations : parsed.declarations.filter((declaration) => { if (names!.has(declaration.name)) return false; names!.add(declaration.name); return true; }); return { bundle: { id: key, declarations }, message: { bundleId: key, selectors: [], locale }, variants: [ { messageBundleId: key, messageLocale: locale, matches: [], pattern: parsed.pattern, }, ], }; } const parsed = parseVariants(key, locale, value); const declarations = unique(parsed.declarations); addInputDeclarationsForLocalReferences(declarations); const selectorNames = orderSelectors( unique(parsed.selectors).map((selector) => selector.name), declarations ); const selectors: VariableReference[] = selectorNames.map((name) => ({ type: "variable-reference", name, })); const undeclaredSelectors = selectors.filter( (selector) => declarations.find((d) => d.name === selector.name) === undefined ); for (const undeclaredSelector of undeclaredSelectors) { declarations.push({ type: "input-variable", name: undeclaredSelector.name, }); } return { bundle: { id: key, declarations, }, message: { bundleId: key, selectors, locale: locale, }, variants: parsed.variants, }; } /** * Local declarations may reference inputs without repeating them as explicit * `input` declarations. Keep the SDK bundle declarations complete so * consumers can generate the correct message function signature. */ function addInputDeclarationsForLocalReferences( declarations: Declaration[] ): void { const declaredNames = new Set( declarations.map((declaration) => declaration.name) ); for (const declaration of declarations) { if (declaration.type !== "local-variable") { continue; } const references: string[] = []; if (declaration.value.arg.type === "variable-reference") { references.push(declaration.value.arg.name); } for (const option of declaration.value.annotation?.options ?? []) { if (option.value.type === "variable-reference") { references.push(option.value.name); } } for (const name of references) { if (declaredNames.has(name)) { continue; } declarations.push({ type: "input-variable", name }); declaredNames.add(name); } } } function parseVariants( bundleId: string, locale: string, value: SimpleMessage | ComplexMessage ): { variants: VariantImport[]; declarations: Declaration[]; selectors: VariableReference[]; } { // single variant if (typeof value === "string") { const parsed = parsePattern(value); return { variants: [ { messageBundleId: bundleId, messageLocale: locale, matches: [], pattern: parsed.pattern, }, ], // legacy reasons that input variables are derived from the pattern declarations: parsed.declarations, selectors: [], }; } const complexMessage = Array.isArray(value) ? value[0] : undefined; if (typeof complexMessage !== "object" || complexMessage === null) { throw new Error( `The message "${bundleId}" (${locale}) is neither a string nor a complex message (an array with an object of declarations, selectors and match): ${truncate(String(JSON.stringify(value)), 100)}` ); } // multi variant const variants: VariantImport[] = []; const selectors: VariableReference[] = ( (complexMessage["selectors"] as string[]) ?? [] ).map((name) => ({ type: "variable-reference", name, })); const declarations = new Set(); for (const declaration of complexMessage["declarations"] ?? ([] as string[])) { declarations.add(parseDeclaration(declaration)); } const detectedSelectors = new Set(); for (const [match, pattern] of matchEntries(complexMessage["match"])) { const parsed = parsePattern(pattern as string); const parsedMatches = parseMatches(match); for (const declaration of parsed.declarations) { let isDuplicate = false; for (const existingDeclaration of declarations) { if (existingDeclaration.name === declaration.name) { isDuplicate = true; break; } } if (isDuplicate) { continue; } declarations.add(declaration); } for (const selector of parsedMatches.selectors) { detectedSelectors.add(selector); } variants.push({ messageBundleId: bundleId, messageLocale: locale, matches: parsedMatches.matches, pattern: parsed.pattern, }); } // A key of the match that the file doesn't list as a selector is one, as // it always was. Except if the file lists the selectors and the key only // matches `*`: it selects nothing then, and listing it would change the // file on export. Earlier versions wrote a plain string next to a plural // in another locale as `"selectors": [], "match": { "count=*": … }`. const listsSelectors = Array.isArray(complexMessage["selectors"]); const selectsSomething = (name: string) => variants.some((variant) => (variant.matches ?? []).some( (match) => match.key === name && match.type === "literal-match" ) ); return { variants, declarations: Array.from(declarations), selectors: unique([ ...selectors, ...Array.from(detectedSelectors).filter( (selector) => listsSelectors === false || selectors.some((listed) => listed.name === selector.name) || selectsSomething(selector.name) ), ]), }; } function parsePattern(value: string): { declarations: Declaration[]; pattern: Pattern; } { // Most messages are plain text. Keep the original string rather than // building it character by character when there is nothing to parse. if ( typeof value === "string" && !value.includes("{") && !value.includes("\\") ) { return { declarations: [], pattern: value.length === 0 ? [] : [{ type: "text", value }], }; } const simple = parseSimplePattern(value); if (simple) return simple; const pattern: Variant["pattern"] = []; const declarations: Declaration[] = []; let buffer = ""; // Collects literal text (including escaped braces/backslashes) until we hit // an expression boundary or the end. const flushBuffer = () => { if (buffer.length > 0) { pattern.push({ type: "text", value: buffer }); buffer = ""; } }; for (let index = 0; index < value.length; index += 1) { const char = value[index]; if (char === "\\") { const next = value[index + 1]; if (next === "{" || next === "}" || next === "\\") { buffer += next; index += 1; continue; } buffer += char; continue; } if (char === "{") { const closingIndex = findPlaceholderClosingIndex(value, index); if (closingIndex === -1) { buffer += char; continue; } const placeholder = value.slice(index + 1, closingIndex); const markupNode = parseMarkupPlaceholder(placeholder); flushBuffer(); if (markupNode) { for (const option of markupNode.options ?? []) { if (option.value.type === "variable-reference") { declarations.push({ type: "input-variable", name: option.value.name, }); } } pattern.push(markupNode); index = closingIndex; continue; } const expression = parseExpressionPlaceholder(placeholder) ?? { type: "expression", arg: { type: "variable-reference", name: placeholder }, }; // this is a heuristic. there is no guarentee that the variable might not be // a local variable. only use the returned declarations in a single variant // context for (const name of expressionReferences(expression)) { declarations.push({ type: "input-variable", name }); } pattern.push(expression); index = closingIndex; continue; } buffer += char; } flushBuffer(); return { declarations, pattern, }; } /** * Scan ordinary input placeholders in chunks. Escapes, annotations, markup, * quotes, and malformed/nested braces retain the full parser's semantics. */ function parseSimplePattern( value: string ): { declarations: Declaration[]; pattern: Pattern } | undefined { if (typeof value !== "string" || value.includes("\\")) return undefined; const pattern: Pattern = []; const declarations: Declaration[] = []; let cursor = 0; while (cursor < value.length) { const opening = value.indexOf("{", cursor); if (opening === -1) { pattern.push({ type: "text", value: value.slice(cursor) }); break; } const closing = value.indexOf("}", opening + 1); if (closing === -1) return undefined; const name = value.slice(opening + 1, closing); if ( name.includes("{") || name.includes("|") || name.includes(":") || name.startsWith("#") || name.startsWith("/") ) return undefined; if (opening > cursor) pattern.push({ type: "text", value: value.slice(cursor, opening) }); pattern.push({ type: "expression", arg: { type: "variable-reference", name }, }); declarations.push({ type: "input-variable", name }); cursor = closing + 1; } return { declarations, pattern }; } /** * Parses an annotated expression `name: function option=value …`, like a local * declaration, e.g. `count: icu:pound offset=1`. Option values are * `$variable`, `|quoted literal|` or a literal without whitespace. * * Returns undefined for anything else, which stays a plain `{name}` * placeholder as before. */ function parseExpressionPlaceholder( placeholder: string ): Expression | undefined { // `name: function`, with whitespace after the colon as export writes it. // Without it, `{user:name}` stays a variable named `user:name` as before. const match = placeholder.match(/^([^\s:|{}]+):\s+([^\s=|{}]+)(.*)$/s); if (match === null) return undefined; const [, name, functionName, rest] = match as unknown as [ string, string, string, string, ]; const options: Option[] = []; let index = 0; while (index < rest.length) { const afterWhitespace = skipWhitespace(rest, index); if (afterWhitespace === rest.length) break; // options are separated by whitespace if (afterWhitespace === index) return undefined; index = afterWhitespace; const optionName = rest.slice(index).match(/^[^\s=|{}$]+/)?.[0]; if (optionName === undefined || rest[index + optionName.length] !== "=") { return undefined; } index += optionName.length + 1; const value = parseExpressionOptionValue(rest, index); if (value === undefined) return undefined; options.push({ name: optionName, value: value.value }); index = value.nextIndex; } return { type: "expression", arg: { type: "variable-reference", name }, annotation: { type: "function-reference", name: functionName, options }, }; } function parseExpressionOptionValue( value: string, index: number ): { value: Option["value"]; nextIndex: number } | undefined { if (value[index] === "|") { // same quoting as markup options return parseMarkupValue(value, index); } const token = value.slice(index).match(/^[^\s|]+/)?.[0]; if (token === undefined) return undefined; const nextIndex = index + token.length; if (token.startsWith("$")) { if (token.length === 1) return undefined; return { value: { type: "variable-reference", name: token.slice(1) }, nextIndex, }; } return { value: { type: "literal", value: token }, nextIndex }; } /** * The variables an expression reads: its argument and variable options. */ function expressionReferences(expression: Expression): string[] { const names: string[] = []; if (expression.arg.type === "variable-reference") { names.push(expression.arg.name); } for (const option of expression.annotation?.options ?? []) { if (option.value.type === "variable-reference") { names.push(option.value.name); } } return names; } function findPlaceholderClosingIndex( value: string, openingIndex: number ): number { let inQuotedLiteral = false; for (let cursor = openingIndex + 1; cursor < value.length; cursor += 1) { const current = value[cursor]!; if (inQuotedLiteral && current === "\\") { cursor += 1; continue; } if (current === "|") { inQuotedLiteral = !inQuotedLiteral; continue; } if (current === "}" && inQuotedLiteral === false) { return cursor; } } return -1; } function parseMarkupPlaceholder(placeholder: string): | { type: "markup-start" | "markup-end" | "markup-standalone"; name: string; options?: Option[]; attributes?: Attribute[]; } | undefined { if (placeholder.startsWith("#")) { const parsed = parseMarkupBody(placeholder.slice(1), true); if (!parsed) { throw new Error(`Invalid markup placeholder: {${placeholder}}`); } const { name, options, attributes, standalone } = parsed; return { type: standalone ? "markup-standalone" : "markup-start", name, ...(options.length > 0 ? { options } : {}), ...(attributes.length > 0 ? { attributes } : {}), }; } if (placeholder.startsWith("/")) { const parsed = parseMarkupBody(placeholder.slice(1), false); if (!parsed) { throw new Error(`Invalid markup placeholder: {${placeholder}}`); } const { name, options, attributes } = parsed; return { type: "markup-end", name, ...(options.length > 0 ? { options } : {}), ...(attributes.length > 0 ? { attributes } : {}), }; } return undefined; } function parseMarkupBody( body: string, allowStandalone: boolean ): | { name: string; options: Option[]; attributes: Attribute[]; standalone: boolean; } | undefined { let index = 0; const options: Option[] = []; const attributes: Attribute[] = []; let standalone = false; index = skipWhitespace(body, index); const nameToken = readNameToken(body, index); if (!nameToken) return undefined; const name = nameToken.value; index = nameToken.nextIndex; while (index < body.length) { index = skipWhitespace(body, index); if (index >= body.length) break; if (allowStandalone && body[index] === "/") { const trailing = body.slice(index + 1).trim(); if (trailing.length > 0) return undefined; standalone = true; index = body.length; break; } if (body[index] === "@") { index += 1; const attributeName = readIdentifier(body, index); if (!attributeName) return undefined; index = attributeName.nextIndex; index = skipWhitespace(body, index); if (body[index] === "=") { index += 1; index = skipWhitespace(body, index); const attributeValue = parseMarkupValue(body, index); if (!attributeValue) return undefined; if (attributeValue.value.type === "variable-reference") { // Attributes only support literal values. return undefined; } attributes.push({ name: attributeName.value, value: attributeValue.value, }); index = attributeValue.nextIndex; } else { attributes.push({ name: attributeName.value, value: true }); } continue; } const optionName = readIdentifier(body, index); if (!optionName) return undefined; index = optionName.nextIndex; index = skipWhitespace(body, index); if (body[index] !== "=") return undefined; index += 1; index = skipWhitespace(body, index); const optionValue = parseMarkupValue(body, index); if (!optionValue) return undefined; options.push({ name: optionName.value, value: optionValue.value, }); index = optionValue.nextIndex; } return { name, options, attributes, standalone }; } function skipWhitespace(value: string, index: number): number { while (index < value.length && /\s/.test(value[index]!)) { index += 1; } return index; } function readNameToken( value: string, index: number ): { value: string; nextIndex: number } | undefined { return readIdentifier(value, index); } function readIdentifier( value: string, index: number ): { value: string; nextIndex: number } | undefined { let cursor = index; while ( cursor < value.length && /\s/.test(value[cursor]!) === false && value[cursor] !== "=" && value[cursor] !== "/" && value[cursor] !== "@" ) { cursor += 1; } const parsed = value.slice(index, cursor); if (parsed.length === 0) return undefined; return { value: parsed, nextIndex: cursor }; } function parseMarkupValue( value: string, index: number ): { value: Option["value"]; nextIndex: number } | undefined { if (index >= value.length) return undefined; if (value[index] === "|") { let cursor = index + 1; let literal = ""; while (cursor < value.length) { const char = value[cursor]!; if (char === "\\") { const next = value[cursor + 1]; if (next === "|" || next === "\\" || next === "}") { literal += next; cursor += 2; continue; } literal += char; cursor += 1; continue; } if (char === "|") { return { value: { type: "literal", value: literal }, nextIndex: cursor + 1, }; } literal += char; cursor += 1; } return undefined; } if (value[index] === "$") { const variable = readIdentifier(value, index + 1); if (!variable) return undefined; return { value: { type: "variable-reference", name: variable.value }, nextIndex: variable.nextIndex, }; } const literal = readIdentifier(value, index); if (!literal) return undefined; return { value: { type: "literal", value: literal.value }, nextIndex: literal.nextIndex, }; } // input: `platform=android,userGender=male` // output: { platform: "android", userGender: "male" } function parseMatches(value: string): { matches: Match[]; selectors: Message["selectors"]; } { const matches: Match[] = []; const selectors: Message["selectors"] = []; const parts = value.split(","); for (const part of parts) { const [key, value] = part.split("=").map((segment) => segment.trim()); if (!key || !value) { continue; } if (value === "*") { matches.push({ type: "catchall-match", key: key, }); } else { matches.push({ type: "literal-match", key, value, }); } selectors.push({ type: "variable-reference", name: key, }); } return { matches, selectors }; } function mergeDeclarations( existing: Declaration[], incoming: Declaration[] ): Declaration[] { if (incoming.length === 0) return existing; if (existing.length === 0) return incoming; if ( existing.length === 1 && incoming.length === 1 && existing[0]!.type === "input-variable" && incoming[0]!.type === "input-variable" && existing[0]!.name === incoming[0]!.name ) return existing; const declarations = [...existing, ...incoming]; if ( declarations.every((declaration) => declaration.type === "input-variable") ) { const names = new Set(); return declarations.filter((declaration) => { if (names.has(declaration.name)) return false; names.add(declaration.name); return true; }); } return unique(declarations); } function unique(items: T[]): T[] { const keys = new Set(); return items.filter((item) => { const key = JSON.stringify(item); if (keys.has(key)) return false; keys.add(key); return true; }); } function parseDeclaration(value: string): Declaration { if (value.startsWith("input")) { return { type: "input-variable", name: value.slice(6).trim(), }; } // local countPlural = count: plural // local countPluralExact = count (un-annotated alias, e.g. ICU `=0`) // local greeting = "hello" else if (value.startsWith("local")) { const match = value.match( /^local\s+([^\s=:]+)\s*=\s*(?:"((?:[^"\\]|\\.)*)"|([^\s=:"]+))\s*(?::\s*([^\s=:]+)(.*))?$/ ); if (match === null) { throw new Error(`Unsupported local declaration: "${value}"`); } const [, name, literal, ref, fn, optionsString] = match; const options: { name: string; value: | { type: "literal"; value: string } | { type: "variable-reference"; name: string }; }[] = []; for (const optionMatch of (optionsString ?? "").matchAll( /(\w+)\s*=\s*([^\s]+)/g )) { const optionName = optionMatch[1]; const optionValue = optionMatch[2]; if (!optionName || !optionValue) { continue; } options.push({ name: optionName, value: optionValue.startsWith("$") && optionValue.length > 1 ? { type: "variable-reference", name: optionValue.slice(1), } : { type: "literal", value: optionValue, }, }); } return { type: "local-variable", name: name!, value: { type: "expression", arg: literal !== undefined ? { type: "literal", value: literal.replace(/\\(.)/g, "$1") } : { type: "variable-reference", name: ref! }, // an un-annotated local is an alias of its argument. omit the // annotation instead of storing `undefined` so the declaration // equals the one other plugins and editors create, e.g. // `.local countPluralExact = {$count}` for an ICU `=0`. ...(fn ? { annotation: { type: "function-reference" as const, name: fn.trim(), options, }, } : {}), }, }; } throw new Error("Unsupported declaration type"); } function truncate(text: string, length: number): string { return text.length > length ? `${text.slice(0, length)}...` : text; }