import { logger } from '../../console/logger.js'; import { logErrorAndExit } from '../../console/logging.js'; import { fileEncodingSkipReason, lottieExpressionsError, xcstringsSourceLanguageMismatchError, } from '../../console/index.js'; import { recordWarning } from '../../state/translateWarnings.js'; import { lottieHasExpressions } from './detectLottieExpressions.js'; import { getRelative, readFile, readBinaryFileBase64, } from '../../fs/findFilepath.js'; import { readFileContent } from '../../fs/fileContent.js'; import { Settings } from '../../types/index.js'; import type { DataFormat, FileToUpload } from '../../types/data.js'; import { SUPPORTED_FILE_EXTENSIONS } from './supportedFiles.js'; import { parseJson } from '../json/parseJson.js'; import { parseXcstringsCatalog, serializeXcstringsSlice, sliceSourceCatalog, type XcstringsCatalog, } from '../xcstrings/parseXcstrings.js'; import { resolveMintlifyRefs, shouldResolveRefs, } from '../../utils/resolveMintlifyRefs.js'; import { storeRefMap } from '../../state/mintlifyRefMap.js'; import parseYaml from '../yaml/parseYaml.js'; import { validateYamlSchema } from '../yaml/utils.js'; import { flattenJson } from '../json/flattenJson.js'; import type { JSONObject } from '../../types/data/json.js'; import YAML from 'yaml'; import { determineLibrary } from '../../fs/determineFramework/index.js'; import { hashStringSync, hashVersionId } from '../../utils/hash.js'; import { api } from '../../utils/api.js'; import { preprocessContent } from './preprocessContent.js'; import { parseKeyedMetadata, type KeyedMetadata, } from '../parseKeyedMetadata.js'; import { buildPublishMap } from '../../utils/resolvePublish.js'; import { CONFIG_FILE_TYPE_TO_FILE_FORMAT, getTransformFormatProperty, } from './transformFormat.js'; /** * Checks if a file path is a metadata companion file (e.g. foo.metadata.json) * AND its corresponding source file (e.g. foo.json) exists in the file list. * If both conditions are true, the metadata file should be skipped as a translation source. */ function isCompanionMetadataFile( filePath: string, allFilePaths: string[] ): boolean { const metadataPattern = /\.metadata\.(json|yaml|yml)$/; if (!metadataPattern.test(filePath)) return false; // Derive the source file path: foo.metadata.json -> foo.json const sourceFilePath = filePath.replace( /\.metadata\.(json|yaml|yml)$/, '.$1' ); return allFilePaths.includes(sourceFilePath); } /** * Warns and records a skipped file when its content does not parse as JSON or * YAML, so the caller can leave the file out of the upload. */ export function isUnparsableFile( content: string, relativePath: string, format: 'JSON' | 'YAML' ): boolean { try { if (format === 'JSON') JSON.parse(content); else YAML.parse(content); return false; } catch { const reason = `${format} file is not parsable`; logger.warn(`Skipping ${relativePath}: ${reason}`); recordWarning('skipped_file', relativePath, reason); return true; } } export async function aggregateFiles( settings: Settings ): Promise<{ files: FileToUpload[]; publishMap: Map }> { // Aggregate all files to translate const files: FileToUpload[] = []; if ( !settings.files || (Object.keys(settings.files.placeholderPaths).length === 1 && settings.files.placeholderPaths.gt) ) { return { files, publishMap: new Map() }; } const { resolvedPaths: filePaths } = settings.files; // Tolerate partially-constructed settings from programmatic callers const requiresReviewPaths = settings.files.requiresReviewPaths ?? new Set(); const skipValidation = settings.options?.skipFileValidation; // Build publish map upfront from resolved paths. const publishMap = buildPublishMap(filePaths, settings); // Process JSON files if (filePaths.json) { const { library, additionalModules } = determineLibrary(); // Determine dataFormat for JSONs let dataFormat: DataFormat; if (library === 'next-intl') { dataFormat = 'ICU'; } else if (library === 'i18next') { if (additionalModules.includes('i18next-icu')) { dataFormat = 'ICU'; } else { dataFormat = 'I18NEXT'; } } else { dataFormat = 'STRING'; } const jsonFiles = filePaths.json .filter((filePath) => !isCompanionMetadataFile(filePath, filePaths.json!)) .map((filePath) => { const content = readFile(filePath); const relativePath = getRelative(filePath); if ( !skipValidation?.json && isUnparsableFile(content, relativePath, 'JSON') ) { return null; } // Resolve $ref before parsing if configured let contentForParsing = content; if (shouldResolveRefs(filePath, settings.options)) { try { const json = JSON.parse(content); const { resolved, refMap } = resolveMintlifyRefs(json, filePath); storeRefMap(refMap); contentForParsing = JSON.stringify(resolved, null, 2); } catch { // JSON parse errors are handled below by parseJson } } const parsedJson = parseJson( contentForParsing, filePath, settings.options || {}, settings.defaultLocale ); // Detect companion metadata file let keyedMetadata: KeyedMetadata | undefined; let parsedContent: JSONObject | undefined; try { parsedContent = JSON.parse(content) as JSONObject; } catch { // Content not parsable — skip metadata detection } if (parsedContent) { const rawMetadata = parseKeyedMetadata(filePath, parsedContent); if (rawMetadata) { // Run metadata through the same include/composite schema as the source // so key paths align at translation time const transformed = parseJson( JSON.stringify(rawMetadata), filePath, settings.options || {}, settings.defaultLocale, false ); const transformedMetadata = JSON.parse(transformed); // Filter metadata to only keep keys that exist in the transformed source // This prevents misaligned entries from wide JSONPath patterns const sourceKeys = new Set(Object.keys(JSON.parse(parsedJson))); const filtered = Object.fromEntries( Object.entries(transformedMetadata).filter(([k]) => sourceKeys.has(k) ) ) as KeyedMetadata; if (Object.keys(filtered).length > 0) { keyedMetadata = filtered; } else { logger.warn( `Companion metadata found for ${relativePath} but no keys aligned with the JSON schema — metadata was not attached` ); } } } return { fileId: hashStringSync(relativePath), versionId: hashVersionId( parsedJson, requiresReviewPaths.has(filePath) ), content: parsedJson, fileName: relativePath, fileFormat: 'JSON' as const, ...getTransformFormatProperty(settings, 'json'), dataFormat, locale: settings.defaultLocale, ...(keyedMetadata && { formatMetadata: { keyedMetadata }, }), } satisfies FileToUpload; }) .filter((file) => { if (!file) return false; if (typeof file.content !== 'string' || !file.content.trim()) { logger.warn(`Skipping ${file.fileName}: JSON file is empty`); recordWarning('skipped_file', file.fileName, 'JSON file is empty'); return false; } return true; }); files.push(...jsonFiles.filter((file) => file !== null)); } // Process YAML files if (filePaths.yaml) { const yamlFiles = filePaths.yaml .filter((filePath) => !isCompanionMetadataFile(filePath, filePaths.yaml!)) .map((filePath) => { const content = readFile(filePath); const relativePath = getRelative(filePath); if ( !skipValidation?.yaml && isUnparsableFile(content, relativePath, 'YAML') ) { return null; } const { content: parsedYaml, fileFormat } = parseYaml( content, filePath, settings.options || {} ); // Detect companion metadata file let keyedMetadata: KeyedMetadata | undefined; try { const parsedYamlContent = YAML.parse(content); const rawMetadata = parseKeyedMetadata(filePath, parsedYamlContent); if (rawMetadata) { const yamlSchema = validateYamlSchema( settings.options || {}, filePath ); if (yamlSchema?.include) { // Flatten metadata through the same include schema as the source const flattened = flattenJson(rawMetadata, yamlSchema.include); // Filter to only keep keys that exist in the transformed source const sourceKeys = new Set(Object.keys(JSON.parse(parsedYaml))); const filtered = Object.fromEntries( Object.entries(flattened).filter(([k]) => sourceKeys.has(k)) ) as KeyedMetadata; if (Object.keys(filtered).length > 0) { keyedMetadata = filtered; } else { logger.warn( `Companion metadata found for ${relativePath} but no keys aligned with the YAML schema — metadata was not attached` ); } } else { keyedMetadata = rawMetadata; } } } catch { // Content not parsable as YAML — skip metadata detection } return { content: parsedYaml, fileName: relativePath, fileFormat, ...getTransformFormatProperty(settings, 'yaml'), fileId: hashStringSync(relativePath), versionId: hashVersionId( parsedYaml, requiresReviewPaths.has(filePath) ), locale: settings.defaultLocale, ...(keyedMetadata && { formatMetadata: { keyedMetadata }, }), } satisfies FileToUpload; }) .filter((file) => { if (!file || typeof file.content !== 'string' || !file.content.trim()) { logger.warn( `Skipping ${file?.fileName ?? 'unknown'}: YAML file is empty` ); recordWarning( 'skipped_file', file?.fileName ?? 'unknown', 'YAML file is empty' ); return false; } return true; }); files.push(...yamlFiles.filter((file) => file !== null)); } // Process Twilio Content JSON files if (filePaths.twilioContentJson) { const twilioContentJsonFiles = filePaths.twilioContentJson .map((filePath) => { const content = readFile(filePath); const relativePath = getRelative(filePath); if ( !skipValidation?.json && isUnparsableFile(content, relativePath, 'JSON') ) { return null; } const parsedJson = parseJson( content, filePath, settings.options || {}, settings.defaultLocale ); return { fileId: hashStringSync(relativePath), versionId: hashVersionId( parsedJson, requiresReviewPaths.has(filePath) ), content: parsedJson, fileName: relativePath, fileFormat: 'TWILIO_CONTENT_JSON' as const, ...getTransformFormatProperty(settings, 'twilioContentJson'), dataFormat: 'STRING' as const, locale: settings.defaultLocale, } satisfies FileToUpload; }) .filter((file) => { if (!file) return false; if (typeof file.content !== 'string' || !file.content.trim()) { logger.warn(`Skipping ${file.fileName}: JSON file is empty`); recordWarning('skipped_file', file.fileName, 'JSON file is empty'); return false; } return true; }); files.push(...twilioContentJsonFiles.filter((file) => file !== null)); } // Process Lottie files (binary zip bundles). Content is read as raw bytes and // carried base64-encoded end-to-end; the downloaded translation is written // back as bytes. No text parsing/merge. if (filePaths.lottie) { // Lottie files carrying After Effects expressions (executable JS) are // rejected outright — collect every offender, then fail once with all of // them named rather than aborting on the first. const expressionOffenders: string[] = []; const lottieFiles = filePaths.lottie .map((filePath) => { const content = readBinaryFileBase64(filePath); const relativePath = getRelative(filePath); if (!content) { logger.warn(`Skipping ${relativePath}: file is empty or unreadable`); recordWarning( 'skipped_file', relativePath, 'File is empty or unreadable' ); return null; } if (lottieHasExpressions(content)) { expressionOffenders.push(relativePath); return null; } return { content, fileName: relativePath, fileFormat: 'LOTTIE' as const, ...getTransformFormatProperty(settings, 'lottie'), fileId: hashStringSync(relativePath), versionId: hashStringSync(content), locale: settings.defaultLocale, } satisfies FileToUpload; }) .filter((file) => file !== null); if (expressionOffenders.length > 0) { logErrorAndExit(lottieExpressionsError(expressionOffenders)); } files.push(...lottieFiles); } // These formats are uploaded verbatim. Their backslash escapes, format // specifiers, timing lines, entity escapes and line breaks must survive // byte-for-byte, so they skip the generic markdown-oriented preprocessing // below. for (const [fileType, fileFormat] of [ ['dotStrings', 'DOT_STRINGS'], ['dotStringsdict', 'DOT_STRINGSDICT'], ['androidStrings', 'ANDROID_STRINGS'], ['srt', 'SRT'], ['resx', 'RESX'], ] as const) { if (!filePaths[fileType]) continue; const verbatimFiles = filePaths[fileType] .map((filePath) => { const relativePath = getRelative(filePath); // Reading is the one place an encoding is resolved; from here on the // content is a UTF-8 string like every other format's. let content: string; try { content = readFileContent(filePath, fileFormat); } catch (error) { // One unreadable file should not abort the whole run: skip it and // report it in the summary const reason = fileEncodingSkipReason(error); logger.warn(`Skipping ${relativePath}: ${reason}`); recordWarning('skipped_file', relativePath, reason); return null; } return { content, fileName: relativePath, fileFormat, ...getTransformFormatProperty(settings, fileType), fileId: hashStringSync(relativePath), versionId: hashVersionId(content, requiresReviewPaths.has(filePath)), locale: settings.defaultLocale, } satisfies FileToUpload; }) .filter((file) => { if (!file) return false; if (!file.content.trim()) { logger.warn(`Skipping ${file.fileName}: File is empty`); recordWarning('skipped_file', file.fileName, 'File is empty'); return false; } return true; }); files.push(...verbatimFiles.filter((file) => file !== null)); } // Process Apple .xcstrings catalogs. One catalog holds every locale; only the // source-language slice is uploaded as the source document. The slice is // hashed into versionId, so an unchanged catalog re-slices byte-identically // and does not re-upload. if (filePaths.xcstrings) { const sourceLanguageMismatches: { file: string; sourceLanguage: string }[] = []; const xcstringsFiles = filePaths.xcstrings .map((filePath) => { const content = readFile(filePath); const relativePath = getRelative(filePath); let catalog: XcstringsCatalog; try { catalog = parseXcstringsCatalog(content); } catch (error) { const reason = error instanceof Error ? error.message : 'xcstrings file is not parsable'; logger.warn(`Skipping ${relativePath}: ${reason}`); recordWarning('skipped_file', relativePath, reason); return null; } // Slicing follows the catalog's own sourceLanguage while the upload is // labeled settings.defaultLocale; a mismatch would upload mislabeled // source content, so it is a configuration error that stops the run // rather than a skipped file. The configured locale may be a custom // alias, so canonical forms are compared. if ( api.resolveCanonicalLocale(catalog.sourceLanguage) !== api.resolveCanonicalLocale(settings.defaultLocale) ) { sourceLanguageMismatches.push({ file: relativePath, sourceLanguage: catalog.sourceLanguage, }); return null; } const sourceSlice = serializeXcstringsSlice( sliceSourceCatalog(catalog) ); return { content: sourceSlice, fileName: relativePath, fileFormat: 'XCSTRINGS' as const, ...getTransformFormatProperty(settings, 'xcstrings'), fileId: hashStringSync(relativePath), versionId: hashVersionId( sourceSlice, requiresReviewPaths.has(filePath) ), locale: settings.defaultLocale, } satisfies FileToUpload; }) .filter((file) => file !== null); if (sourceLanguageMismatches.length > 0) { logErrorAndExit( xcstringsSourceLanguageMismatchError( sourceLanguageMismatches, settings.defaultLocale ) ); } files.push(...xcstringsFiles); } for (const fileType of SUPPORTED_FILE_EXTENSIONS) { if ( fileType === 'json' || fileType === 'yaml' || fileType === 'twilioContentJson' || fileType === 'lottie' || fileType === 'dotStrings' || fileType === 'dotStringsdict' || fileType === 'androidStrings' || fileType === 'xcstrings' || fileType === 'srt' || fileType === 'resx' ) continue; if (filePaths[fileType]) { const parsed = filePaths[fileType] .map((filePath) => { const content = readFile(filePath); const relativePath = getRelative(filePath); const processed = preprocessContent( content, relativePath, fileType, settings ); if (typeof processed !== 'string') { logger.warn(`Skipping ${relativePath}: ${processed.skip}`); recordWarning('skipped_file', relativePath, processed.skip); return null; } return { content: processed, fileName: relativePath, fileFormat: CONFIG_FILE_TYPE_TO_FILE_FORMAT[fileType], ...getTransformFormatProperty(settings, fileType), fileId: hashStringSync(relativePath), versionId: hashVersionId( processed, requiresReviewPaths.has(filePath) ), locale: settings.defaultLocale, } satisfies FileToUpload; }) .filter((file) => { if ( !file || typeof file.content !== 'string' || !file.content.trim() ) { logger.warn( `Skipping ${file?.fileName ?? 'unknown'}: File is empty after sanitization` ); recordWarning( 'skipped_file', file?.fileName ?? 'unknown', 'File is empty after sanitization' ); return false; } return true; }); files.push(...parsed.filter((file) => file !== null)); } } // Remove stale entries for files that were skipped during validation const validFileIds = new Set(files.map((f) => f.fileId)); for (const fileId of publishMap.keys()) { if (!validFileIds.has(fileId)) { publishMap.delete(fileId); } } return { files, publishMap }; }