/** * Bidirectional sync between Markdown files and JSONL/SQLite */ import * as fs from "fs"; import * as path from "path"; import type Database from "better-sqlite3"; import { parseMarkdownFile, updateFrontmatterFile, writeMarkdownFile, } from "./markdown.js"; import type { CrossReference } from "./markdown.js"; import { getSpec, getSpecByFilePath, createSpec, updateSpec, } from "./operations/specs.js"; import { getIssue, createIssue, updateIssue } from "./operations/issues.js"; import { addRelationship } from "./operations/relationships.js"; import { getTags, setTags } from "./operations/tags.js"; import { listFeedback, updateFeedback } from "./operations/feedback.js"; import { relocateFeedbackAnchor } from "./operations/feedback-anchors.js"; import { exportToJSONL } from "./export.js"; import { generateSpecId, generateIssueId } from "./id-generator.js"; import type { Spec, Issue, IssueStatus } from "@sudocode-ai/types"; import { isValidIssueStatus } from "./validation.js"; export interface MarkdownSyncResult { success: boolean; action: "created" | "updated" | "no-change"; entityId: string; entityType: "spec" | "issue"; error?: string; } export interface SyncOptions { /** * Directory for JSONL output (default: .sudocode) */ outputDir?: string; /** * Whether to auto-export to JSONL after sync (default: true) */ autoExport?: boolean; /** * User performing the sync */ user?: string; /** * Auto-initialize missing frontmatter fields (default: true) * If true, generates missing IDs and provides default values * If false, rejects files with missing required fields */ autoInitialize?: boolean; /** * Whether to write back initialized frontmatter to the file (default: true) * Only applies when autoInitialize is true */ writeBackFrontmatter?: boolean; } /** * Remove internal fields that shouldn't be in markdown frontmatter * These fields are only for database tracking, not for the markdown files */ function cleanFrontmatterForMarkdown( data: Record ): Record { const { file_path, entity_type, ...clean } = data; return clean; } /** * Initialize missing frontmatter fields */ function initializeFrontmatter( db: Database.Database, data: Record, entityType: "spec" | "issue", mdPath: string, outputDir: string, _user: string ): Record { const now = new Date().toISOString(); const initialized = { ...data }; // Generate ID and UUID if missing if (!initialized.id) { const generated = entityType === "spec" ? generateSpecId(db, outputDir) : generateIssueId(db, outputDir); initialized.id = generated.id; initialized.uuid = generated.uuid; } // Extract title from content if missing if (!initialized.title) { const content = fs.readFileSync(mdPath, "utf8"); const titleMatch = content.match(/^#\s+(.+)$/m); initialized.title = titleMatch ? titleMatch[1] : path.basename(mdPath, ".md"); } // Set timestamps if (!initialized.created_at) { initialized.created_at = now; } if (!initialized.updated_at) { initialized.updated_at = now; } // Set type-specific defaults if (entityType === "spec") { if (!initialized.priority && initialized.priority !== 0) initialized.priority = 2; } else { // issue if (!initialized.status) initialized.status = "open"; if (!initialized.priority && initialized.priority !== 0) initialized.priority = 2; } return initialized; } /** * Sync a markdown file to JSONL and SQLite * Parses markdown, updates database, optionally exports to JSONL */ export async function syncMarkdownToJSONL( db: Database.Database, mdPath: string, options: SyncOptions = {} ): Promise { const { outputDir = ".sudocode", autoExport = true, user = "system", autoInitialize = true, writeBackFrontmatter = true, } = options; try { // Parse markdown file with database validation for cross-references const parsed = parseMarkdownFile(mdPath, db, outputDir); let { data, content, references } = parsed; // Determine entity type from frontmatter or file path const entityType = determineEntityType(data, mdPath); let entityId = data.id as string | undefined; const frontmatterId = entityId; // Keep track of ID from frontmatter for validation // Calculate relative file path from outputDir const relPath = path.relative(outputDir, mdPath); const filePath = relPath.startsWith("..") ? path.relative(process.cwd(), mdPath) : relPath; // For specs, determine which spec this file corresponds to // Support two scenarios: rename (ID stays, path changes) and path-based lookup (no ID) let existingByPath: any = null; let existingById: any = null; if (entityType === "spec") { // Always set the calculated file path in data (for create/update) data.file_path = filePath; // Check if a spec exists with this ID (from frontmatter) if (frontmatterId) { existingById = getSpec(db, frontmatterId); } // Check if a spec exists at this file path existingByPath = getSpecByFilePath(db, filePath); // Scenario 1: Spec exists with this ID if (existingById) { entityId = existingById.id; // Check if file was renamed (ID exists but different path) if (existingById.file_path !== filePath) { console.log( `[sync] File renamed: ${existingById.file_path} → ${filePath}` ); // Check if another spec already exists at new path (conflict) if (existingByPath && existingByPath.id !== existingById.id) { console.warn( `[sync] Warning: File path conflict! Spec ${existingByPath.id} already exists at ${filePath}. Keeping ${existingById.id} and updating path.` ); } } } // Scenario 2: No ID in frontmatter, but spec exists at this path else if (existingByPath) { entityId = existingByPath.id; data.id = entityId; } // Scenario 3: ID in frontmatter differs from spec at this path (user changed ID) else if (frontmatterId && existingByPath) { // This shouldn't happen due to earlier checks, but handle it const correctId = existingByPath.id; console.warn( `[sync] Warning: ID in frontmatter (${frontmatterId}) differs from existing spec ID (${correctId}) for ${filePath}. Using existing ID.` ); entityId = correctId; data.id = entityId; } } // Handle missing frontmatter if (!entityId) { if (autoInitialize) { // Auto-initialize missing fields data = initializeFrontmatter( db, data, entityType, mdPath, outputDir, user ); entityId = data.id as string; // Write back frontmatter if requested if (writeBackFrontmatter) { // Filter out internal fields that shouldn't be in markdown frontmatter const cleanData = cleanFrontmatterForMarkdown(data); updateFrontmatterFile(mdPath, cleanData); } } else { return { success: false, action: "no-change", entityId: "", entityType, error: "Missing id in frontmatter (auto-initialization disabled)", }; } } // Check if entity exists by ID const existing = entityType === "spec" ? getSpec(db, entityId) : getIssue(db, entityId); const isNew = !existing; // Get file modification time to use as updated_at timestamp // This ensures database timestamp matches when the file was actually modified const fileStat = fs.statSync(mdPath); const fileModTime = new Date(fileStat.mtimeMs).toISOString(); if (entityType === "spec") { await syncSpec( db, entityId, data, content, references, isNew, user, fileModTime ); } else { await syncIssue( db, entityId, data, content, references, isNew, user, fileModTime ); } // Auto-export to JSONL if enabled if (autoExport) { await exportToJSONL(db, { outputDir, }); } return { success: true, action: isNew ? "created" : "updated", entityId, entityType, }; } catch (error) { return { success: false, action: "no-change", entityId: "", entityType: "spec", error: error instanceof Error ? error.message : String(error), }; } } /** * Sync from JSONL/SQLite to markdown file * Writes full content (frontmatter + body) from database to markdown */ export async function syncJSONLToMarkdown( db: Database.Database, entityId: string, entityType: "spec" | "issue", mdPath: string ): Promise { try { // Get entity from database const entity = entityType === "spec" ? getSpec(db, entityId) : getIssue(db, entityId); if (!entity) { return { success: false, action: "no-change", entityId, entityType, error: `${entityType} not found: ${entityId}`, }; } // Get relationships and tags const { getOutgoingRelationships } = await import( "./operations/relationships.js" ); const relationships = getOutgoingRelationships(db, entityId, entityType); const tags = getTags(db, entityId, entityType); // Build frontmatter const frontmatter = entityToFrontmatter( entity, entityType, relationships, tags ); // Check if file exists (for action reporting) const fileExists = fs.existsSync(mdPath); // Write full content (frontmatter + body) from database // When this function is called, sync direction has already determined // that database is the source of truth, so we write everything const content = entity.content || ""; writeMarkdownFile(mdPath, frontmatter, content); return { success: true, action: fileExists ? "updated" : "created", entityId, entityType, }; } catch (error) { return { success: false, action: "no-change", entityId, entityType, error: error instanceof Error ? error.message : String(error), }; } } /** * Determine entity type from frontmatter or file path */ function determineEntityType( frontmatter: Record, filePath: string ): "spec" | "issue" { // Check frontmatter type field if (frontmatter.entity_type === "issue") { return "issue"; } if (frontmatter.entity_type === "spec") { return "spec"; } // Check file path if (filePath.includes("/issues/") || filePath.includes("/issue-")) { return "issue"; } if (filePath.includes("/specs/") || filePath.includes("/spec-")) { return "spec"; } // Default to spec return "spec"; } /** * Sync spec data to database */ async function syncSpec( db: Database.Database, id: string, frontmatter: Record, content: string, references: CrossReference[], isNew: boolean, user: string, fileModTime?: string ): Promise { // Get old content before updating (for anchor relocation) const oldSpec = isNew ? null : getSpec(db, id); const oldContent = oldSpec?.content || ""; const specData: Partial = { id, title: frontmatter.title || "Untitled", file_path: frontmatter.file_path || "", content, priority: frontmatter.priority ?? 2, parent_id: frontmatter.parent_id || undefined, }; if (isNew) { createSpec(db, { ...specData, updated_at: fileModTime, } as any); } else { // Check if content has actually changed const contentChanged = oldSpec?.title !== specData.title || oldSpec?.content !== specData.content || oldSpec?.priority !== specData.priority || oldSpec?.parent_id !== specData.parent_id || oldSpec?.file_path !== specData.file_path; // Only update timestamp if content actually changed // Note: external_links is handled separately via JSONL import, not markdown sync updateSpec(db, id, { ...specData, ...(contentChanged && fileModTime ? { updated_at: fileModTime } : {}), } as any); // Relocate feedback anchors if content changed if (oldContent !== content) { const feedbackList = listFeedback(db, { to_id: id }); if (feedbackList.length > 0) { // Relocate each feedback anchor for (const feedback of feedbackList) { const oldAnchor = typeof feedback.anchor === "string" ? JSON.parse(feedback.anchor) : feedback.anchor; // Skip feedback without anchors - they should be preserved unchanged if (!oldAnchor) { continue; } // Relocate the anchor const newAnchor = relocateFeedbackAnchor( oldContent, content, oldAnchor ); // Update feedback with new anchor (newAnchor is never null here since oldAnchor was checked) if (newAnchor) { updateFeedback(db, feedback.id, { anchor: newAnchor, }); } } } } } // Sync tags if (frontmatter.tags && Array.isArray(frontmatter.tags)) { setTags(db, id, "spec", frontmatter.tags); } // Sync relationships from cross-references await syncRelationships( db, id, "spec", references, frontmatter.relationships, user ); } /** * Sync issue data to database */ async function syncIssue( db: Database.Database, id: string, frontmatter: Record, content: string, references: CrossReference[], isNew: boolean, user: string, fileModTime?: string ): Promise { const oldIssue = isNew ? null : getIssue(db, id); // Validate and sanitize status from frontmatter let status: IssueStatus = "open"; // default if (frontmatter.status) { if (isValidIssueStatus(frontmatter.status)) { status = frontmatter.status as IssueStatus; } else { console.warn( `Warning: Invalid status "${frontmatter.status}" in ${id}, defaulting to "open"` ); } } const issueData: Partial = { id, title: frontmatter.title || "Untitled", content, status, priority: frontmatter.priority ?? 2, assignee: frontmatter.assignee || undefined, parent_id: frontmatter.parent_id || undefined, }; if (isNew) { createIssue(db, { ...issueData, updated_at: fileModTime, } as any); } else { // Check if content has actually changed const contentChanged = oldIssue?.title !== issueData.title || oldIssue?.content !== issueData.content || oldIssue?.status !== issueData.status || oldIssue?.priority !== issueData.priority || oldIssue?.assignee !== issueData.assignee || oldIssue?.parent_id !== issueData.parent_id; // Only update timestamp if content actually changed // Note: external_links is handled separately via JSONL import, not markdown sync updateIssue(db, id, { ...issueData, ...(contentChanged && fileModTime ? { updated_at: fileModTime } : {}), } as any); } // Sync tags if (frontmatter.tags && Array.isArray(frontmatter.tags)) { setTags(db, id, "issue", frontmatter.tags); } // Sync relationships from cross-references await syncRelationships( db, id, "issue", references, frontmatter.relationships, user ); } /** * Sync relationships from cross-references and frontmatter * Preserves existing relationships not in frontmatter */ async function syncRelationships( db: Database.Database, entityId: string, entityType: "spec" | "issue", references: CrossReference[], frontmatterRels?: Array<{ // Support both formats: // 1. DB format from entityToFrontmatter: { from_id, from_type, to_id, to_type, relationship_type } // 2. Simplified format: { target_id, target_type, relationship_type } target_id?: string; target_type?: string; to_id?: string; to_type?: string; relationship_type: string; }>, _user: string = "system" ): Promise { // Get existing relationships (all outgoing) const { getOutgoingRelationships } = await import( "./operations/relationships.js" ); const existing = getOutgoingRelationships(db, entityId, entityType); const existingSet = new Set( existing.map((r) => `${r.relationship_type}:${r.to_type}:${r.to_id}`) ); // Add relationships from cross-references (defaults to 'references' type) for (const ref of references) { const relType = ref.relationshipType || "references"; const key = `${relType}:${ref.type}:${ref.id}`; if (!existingSet.has(key)) { try { addRelationship(db, { from_id: entityId, from_type: entityType, to_id: ref.id, to_type: ref.type, relationship_type: relType as any, metadata: ref.anchor ? JSON.stringify({ anchor: ref.anchor }) : undefined, }); } catch (error) { // Ignore errors (e.g., target not found, duplicate) } } } // Add relationships from frontmatter // Handle both DB format (to_id/to_type) and simplified format (target_id/target_type) if (frontmatterRels && Array.isArray(frontmatterRels)) { for (const rel of frontmatterRels) { // Support both formats const toId = rel.to_id || rel.target_id; const toType = rel.to_type || rel.target_type; if (!toId || !toType) { // Skip invalid relationships continue; } const key = `${rel.relationship_type}:${toType}:${toId}`; if (!existingSet.has(key)) { try { addRelationship(db, { from_id: entityId, from_type: entityType, to_id: toId, to_type: toType as "spec" | "issue", relationship_type: rel.relationship_type as any, }); } catch (error) { // Ignore errors } } } } // Note: We preserve existing relationships not in frontmatter // This allows relationships added via CLI/UI to persist } /** * Convert entity to frontmatter object */ function entityToFrontmatter( entity: Spec | Issue, entityType: "spec" | "issue", relationships: Array, tags: string[] ): Record { const base: Record = { id: entity.id, title: entity.title, priority: entity.priority, created_at: entity.created_at, }; // Only add optional fields if they have values if (entity.parent_id) base.parent_id = entity.parent_id; if (tags.length > 0) base.tags = tags; if (relationships.length > 0) base.relationships = relationships; if (entityType === "spec") { return base; } else { const issue = entity as Issue; const result: Record = { ...base, status: issue.status, }; // Only add optional issue fields if they have values if (issue.assignee) result.assignee = issue.assignee; if (issue.closed_at) result.closed_at = issue.closed_at; return result; } }