/** * Memory operations and diagnostic command factories. */ import * as p from "@clack/prompts"; import { ContextOnlyReplayError, type CreatedMemory, compareMemoryRoleReports, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS, type ExtractionBenchmarkScore, type ExtractionModelCostEstimate, type ExtractionReplayObserverIdentity, type ExtractionReplayResult, type ExtractionStructuralDiagnostics, estimateExtractionModelCost, extractionReplayObserverIdentity, getExtractionBenchmarkProfile, getExtractionModelPricing, getInjectionEvalScenarioPack, getInjectionEvalScenarioPrompts, getMemoryArtifactReport, getMemoryRoleReport, getRawEventRelinkPlan, getRawEventRelinkReport, getSessionExtractionEval, getSessionExtractionEvalScenario, loadObserverConfig, MemoryStore, ObserverClient, ObserverOutputError, type ObserverOutputMode, ObserverOutputTransportError, type ObserverTokenUsage, observerBaseUrlForProviderOverride, replayBatchExtraction, replayBatchExtractionWithTierRouting, resolveDbPath, resolveProject, scoreExtractionBenchmarkOutput, } from "@codemem/core"; import { Command } from "commander"; import { helpStyle } from "../help-style.js"; import { addDbOption, addJsonOption, type DbOpts, emitJsonError, type JsonOpts, resolveDbOpt, } from "../shared-options.js"; import { createDistillCommand } from "./distill.js"; import { createExportMemoriesCommand } from "./export-memories.js"; import { createImportMemoriesCommand } from "./import-memories.js"; import { createContextCommand, createPackCommand } from "./pack.js"; import { createRecentCommand } from "./recent.js"; import { createSearchCommand } from "./search.js"; import { createStatsCommand } from "./stats.js"; /** Parse a strict positive integer, rejecting prefixes like "12abc". */ function parseStrictPositiveId(value: string): number | null { if (!/^\d+$/.test(value.trim())) return null; const n = Number(value.trim()); return Number.isFinite(n) && n >= 1 && Number.isInteger(n) ? n : null; } interface ContextOnlyReplayOutcome { status: "context_only"; code: "delegated_brief_context_only"; evaluated: false; batchId: number; scenarioId: string; message: string; } function contextOnlyReplayOutcome( error: ContextOnlyReplayError, target: { batchId: number; scenarioId: string }, ): ContextOnlyReplayOutcome { return { ...target, status: "context_only", code: error.code, evaluated: false, message: error.message, }; } function emitContextOnlyReplayOutcome(outcome: ContextOnlyReplayOutcome, opts: JsonOpts): void { if (opts.json) console.log(JSON.stringify(outcome, null, 2)); else p.log.info(outcome.message); process.exitCode = 0; } export function resolveOpenAIResponsesOverride( cliEnabled: boolean | undefined, configured: boolean | undefined, ): boolean | undefined { return cliEnabled === true ? true : configured; } function showMemoryAction(idStr: string, opts: DbOpts & JsonOpts): void { const memoryId = parseStrictPositiveId(idStr); if (memoryId === null) { if (opts.json) { emitJsonError("invalid_id", `Invalid memory ID: ${idStr}`); } else { p.log.error(`Invalid memory ID: ${idStr}`); process.exitCode = 1; } return; } const store = new MemoryStore(resolveDbPath(resolveDbOpt(opts))); try { const item = store.get(memoryId); if (!item) { if (opts.json) { emitJsonError("not_found", `Memory ${memoryId} not found`); } else { p.log.error(`Memory ${memoryId} not found`); process.exitCode = 1; } return; } if (opts.json) { console.log(JSON.stringify(item, null, 2)); } else { // Human-readable format console.log(`#${item.id} [${item.kind}] ${item.title}`); if (item.subtitle) console.log(` ${item.subtitle}`); console.log(` created: ${item.created_at} confidence: ${item.confidence}`); if (item.tags_text) console.log(` tags: ${item.tags_text}`); if (item.body_text) { const preview = item.body_text.length > 300 ? `${item.body_text.slice(0, 300)}…` : item.body_text; console.log(`\n${preview}`); } } } finally { store.close(); } } function forgetMemoryAction(idStr: string, opts: DbOpts & JsonOpts): void { const memoryId = parseStrictPositiveId(idStr); if (memoryId === null) { if (opts.json) { emitJsonError("invalid_id", `Invalid memory ID: ${idStr}`); } else { p.log.error(`Invalid memory ID: ${idStr}`); process.exitCode = 1; } return; } const store = new MemoryStore(resolveDbPath(resolveDbOpt(opts))); try { if (!store.forgetForUser(memoryId)) { if (opts.json) { emitJsonError("not_found", `Memory ${memoryId} not found`); } else { p.log.error(`Memory ${memoryId} not found`); process.exitCode = 1; } return; } if (opts.json) { console.log(JSON.stringify({ id: memoryId, status: "forgotten" })); } else { p.log.success(`Memory ${memoryId} marked inactive`); } } finally { store.close(); } } interface RememberMemoryOptions extends DbOpts, JsonOpts { kind: string; title: string; body: string; tags?: string[]; project?: string; confidence?: string; } function parseConfidence(value: string | undefined): number { const n = value == null || value.trim() === "" ? 0.5 : Number(value); if (!Number.isFinite(n)) return 0.5; return Math.min(1, Math.max(0, n)); } function createManualMemory(store: MemoryStore, opts: RememberMemoryOptions) { const project = resolveProject(process.cwd(), opts.project ?? null); const createdMemories: CreatedMemory[] = []; return store.db .transaction(() => { const newSessionId = store.startSession({ cwd: process.cwd(), project, user: process.env.USER ?? "unknown", toolVersion: "manual", metadata: { manual: true }, }); const memId = store.rememberForUser( newSessionId, opts.kind, opts.title, opts.body, parseConfidence(opts.confidence), opts.tags, undefined, { createdMemories }, ); const item = store.get(memId); if (!item) throw new Error("unauthorized_scope"); store.endSession(newSessionId, { manual: true }); return { memId, sessionId: newSessionId, createdMemories }; }) .immediate(); } async function rememberMemoryAction(opts: RememberMemoryOptions): Promise { const store = new MemoryStore(resolveDbPath(resolveDbOpt(opts))); let sessionId: number | null = null; try { const result = createManualMemory(store, opts); sessionId = result.sessionId; const memId = result.memId; store.enqueueCommittedVectorWrites(result.createdMemories); await store.flushPendingVectorWrites(); if (opts.json) { console.log(JSON.stringify({ id: memId })); } else { p.log.success(`Stored memory ${memId}`); } } catch (err) { if (sessionId !== null) { try { store.endSession(sessionId, { manual: true, error: true }); } catch { // ignore — already in error path } } const message = err instanceof Error ? err.message : String(err); if (opts.json) { emitJsonError("remember_failed", message); } else { p.log.error(`Failed to store memory: ${message}`); process.exitCode = 1; } } finally { store.close(); } } function createShowMemoryCommand(): Command { const cmd = new Command("show") .configureHelp(helpStyle) .description("Show a memory") .argument("", "memory ID"); addDbOption(cmd); addJsonOption(cmd); cmd.action(showMemoryAction); return cmd; } function createForgetMemoryCommand(): Command { const cmd = new Command("forget") .configureHelp(helpStyle) .description("Deactivate a memory") .argument("", "memory ID"); addDbOption(cmd); addJsonOption(cmd); cmd.action(forgetMemoryAction); return cmd; } function createRememberMemoryCommand(): Command { const cmd = new Command("remember") .configureHelp(helpStyle) .description("Store a memory") .requiredOption("-k, --kind ", "memory kind (discovery, decision, feature, bugfix, etc.)") .requiredOption("-t, --title ", "memory title") .requiredOption("-b, --body <body>", "memory body text") .option("--tags <tags...>", "tags (space-separated)") .option("--project <project>", "project name (defaults to git repo root)") .option("--confidence <n>", "confidence 0-1", "0.5"); addDbOption(cmd); addJsonOption(cmd); cmd.action(rememberMemoryAction); return cmd; } type MemoryRoleReport = ReturnType<typeof getMemoryRoleReport>; type MemoryRoleComparison = ReturnType<typeof compareMemoryRoleReports>; interface MemoryRoleOptions extends JsonOpts { project?: string; allProjects?: boolean; probe?: string[]; scenario?: string[]; inactive?: boolean; } function resolveMemoryRoleReportInput(opts: MemoryRoleOptions) { const project = opts.allProjects === true ? null : opts.project?.trim() || process.env.CODEMEM_PROJECT?.trim() || resolveProject(process.cwd(), null); const invalidScenario = (opts.scenario ?? []).find( (id) => getInjectionEvalScenarioPack(id) == null, ); if (invalidScenario) { throw new Error(`Unknown eval scenario pack: ${invalidScenario}`); } const probes = [...(opts.probe ?? []), ...getInjectionEvalScenarioPrompts(opts.scenario ?? [])]; return { project, allProjects: opts.allProjects === true, includeInactive: opts.inactive === true, probes, }; } function renderMemoryRoleSummary(result: MemoryRoleReport): void { p.intro("codemem diagnostics memory roles"); p.log.info( [ `Memories: ${result.totals.memories}`, `Active: ${result.totals.active}`, `Sessions: ${result.totals.sessions}`, ].join("\n"), ); p.log.info("Counts by role:"); for (const [role, count] of Object.entries(result.counts_by_role)) { p.log.message(` ${role.padEnd(10)} ${String(count)}`); } p.log.info("Counts by mapping:"); p.log.message(` mapped ${result.counts_by_mapping.mapped}`); p.log.message(` unmapped ${result.counts_by_mapping.unmapped}`); p.log.info("Summary lineages:"); p.log.message(` session_summary ${result.summary_lineages.session_summary}`); p.log.message(` legacy_metadata_summary ${result.summary_lineages.legacy_metadata_summary}`); p.log.message(` summary_mapped ${result.summary_mapping.mapped}`); p.log.message(` summary_unmapped ${result.summary_mapping.unmapped}`); p.log.info("Project quality:"); for (const [bucket, count] of Object.entries(result.project_quality)) { p.log.message(` ${bucket.padEnd(12)} ${String(count)}`); } p.log.info("Session classes:"); for (const [bucket, count] of Object.entries(result.session_class_buckets)) { p.log.message(` ${bucket.padEnd(20)} ${String(count)}`); } p.log.info("Summary dispositions:"); for (const [bucket, count] of Object.entries(result.summary_disposition_buckets)) { p.log.message(` ${bucket.padEnd(20)} ${String(count)}`); } } function renderMemoryRoleProbe(probe: MemoryRoleReport["probe_results"][number]): void { p.log.message(` query: ${probe.query}`); if (probe.scenario_id) { p.log.message( ` scenario: ${probe.scenario_id} (${probe.scenario_category ?? "unknown"})${probe.scenario_title ? ` — ${probe.scenario_title}` : ""}`, ); } p.log.message(` mode: ${probe.mode}`); p.log.message( ` top roles: durable=${probe.top_role_counts.durable} recap=${probe.top_role_counts.recap} ephemeral=${probe.top_role_counts.ephemeral} general=${probe.top_role_counts.general}`, ); p.log.message( ` top mapping: mapped=${probe.top_mapping_counts.mapped} unmapped=${probe.top_mapping_counts.unmapped}`, ); p.log.message( ` burden: recap_share=${probe.top_burden.recap_share.toFixed(2)} unmapped_share=${probe.top_burden.unmapped_share.toFixed(2)} recap_unmapped_share=${probe.top_burden.recap_unmapped_share.toFixed(2)}`, ); if (probe.simulated_demoted_unmapped_recap) { p.log.message( ` simulated demote-unmapped-recap burden: recap_share=${probe.simulated_demoted_unmapped_recap.top_burden.recap_share.toFixed(2)} unmapped_share=${probe.simulated_demoted_unmapped_recap.top_burden.unmapped_share.toFixed(2)} recap_unmapped_share=${probe.simulated_demoted_unmapped_recap.top_burden.recap_unmapped_share.toFixed(2)}`, ); } if (probe.simulated_demoted_unmapped_recap_and_ephemeral) { p.log.message( ` simulated demote-unmapped-recap+ephemeral burden: recap_share=${probe.simulated_demoted_unmapped_recap_and_ephemeral.top_burden.recap_share.toFixed(2)} unmapped_share=${probe.simulated_demoted_unmapped_recap_and_ephemeral.top_burden.unmapped_share.toFixed(2)} recap_unmapped_share=${probe.simulated_demoted_unmapped_recap_and_ephemeral.top_burden.recap_unmapped_share.toFixed(2)}`, ); } if (probe.scenario_score) { p.log.message( ` scenario score: mode_match=${probe.scenario_score.mode_match ? "yes" : "no"} top1_primary=${probe.scenario_score.primary_in_top1 ? "yes" : "no"} top3_primary=${probe.scenario_score.primary_in_top3_count} top1_anti=${probe.scenario_score.anti_signal_in_top1 ? "yes" : "no"} primary=${probe.scenario_score.primary_match_count} anti=${probe.scenario_score.anti_signal_count} recap=${probe.scenario_score.recap_count} unmapped_recap=${probe.scenario_score.unmapped_recap_count} chatter=${probe.scenario_score.administrative_chatter_count} net=${probe.scenario_score.score}`, ); } for (const item of probe.items.slice(0, 5)) { p.log.message( ` [${item.id}] (${item.kind}/${item.role}/${item.mapping}/${item.session_class}/${item.summary_disposition}) ${item.title} — ${item.role_reason}`, ); } } function renderMemoryRoleReport(result: MemoryRoleReport): void { renderMemoryRoleSummary(result); if (result.probe_results.length > 0) { p.log.info("Probe results:"); for (const probe of result.probe_results) { renderMemoryRoleProbe(probe); } } p.outro("done"); } function createMemoryRoleReportCommand(): Command { const cmd = new Command("role-report") .configureHelp(helpStyle) .description("Report memory roles") .option("--project <project>", "project name or path (defaults to git repo root)") .option("--all-projects", "analyze across all projects") .option( "--probe <query>", "run a retrieval probe query against the snapshot", (value, prev: string[]) => [...prev, value], [], ) .option( "--scenario <id>", "run a named injection-first eval scenario pack (can be repeated)", (value, prev: string[]) => [...prev, value], [], ) .option("--inactive", "include inactive memories"); addDbOption(cmd); addJsonOption(cmd); cmd.action((opts: DbOpts & MemoryRoleOptions) => { try { const input = resolveMemoryRoleReportInput(opts); const result = getMemoryRoleReport(resolveDbOpt(opts), input); if (opts.json) { console.log(JSON.stringify(result, null, 2)); return; } renderMemoryRoleReport(result); } catch (error) { const message = error instanceof Error ? error.message : "Role report failed"; if (opts.json) { emitJsonError("role_report_failed", message); } else { p.log.error(message); process.exitCode = 1; } return; } }); return cmd; } function renderMemoryRoleComparisonSummary(result: MemoryRoleComparison): void { p.intro("codemem diagnostics memory compare"); p.log.info( [ `Baseline sessions: ${result.baseline.totals.sessions}`, `Candidate sessions: ${result.candidate.totals.sessions}`, `Delta sessions: ${result.delta.totals.sessions}`, `Mapped delta: ${result.delta.counts_by_mapping.mapped}`, `Unmapped delta: ${result.delta.counts_by_mapping.unmapped}`, `Summary mapped delta: ${result.delta.summary_mapping.mapped}`, `Summary unmapped delta: ${result.delta.summary_mapping.unmapped}`, ].join("\n"), ); p.log.info("Role deltas:"); for (const [role, count] of Object.entries(result.delta.counts_by_role)) { p.log.message(` ${role.padEnd(10)} ${String(count)}`); } p.log.info("Session class deltas:"); for (const [bucket, count] of Object.entries(result.delta.session_class_buckets)) { p.log.message(` ${bucket.padEnd(20)} ${String(count)}`); } p.log.info("Summary disposition deltas:"); for (const [bucket, count] of Object.entries(result.delta.summary_disposition_buckets)) { p.log.message(` ${bucket.padEnd(20)} ${String(count)}`); } } function renderMemoryRoleScenarioDelta( delta: NonNullable<MemoryRoleComparison["probe_comparisons"][number]["delta_scenario_score"]>, ): void { p.log.message( ` scenario delta: mode_match=${delta.mode_match ?? "-"} top1_primary=${delta.primary_in_top1 ?? "-"} top3_primary=${delta.primary_in_top3_count ?? "-"} top1_anti=${delta.anti_signal_in_top1 ?? "-"} primary=${delta.primary_match_count ?? "-"} anti=${delta.anti_signal_count ?? "-"} recap=${delta.recap_count ?? "-"} unmapped_recap=${delta.unmapped_recap_count ?? "-"} chatter=${delta.administrative_chatter_count ?? "-"} net=${delta.score ?? "-"}`, ); } function renderMemoryRoleProbeComparison( probe: MemoryRoleComparison["probe_comparisons"][number], ): void { p.log.message(` query: ${probe.query}`); p.log.message( ` modes: baseline=${probe.baseline_mode ?? "-"} candidate=${probe.candidate_mode ?? "-"}`, ); p.log.message( ` overlap: shared_top_keys=${probe.shared_item_keys.length} baseline_top=${probe.baseline_item_ids.slice(0, 5).join(",") || "-"} candidate_top=${probe.candidate_item_ids.slice(0, 5).join(",") || "-"}`, ); if (probe.delta_top_burden) { p.log.message( ` burden delta: recap_share=${probe.delta_top_burden.recap_share.toFixed(2)} unmapped_share=${probe.delta_top_burden.unmapped_share.toFixed(2)} recap_unmapped_share=${probe.delta_top_burden.recap_unmapped_share.toFixed(2)}`, ); } if (probe.delta_top_mapping_counts) { p.log.message( ` mapping delta: mapped=${probe.delta_top_mapping_counts.mapped} unmapped=${probe.delta_top_mapping_counts.unmapped}`, ); } if (probe.baseline_scenario_score || probe.candidate_scenario_score) { p.log.message( ` scenario scores: baseline=${probe.baseline_scenario_score?.score ?? "-"} candidate=${probe.candidate_scenario_score?.score ?? "-"}`, ); } if (probe.delta_scenario_score) { renderMemoryRoleScenarioDelta(probe.delta_scenario_score); } } function renderMemoryRoleComparison(result: MemoryRoleComparison): void { renderMemoryRoleComparisonSummary(result); if (result.probe_comparisons.length > 0) { p.log.info("Probe comparisons:"); for (const probe of result.probe_comparisons) { renderMemoryRoleProbeComparison(probe); } } p.outro("done"); } function createMemoryRoleCompareCommand(): Command { const cmd = new Command("role-compare") .configureHelp(helpStyle) .description("Compare memory quality across databases") .argument("<baseline_db>", "baseline sqlite database path") .argument("<candidate_db>", "candidate sqlite database path") .option("--project <project>", "project name or path (defaults to git repo root)") .option("--all-projects", "analyze across all projects") .option( "--probe <query>", "run a retrieval probe query against both snapshots", (value, prev: string[]) => [...prev, value], [], ) .option( "--scenario <id>", "run a named injection-first eval scenario pack (can be repeated)", (value, prev: string[]) => [...prev, value], [], ) .option("--inactive", "include inactive memories"); addJsonOption(cmd); cmd.action((baselineDb: string, candidateDb: string, opts: MemoryRoleOptions) => { try { const input = resolveMemoryRoleReportInput(opts); const result = compareMemoryRoleReports(baselineDb, candidateDb, input); if (opts.json) { console.log(JSON.stringify(result, null, 2)); return; } renderMemoryRoleComparison(result); } catch (error) { const message = error instanceof Error ? error.message : "Role compare failed"; if (opts.json) { emitJsonError("role_compare_failed", message); } else { p.log.error(message); process.exitCode = 1; } return; } }); return cmd; } function renderMemoryArtifactReport(result: ReturnType<typeof getMemoryArtifactReport>): void { p.intro("codemem diagnostics memory artifacts"); p.log.info( [ `Memories: ${result.totals.memories}`, `Active: ${result.totals.active}`, `Sessions: ${result.totals.sessions}`, ].join("\n"), ); p.log.info("Counts by artifact:"); for (const [artifact, count] of Object.entries(result.counts_by_artifact)) { p.log.message(` ${artifact.padEnd(16)} ${String(count)}`); } p.log.info("Counts by action:"); for (const [action, count] of Object.entries(result.counts_by_action)) { p.log.message(` ${action.padEnd(14)} ${String(count)}`); } const topReasons = Object.entries(result.counts_by_reason) .sort(([, a], [, b]) => b - a) .slice(0, 10); if (topReasons.length > 0) { p.log.info("Top reasons:"); for (const [reason, count] of topReasons) { p.log.message(` ${reason.padEnd(36)} ${String(count)}`); } } p.log.info(`High-confidence telemetry: ${result.high_confidence_telemetry.total}`); p.outro("done"); } function createMemoryArtifactReportCommand(): Command { const cmd = new Command("artifact-report") .configureHelp(helpStyle) .description("Report legacy memory artifacts without changing data") .option("--project <project>", "project name or path (defaults to git repo root)") .option("--all-projects", "analyze across all projects") .option("--inactive", "include inactive memories"); addDbOption(cmd); addJsonOption(cmd); cmd.action( ( opts: DbOpts & JsonOpts & { project?: string; allProjects?: boolean; inactive?: boolean; }, ) => { try { const project = opts.allProjects === true ? null : opts.project?.trim() || process.env.CODEMEM_PROJECT?.trim() || resolveProject(process.cwd(), null); const result = getMemoryArtifactReport(resolveDbOpt(opts), { project, allProjects: opts.allProjects === true, includeInactive: opts.inactive === true, }); if (opts.json) { console.log(JSON.stringify(result, null, 2)); return; } renderMemoryArtifactReport(result); } catch (error) { const message = error instanceof Error ? error.message : "Artifact report failed"; if (opts.json) { emitJsonError("artifact_report_failed", message); } else { p.log.error(message); process.exitCode = 1; } return; } }, ); return cmd; } function createMemoryExtractionReportCommand(): Command { const cmd = new Command("extraction-report") .configureHelp(helpStyle) .description("Score extracted memories against a scenario") .option("--session-id <id>", "session ID to evaluate") .option("--batch-id <id>", "raw-event flush batch ID to evaluate") .requiredOption("--scenario <id>", "built-in extraction eval scenario ID") .option("--inactive", "include inactive memories"); addDbOption(cmd); addJsonOption(cmd); cmd.action( ( opts: DbOpts & JsonOpts & { sessionId: string; batchId?: string; scenario: string; inactive?: boolean; }, ) => { try { const sessionIdInput = opts.sessionId?.trim() ?? ""; const batchIdInput = opts.batchId?.trim() ?? ""; const hasSessionId = sessionIdInput.length > 0; const hasBatchId = batchIdInput.length > 0; if (hasSessionId === hasBatchId) { throw new Error("Provide exactly one of --session-id or --batch-id"); } const sessionId = hasSessionId ? parseStrictPositiveId(sessionIdInput) : null; if (hasSessionId && sessionId === null) { throw new Error(`Invalid session ID: ${sessionIdInput || opts.sessionId}`); } const batchId = hasBatchId ? parseStrictPositiveId(batchIdInput) : null; if (hasBatchId && batchId === null) { throw new Error(`Invalid batch ID: ${batchIdInput || opts.batchId}`); } const scenarioId = opts.scenario?.trim() ?? ""; const scenario = getSessionExtractionEvalScenario(scenarioId); if (!scenario) { throw new Error(`Unknown extraction eval scenario: ${scenarioId || opts.scenario}`); } const result = batchId != null ? getSessionExtractionEval(resolveDbOpt(opts), { batchId, scenarioId: scenario.id, includeInactive: opts.inactive === true, }) : getSessionExtractionEval(resolveDbOpt(opts), { sessionId: sessionId as number, scenarioId: scenario.id, includeInactive: opts.inactive === true, }); if (opts.json) { console.log(JSON.stringify(result, null, 2)); return; } p.intro("codemem diagnostics extraction report"); p.log.info( [ `Scenario: ${result.scenario.id} — ${result.scenario.title}`, `Target: ${result.target.type}${result.target.batchId != null ? ` #${result.target.batchId}` : ""}`, `Session: ${result.session.id} (${result.session.project ?? "no-project"})`, `Session class: ${result.session.sessionClass}`, `Summary disposition: ${result.session.summaryDisposition}`, ].join("\n"), ); p.log.info( [ `Pass: ${result.pass ? "yes" : "no"}`, `Summary count: ${result.counts.summaries}`, `Observation count: ${result.counts.observations}`, `Summary thread coverage: ${result.coverage.summaryThreadCoverage}`, `Observation thread coverage: ${result.coverage.observationThreadCoverage}`, `Total thread coverage: ${result.coverage.totalThreadCoverage}`, `Duplicate observation threads: ${result.coverage.duplicateObservationThreads}`, ].join("\n"), ); if (result.failureReasons.length > 0) { p.log.warn("Failure reasons:"); for (const reason of result.failureReasons) { p.log.message(` - ${reason}`); } } p.log.info("Thread coverage:"); for (const thread of result.threads) { p.log.message( ` ${thread.id.padEnd(22)} summary=${thread.summaryMatch ? "yes" : "no"} observations=${thread.observationMatch ? "yes" : "no"}`, ); } p.outro("done"); } catch (error) { const message = error instanceof Error ? error.message : "Extraction report failed"; if (opts.json) { emitJsonError("extraction_report_failed", message); } else { p.log.error(message); process.exitCode = 1; } return; } }, ); return cmd; } function createMemoryExtractionReplayCommand(): Command { const cmd = new Command("extraction-replay") .configureHelp(helpStyle) .description("Replay and score extraction without saving memories") .requiredOption("--batch-id <id>", "raw-event flush batch ID to replay") .option( "--transcript-budget <chars>", "override replay transcript budget in characters (replay only)", ) .option("--observer-tier-routing", "use replay-only benchmark-backed observer tier routing") .option("--observer-temperature <value>", "override observer temperature for replay only") .option("--openai-responses", "use OpenAI Responses API for replay only") .option( "--reasoning-effort <level>", "set OpenAI reasoning.effort for replay only (responses path)", ) .option( "--reasoning-summary <mode>", "set OpenAI reasoning.summary for replay only (responses path)", ) .option( "--max-output-tokens <n>", "override OpenAI max_output_tokens for replay only (responses path)", ) .requiredOption("--scenario <id>", "built-in extraction eval scenario ID"); addDbOption(cmd); addJsonOption(cmd); cmd.action( async ( opts: DbOpts & JsonOpts & { batchId: string; observerTierRouting?: boolean; openaiResponses?: boolean; reasoningEffort?: string; reasoningSummary?: string; maxOutputTokens?: string; observerTemperature?: string; transcriptBudget?: string; scenario: string; }, ) => { try { const batchIdInput = opts.batchId?.trim() ?? ""; const batchId = parseStrictPositiveId(batchIdInput); if (batchId === null) { throw new Error(`Invalid batch ID: ${batchIdInput || opts.batchId}`); } const scenarioId = opts.scenario?.trim() ?? ""; const scenario = getSessionExtractionEvalScenario(scenarioId); if (!scenario) { throw new Error(`Unknown extraction eval scenario: ${scenarioId || opts.scenario}`); } const transcriptBudgetInput = opts.transcriptBudget?.trim() ?? ""; const transcriptBudget = transcriptBudgetInput.length > 0 ? parseStrictPositiveId(transcriptBudgetInput) : null; if (transcriptBudgetInput.length > 0 && transcriptBudget === null) { throw new Error( `Invalid transcript budget: ${transcriptBudgetInput || opts.transcriptBudget}`, ); } const observerTemperatureInput = opts.observerTemperature?.trim() ?? ""; let observerTemperature: number | undefined; if (observerTemperatureInput.length > 0) { const parsed = Number(observerTemperatureInput); if (!Number.isFinite(parsed)) { throw new Error( `Invalid observer temperature: ${observerTemperatureInput || opts.observerTemperature}`, ); } observerTemperature = parsed; } const maxOutputTokensInput = opts.maxOutputTokens?.trim() ?? ""; const maxOutputTokens = maxOutputTokensInput.length > 0 ? parseStrictPositiveId(maxOutputTokensInput) : null; if (maxOutputTokensInput.length > 0 && maxOutputTokens === null) { throw new Error( `Invalid max output tokens: ${maxOutputTokensInput || opts.maxOutputTokens}`, ); } const observerConfig = loadObserverConfig(); const observerConfigWithOverrides = { ...observerConfig, observerTemperature: observerTemperature ?? observerConfig.observerTemperature, observerOpenAIUseResponses: resolveOpenAIResponsesOverride( opts.openaiResponses, observerConfig.observerOpenAIUseResponses, ), observerReasoningEffort: opts.reasoningEffort === undefined ? observerConfig.observerReasoningEffort : opts.reasoningEffort.trim() || null, observerReasoningSummary: opts.reasoningSummary === undefined ? observerConfig.observerReasoningSummary : opts.reasoningSummary.trim() || null, observerMaxOutputTokens: maxOutputTokens ?? observerConfig.observerMaxOutputTokens ?? observerConfig.observerMaxTokens, observerExplicitConfigKeys: maxOutputTokens === null ? observerConfig.observerExplicitConfigKeys : [ ...new Set([ ...(observerConfig.observerExplicitConfigKeys ?? []), "observerMaxOutputTokens", ]), ], }; const observer = new ObserverClient(observerConfigWithOverrides); const result = opts.observerTierRouting === true ? await replayBatchExtractionWithTierRouting( resolveDbOpt(opts), observerConfigWithOverrides, { batchId, scenarioId: scenario.id, transcriptBudget: transcriptBudget ?? undefined, }, ) : await replayBatchExtraction(resolveDbOpt(opts), observer, { batchId, scenarioId: scenario.id, transcriptBudget: transcriptBudget ?? undefined, }); if (opts.json) { console.log(JSON.stringify(result, null, 2)); return; } p.intro("codemem diagnostics extraction replay"); p.log.info( [ `Scenario: ${result.scenario.id} — ${result.scenario.title}`, `Batch: ${result.target.batchId}`, `Session: ${result.target.sessionId}`, `Observer: ${result.observer.provider}/${result.observer.model}`, `Tier: ${result.observer.tier ?? "manual"}`, `OpenAI Responses: ${result.observer.openaiUseResponses ? "yes" : "no"}`, `Reasoning effort: ${result.observer.reasoningEffort ?? "none"}`, `Classification: ${result.classification.status}`, `Pass: ${result.evaluation.pass ? "yes" : "no"}`, ].join("\n"), ); if (result.classification.reason) { p.log.message(`Classification reason: ${result.classification.reason}`); } if (result.evaluation.failureReasons.length > 0) { p.log.warn("Failure reasons:"); for (const reason of result.evaluation.failureReasons) { p.log.message(` - ${reason}`); } } p.log.info( [ `Fresh summaries: ${result.evaluation.counts.summaries}`, `Fresh observations: ${result.evaluation.counts.observations}`, `Summary thread coverage: ${result.evaluation.coverage.summaryThreadCoverage}`, `Observation thread coverage: ${result.evaluation.coverage.observationThreadCoverage}`, `Total thread coverage: ${result.evaluation.coverage.totalThreadCoverage}`, ].join("\n"), ); p.outro("done"); } catch (error) { if (error instanceof ContextOnlyReplayError) { emitContextOnlyReplayOutcome( contextOnlyReplayOutcome(error, { batchId: Number(opts.batchId), scenarioId: opts.scenario.trim(), }), opts, ); return; } const message = error instanceof Error ? error.message : "Extraction replay failed"; if (opts.json) { emitJsonError("extraction_replay_failed", message); } else { p.log.error(message); process.exitCode = 1; } return; } }, ); return cmd; } interface BenchmarkDispositionQuality { summaryDisposition: { expected: string; actual: string; score: number | null; }; } export function reconcileExtractionBenchmarkStatus<T extends BenchmarkDispositionQuality>(input: { purpose: "shape_quality" | "replay_robustness"; classification: { status: "pass" | "shape_fail" | "observer_no_output"; reason: string; }; finalFailureReasons: string[]; initialQuality: T | null; finalQuality: T | null; }): { status: "pass" | "shape_fail" | "observer_no_output"; reason: string; quality: T | null; initialQuality: T | null; } { const quality = input.finalQuality; let status = input.classification.status; let reason = input.classification.reason; if (input.purpose === "shape_quality" && quality && status !== "observer_no_output") { if (quality.summaryDisposition.score === 0) { status = "shape_fail"; reason = `summary disposition ${quality.summaryDisposition.actual} does not satisfy expected ${quality.summaryDisposition.expected}`; } else if ( status === "shape_fail" && quality.summaryDisposition.actual === "skip" && input.finalFailureReasons.length > 0 && input.finalFailureReasons.every((failure) => failure.startsWith("summary count ")) ) { status = "pass"; reason = "valid low-signal skip satisfies benchmark disposition"; } } return { status, reason, quality, initialQuality: input.initialQuality }; } interface BenchmarkReasoningSettings { reasoningEffort: string | null; reasoningSummary: string | null; } export function summarizeBenchmarkReasoning( runs: readonly BenchmarkReasoningSettings[], fallback: BenchmarkReasoningSettings, ): BenchmarkReasoningSettings { const source = runs[0] ?? fallback; const reasoningEfforts = new Set(runs.map((run) => run.reasoningEffort)); const reasoningSummaries = new Set(runs.map((run) => run.reasoningSummary)); return { reasoningEffort: reasoningEfforts.size > 1 ? "mixed" : source.reasoningEffort, reasoningSummary: reasoningSummaries.size > 1 ? "mixed" : source.reasoningSummary, }; } function benchmarkSecondAttemptValue<T>(attempted: boolean, value: T | null): T | null { return attempted ? value : null; } function benchmarkSecondAttemptList<T>(attempted: boolean, value: T[]): T[] { return attempted ? value : []; } function buildBenchmarkSecondAttempt( result: ExtractionReplayResult, input: { attempted: boolean; reason: string | null; quality: ExtractionBenchmarkScore | null }, ) { return { attempted: input.attempted, reason: input.reason, raw: benchmarkSecondAttemptValue(input.attempted, result.observer.repairedRaw), status: benchmarkSecondAttemptValue( input.attempted, result.repairedClassification?.status ?? null, ), resultReason: benchmarkSecondAttemptValue( input.attempted, result.repairedClassification?.reason ?? null, ), pass: benchmarkSecondAttemptValue(input.attempted, result.repairedEvaluation?.pass ?? null), failureReasons: benchmarkSecondAttemptList( input.attempted, result.repairedEvaluation?.failureReasons ?? [], ), summaries: benchmarkSecondAttemptValue( input.attempted, result.repairedEvaluation?.counts.summaries ?? null, ), observations: benchmarkSecondAttemptValue( input.attempted, result.repairedEvaluation?.counts.observations ?? null, ), diagnostics: benchmarkSecondAttemptValue(input.attempted, result.observer.repairedDiagnostics), elapsedMs: benchmarkSecondAttemptValue(input.attempted, result.observer.repairedElapsedMs), usage: benchmarkSecondAttemptValue(input.attempted, result.observer.repairedUsage), quality: benchmarkSecondAttemptValue(input.attempted, input.quality), }; } interface ExtractionBenchmarkReportingRun { requestedOutputMode: ObserverOutputMode; actualOutputMode: ObserverOutputMode; outputValidation: "valid" | "invalid" | "not_applicable"; repairAttempted: boolean; retryAttempted: boolean; observations: number; telemetry: { totalElapsedMs: number | null; totalUsage: ObserverTokenUsage | null }; quality: ExtractionBenchmarkScore | null; } interface ExtractionBenchmarkModeMetrics { runs: number; validOutputs: number; validOutputRate: number | null; repairAttempts: number; repairRate: number | null; retryAttempts: number; retryRate: number | null; repairOrRetryAttempts: number; repairOrRetryRate: number | null; knownLatencyRuns: number; totalLatencyMs: number; meanLatencyMs: number | null; knownTokenRuns: number; inputTokens: number; outputTokens: number; totalTokens: number; retainedObservations: number; meanRetainedObservations: number | null; summaryDispositionEvaluated: number; summaryDispositionMatches: number; summaryDispositionMatchRate: number | null; structuralQualityEvaluated: number; contractIntegrityRate: number | null; semanticQualityEvaluated: number; meanSemanticQuality: number | null; } type BenchmarkCostUnavailableReason = | "missing_usage" | "unknown_model_pricing" | "model_fallback_unresolved" | null; export function estimateBenchmarkAttemptCost( model: string | null, usage: ObserverTokenUsage | null, ): { total: ExtractionModelCostEstimate | null; unavailableReason: BenchmarkCostUnavailableReason; } { if (model === null) return { total: null, unavailableReason: "model_fallback_unresolved" }; const total = estimateExtractionModelCost(model, usage); if (total) return { total, unavailableReason: null }; if (usage === null) return { total: null, unavailableReason: "missing_usage" }; return { total: null, unavailableReason: "unknown_model_pricing" }; } function rate(numerator: number, denominator: number): number | null { return denominator > 0 ? numerator / denominator : null; } function mean(values: number[]): number | null { return values.length > 0 ? values.reduce((sum, value) => sum + value, 0) / values.length : null; } function countOutputModes( runs: readonly ExtractionBenchmarkReportingRun[], key: "requestedOutputMode" | "actualOutputMode", ): Partial<Record<ObserverOutputMode, number>> { const counts: Partial<Record<ObserverOutputMode, number>> = {}; for (const run of runs) counts[run[key]] = (counts[run[key]] ?? 0) + 1; return counts; } function collectBenchmarkMetricSamples(runs: readonly ExtractionBenchmarkReportingRun[]) { return { latencies: runs.flatMap((run) => run.telemetry.totalElapsedMs == null ? [] : [run.telemetry.totalElapsedMs], ), usages: runs.flatMap((run) => run.telemetry.totalUsage == null ? [] : [run.telemetry.totalUsage], ), summaryScores: runs.flatMap((run) => run.quality == null ? [] : [run.quality.summaryDisposition.score], ), structuralScores: runs.flatMap((run) => { if (run.outputValidation === "invalid") return [0]; if (run.outputValidation === "valid") return [1]; return run.quality == null ? [] : [run.quality.schemaCompliance.score]; }), semanticScores: runs.flatMap((run) => run.quality == null ? [] : semanticQualityScore(run.quality), ), }; } function semanticQualityScore(quality: ExtractionBenchmarkScore): number[] { if (quality.reviewStatus !== "reviewed") return []; const dimensions: Array<[number | null | undefined, number]> = [ [quality.summaryDisposition.score, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.summaryDisposition], [quality.requiredRecall?.score, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.requiredRecall], [quality.optionalRecall?.score, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.optionalRecall], [quality.worthinessPrecision?.score, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.worthinessPrecision], [quality.summaryBreadth?.score, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.summaryBreadth], [ quality.observationSignals?.redundancyAvoidance, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.redundancyAvoidance, ], [quality.observationSignals?.segmentation, EXTRACTION_BENCHMARK_QUALITY_WEIGHTS.segmentation], ]; const available = dimensions.filter((entry): entry is [number, number] => entry[0] != null); const weight = available.reduce((sum, [, dimensionWeight]) => sum + dimensionWeight, 0); if (weight === 0) return []; return [ available.reduce((sum, [score, dimensionWeight]) => sum + score * dimensionWeight, 0) / weight, ]; } function benchmarkModeMetrics( runs: readonly ExtractionBenchmarkReportingRun[], ): ExtractionBenchmarkModeMetrics { const validOutputs = runs.filter( (run) => run.outputValidation === "valid" || run.quality?.schemaCompliance.compliant === true, ).length; const repairAttempts = runs.filter((run) => run.repairAttempted).length; const retryAttempts = runs.filter((run) => run.retryAttempted).length; const repairOrRetryAttempts = runs.filter( (run) => run.repairAttempted || run.retryAttempted, ).length; const { latencies, usages, summaryScores, structuralScores, semanticScores } = collectBenchmarkMetricSamples(runs); const retainedObservations = runs.reduce((sum, run) => sum + run.observations, 0); const matchingSummaryDispositions = summaryScores.filter((score) => score === 1).length; return { runs: runs.length, validOutputs, validOutputRate: rate(validOutputs, runs.length), repairAttempts, repairRate: rate(repairAttempts, runs.length), retryAttempts, retryRate: rate(retryAttempts, runs.length), repairOrRetryAttempts, repairOrRetryRate: rate(repairOrRetryAttempts, runs.length), knownLatencyRuns: latencies.length, totalLatencyMs: latencies.reduce((sum, value) => sum + value, 0), meanLatencyMs: mean(latencies), knownTokenRuns: usages.length, inputTokens: usages.reduce((sum, usage) => sum + usage.inputTokens, 0), outputTokens: usages.reduce((sum, usage) => sum + usage.outputTokens, 0), totalTokens: usages.reduce( (sum, usage) => sum + (usage.totalTokens ?? usage.inputTokens + usage.outputTokens), 0, ), retainedObservations, meanRetainedObservations: rate(retainedObservations, runs.length), summaryDispositionEvaluated: summaryScores.length, summaryDispositionMatches: matchingSummaryDispositions, summaryDispositionMatchRate: rate(matchingSummaryDispositions, summaryScores.length), structuralQualityEvaluated: structuralScores.length, contractIntegrityRate: mean(structuralScores), semanticQualityEvaluated: semanticScores.length, meanSemanticQuality: mean(semanticScores), }; } export function summarizeExtractionBenchmarkReporting( runs: readonly ExtractionBenchmarkReportingRun[], ): { requestedOutputModes: Partial<Record<ObserverOutputMode, number>>; actualOutputModes: Partial<Record<ObserverOutputMode, number>>; overall: ExtractionBenchmarkModeMetrics; byActualOutputMode: Partial<Record<ObserverOutputMode, ExtractionBenchmarkModeMetrics>>; } { const byActualOutputMode: Partial<Record<ObserverOutputMode, ExtractionBenchmarkModeMetrics>> = {}; for (const mode of ["json_schema", "forced_tool", "legacy_xml"] as const) { const modeRuns = runs.filter((run) => run.actualOutputMode === mode); if (modeRuns.length > 0) byActualOutputMode[mode] = benchmarkModeMetrics(modeRuns); } return { requestedOutputModes: countOutputModes(runs, "requestedOutputMode"), actualOutputModes: countOutputModes(runs, "actualOutputMode"), overall: benchmarkModeMetrics(runs), byActualOutputMode, }; } type ExtractionBenchmarkAttemptStatus = { iteration: number; batchId: number; purpose: "shape_quality" | "replay_robustness"; status: "pass" | "shape_fail" | "observer_no_output" | "output_failure"; expectedTier: string | null; tier: string | null; }; interface ExtractionBenchmarkContextOnlySkip extends ContextOnlyReplayOutcome { iteration: number; purpose: "shape_quality" | "replay_robustness"; } export function summarizeExtractionBenchmarkAttempts( attempts: readonly ExtractionBenchmarkAttemptStatus[], batches: readonly Pick<ExtractionBenchmarkAttemptStatus, "batchId" | "purpose">[], ) { return { total: attempts.length, shapeQualityTotal: attempts.filter((attempt) => attempt.purpose === "shape_quality").length, shapeQualityPasses: attempts.filter( (attempt) => attempt.purpose === "shape_quality" && attempt.status === "pass", ).length, shapeQualityFails: attempts.filter( (attempt) => attempt.purpose === "shape_quality" && attempt.status !== "pass", ).length, expectedTierTotal: attempts.filter((attempt) => attempt.expectedTier != null).length, expectedTierMatches: attempts.filter( (attempt) => attempt.expectedTier != null && attempt.expectedTier === attempt.tier, ).length, robustnessNoOutput: attempts.filter( (attempt) => attempt.purpose === "replay_robustness" && (attempt.status === "observer_no_output" || attempt.status === "output_failure"), ).length, perBatchStability: batches.map((batch) => { const batchAttempts = attempts .filter((attempt) => attempt.batchId === batch.batchId) .toSorted((left, right) => left.iteration - right.iteration); const passes = batchAttempts.filter((attempt) => attempt.status === "pass").length; return { batchId: batch.batchId, purpose: batch.purpose, passes, total: batchAttempts.length, passRate: rate(passes, batchAttempts.length), statuses: batchAttempts.map((attempt) => attempt.status), }; }), }; } function cliObserverProviderOverride( cfg: ReturnType<typeof loadObserverConfig>, providerFlag: string | undefined, ) { const observerProvider = providerFlag?.trim() || cfg.observerProvider; return { observerProvider, observerBaseUrl: observerBaseUrlForProviderOverride(cfg, observerProvider || ""), }; } function createMemoryExtractionBenchmarkCommand(): Command { const cmd = new Command("extraction-benchmark") .configureHelp(helpStyle) .description("Benchmark extraction cost and quality") .requiredOption("--benchmark <id>", "benchmark profile id") .option("--observer-provider <provider>", "override observer provider for this benchmark run") .option("--observer-model <model>", "override observer model for this benchmark run") .option("--observer-tier-routing", "use replay-only benchmark-backed observer tier routing") .option("--openai-responses", "use OpenAI Responses API for this benchmark run") .option( "--reasoning-effort <level>", "set OpenAI reasoning.effort for this benchmark run (responses path)", ) .option( "--reasoning-summary <mode>", "set OpenAI reasoning.summary for this benchmark run (responses path)", ) .option( "--max-output-tokens <n>", "override OpenAI max_output_tokens for this benchmark run (responses path)", ) .option( "--observer-temperature <value>", "override observer temperature for this benchmark run", ) .option( "--transcript-budget <chars>", "override replay transcript budget in characters for this benchmark run", ) .option( "--repetitions <n>", "run every benchmark batch 1-10 times to measure model stability", "1", ); addDbOption(cmd); addJsonOption(cmd); cmd.action( async ( opts: DbOpts & JsonOpts & { benchmark: string; observerProvider?: string; observerModel?: string; observerTierRouting?: boolean; openaiResponses?: boolean; reasoningEffort?: string; reasoningSummary?: string; maxOutputTokens?: string; observerTemperature?: string; transcriptBudget?: string; repetitions?: string; }, ) => { try { const benchmarkId = opts.benchmark?.trim() ?? ""; const benchmark = getExtractionBenchmarkProfile(benchmarkId); if (!benchmark) { throw new Error(`Unknown extraction benchmark: ${benchmarkId || opts.benchmark}`); } const transcriptBudgetInput = opts.transcriptBudget?.trim() ?? ""; const transcriptBudget = transcriptBudgetInput.length > 0 ? parseStrictPositiveId(transcriptBudgetInput) : null; if (transcriptBudgetInput.length > 0 && transcriptBudget === null) { throw new Error( `Invalid transcript budget: ${transcriptBudgetInput || opts.transcriptBudget}`, ); } const observerTemperatureInput = opts.observerTemperature?.trim() ?? ""; let observerTemperature: number | undefined; if (observerTemperatureInput.length > 0) { const parsed = Number(observerTemperatureInput); if (!Number.isFinite(parsed)) { throw new Error( `Invalid observer temperature: ${observerTemperatureInput || opts.observerTemperature}`, ); } observerTemperature = parsed; } const maxOutputTokensInput = opts.maxOutputTokens?.trim() ?? ""; const maxOutputTokens = maxOutputTokensInput.length > 0 ? parseStrictPositiveId(maxOutputTokensInput) : null; if (maxOutputTokensInput.length > 0 && maxOutputTokens === null) { throw new Error( `Invalid max output tokens: ${maxOutputTokensInput || opts.maxOutputTokens}`, ); } const repetitionsInput = opts.repetitions?.trim() ?? "1"; const repetitions = parseStrictPositiveId(repetitionsInput); if (repetitions === null || repetitions > 10) { throw new Error(`Invalid repetitions: ${repetitionsInput || opts.repetitions}`); } const observerConfig = loadObserverConfig(); const observerConfigWithOverrides = { ...observerConfig, ...cliObserverProviderOverride(observerConfig, opts.observerProvider), observerModel: opts.observerModel?.trim() || observerConfig.observerModel, observerTemperature: observerTemperature ?? observerConfig.observerTemperature, observerOpenAIUseResponses: resolveOpenAIResponsesOverride( opts.openaiResponses, observerConfig.observerOpenAIUseResponses, ), observerReasoningEffort: opts.reasoningEffort === undefined ? observerConfig.observerReasoningEffort : opts.reasoningEffort.trim() || null, observerReasoningSummary: opts.reasoningSummary === undefined ? observerConfig.observerReasoningSummary : opts.reasoningSummary.trim() || null, observerMaxOutputTokens: maxOutputTokens ?? observerConfig.observerMaxOutputTokens ?? observerConfig.observerMaxTokens, observerExplicitConfigKeys: maxOutputTokens === null ? observerConfig.observerExplicitConfigKeys : [ ...new Set([ ...(observerConfig.observerExplicitConfigKeys ?? []), "observerMaxOutputTokens", ]), ], }; const observer = new ObserverClient(observerConfigWithOverrides); const outputFailures: Array< ExtractionBenchmarkReportingRun & { iteration: number; batchId: number; purpose: "shape_quality" | "replay_robustness"; status: "output_failure"; reason: string; expectedTier: string | null; provider: string; model: string | null; transport: string; requestedModel: string | null; resolvedModel: string | null; modelFallbackApplied: boolean; modelFallbackReason: string | null; tier: "simple" | "rich" | null; openaiUseResponses: boolean; reasoningEffort: string | null; reasoningSummary: string | null; maxOutputTokens: number | null; temperature: number | null; cost: { total: ExtractionModelCostEstimate | null; unavailableReason: | "missing_usage" | "unknown_model_pricing" | "model_fallback_unresolved" | null; }; } > = []; const runs = [] as Array<{ iteration: number; batchId: number; sessionId: number; label: string; purpose: "shape_quality" | "replay_robustness"; complexity: string; scenarioId: string; expectedTier: string | null; expectedSummaryDisposition: "required" | "optional" | "skip"; analysis: { eventSpan: number; promptCount: number; toolCount: number; transcriptLength: number; }; status: "pass" | "shape_fail" | "observer_no_output"; reason: string; tier: string; provider: string; model: string; transport: string; requestedModel: string; resolvedModel: string | null; modelFallbackApplied: boolean; modelFallbackReason: string | null; requestedOutputMode: ObserverOutputMode; actualOutputMode: ObserverOutputMode; outputValidation: "valid" | "invalid" | "not_applicable"; openaiUseResponses: boolean; reasoningEffort: string | null; reasoningSummary: string | null; maxOutputTokens: number | null; temperature: number | null; summaries: number; observations: number; repairApplied: boolean; repairAttempted: boolean; retryAttempted: boolean; retryReason: string | null; initial: { raw: string | null; status: "pass" | "shape_fail" | "observer_no_output"; reason: string; pass: boolean; failureReasons: string[]; summaries: number; observations: number; diagnostics: ExtractionStructuralDiagnostics | null; elapsedMs: number | null; usage: ObserverTokenUsage | null; quality: ExtractionBenchmarkScore | null; }; repair: { applied: boolean; raw: string | null; status: "pass" | "shape_fail" | "observer_no_output" | null; reason: string | null; pass: boolean | null; failureReasons: string[]; summaries: number | null; observations: number | null; diagnostics: ExtractionStructuralDiagnostics | null; elapsedMs: number | null; usage: ObserverTokenUsage | null; quality: ExtractionBenchmarkScore | null; }; retry: { attempted: boolean; reason: string | null; raw: string | null; status: "pass" | "shape_fail" | "observer_no_output" | null; resultReason: string | null; pass: boolean | null; failureReasons: string[]; summaries: number | null; observations: number | null; diagnostics: ExtractionStructuralDiagnostics | null; elapsedMs: number | null; usage: ObserverTokenUsage | null; quality: ExtractionBenchmarkScore | null; }; telemetry: { totalElapsedMs: number | null; totalUsage: ObserverTokenUsage | null; }; pricing: ReturnType<typeof getExtractionModelPricing>; cost: { initial: ExtractionModelCostEstimate | null; repair: ExtractionModelCostEstimate | null; retry: ExtractionModelCostEstimate | null; total: ExtractionModelCostEstimate | null; unavailableReason: | "missing_usage" | "unknown_model_pricing" | "model_fallback_unresolved" | null; }; quality: ExtractionBenchmarkScore | null; }>; const contextOnlySkips: ExtractionBenchmarkContextOnlySkip[] = []; for (let iteration = 1; iteration <= repetitions; iteration += 1) { for (const batch of benchmark.batches) { const scenarioId = batch.scenarioId ?? benchmark.scenarioId; let selectedObserver: ExtractionReplayObserverIdentity = extractionReplayObserverIdentity(observer, null); let result: Awaited<ReturnType<typeof replayBatchExtraction>>; try { result = opts.observerTierRouting === true ? await replayBatchExtractionWithTierRouting( resolveDbOpt(opts), observerConfigWithOverrides, { batchId: batch.batchId, scenarioId, transcriptBudget: transcriptBudget ?? undefined, onOutputFailure: (context) => { selectedObserver = context; }, }, ) : await replayBatchExtraction(resolveDbOpt(opts), observer, { batchId: batch.batchId, scenarioId, transcriptBudget: transcriptBudget ?? undefined, }); } catch (error) { if (error instanceof ContextOnlyReplayError) { contextOnlySkips.push({ ...contextOnlyReplayOutcome(error, { batchId: batch.batchId, scenarioId }), iteration, purpose: batch.purpose, }); continue; } if ( !(error instanceof ObserverOutputError) && !(error instanceof ObserverOutputTransportError) ) { throw error; } if (opts.observerTierRouting !== true) { selectedObserver = extractionReplayObserverIdentity(observer, null); } const failureCost = estimateBenchmarkAttemptCost( selectedObserver.model, error.telemetry.totalUsage, ); outputFailures.push({ iteration, batchId: batch.batchId, purpose: batch.purpose, status: "output_failure", reason: error instanceof ObserverOutputError ? error.reason : error.code, expectedTier: batch.expectedTier ?? null, ...selectedObserver, requestedOutputMode: error.diagnostics.requestedMode, actualOutputMode: error.diagnostics.actualMode, outputValidation: "invalid", repairAttempted: error.diagnostics.repairAttempted, retryAttempted: error.diagnostics.retryAttempted, observations: 0, telemetry: error.telemetry, cost: failureCost, quality: null, }); continue; } const costModel = result.observer.modelFallbackApplied ? result.observer.resolvedModel : (result.observer.resolvedModel ?? result.observer.model); const initialCost = costModel ? estimateExtractionModelCost(costModel, result.observer.initialUsage) : null; const secondAttemptCost = costModel ? estimateExtractionModelCost(costModel, result.observer.repairedUsage) : null; const repairCost = benchmarkSecondAttemptValue( result.observer.repairAttempted, secondAttemptCost, ); const retryCost = benchmarkSecondAttemptValue( result.observer.retryAttempted, secondAttemptCost, ); const totalCost = costModel ? estimateExtractionModelCost(costModel, result.observer.totalUsage) : null; const pricing = costModel ? getExtractionModelPricing(costModel) : null; const costUnavailableReason = totalCost ? null : result.observer.modelFallbackApplied && !result.observer.resolvedModel ? "model_fallback_unresolved" : result.observer.totalUsage == null ? "missing_usage" : "unknown_model_pricing"; const initialQuality = result.observer.initialDiagnostics ? scoreExtractionBenchmarkOutput({ parsed: result.observer.initialParsed, diagnostics: result.observer.initialDiagnostics, review: batch.review ?? { status: "unreviewed", reviewerNotes: "No durable-fact review has been recorded for this batch.", }, estimatedCostUsd: initialCost?.totalCostUsd ?? null, expectedSummaryDisposition: batch.expectedSummaryDisposition, }) : null; const secondAttemptQuality = result.observer.repairedParsed && result.observer.repairedDiagnostics ? scoreExtractionBenchmarkOutput({ parsed: result.observer.repairedParsed, diagnostics: result.observer.repairedDiagnostics, review: batch.review ?? { status: "unreviewed", reviewerNotes: "No durable-fact review has been recorded for this batch.", }, estimatedCostUsd: secondAttemptCost?.totalCostUsd ?? null, expectedSummaryDisposition: batch.expectedSummaryDisposition, }) : null; const finalQuality = result.observer.diagnostics ? scoreExtractionBenchmarkOutput({ parsed: result.observer.parsed, diagnostics: result.observer.diagnostics, review: batch.review ?? { status: "unreviewed", reviewerNotes: "No durable-fact review has been recorded for this batch.", }, estimatedCostUsd: totalCost?.totalCostUsd ?? null, expectedSummaryDisposition: batch.expectedSummaryDisposition, }) : null; const reconciled = reconcileExtractionBenchmarkStatus({ purpose: batch.purpose, classification: result.classification, finalFailureReasons: result.evaluation.failureReasons, initialQuality, finalQuality, }); const repairAttempt = buildBenchmarkSecondAttempt(result, { attempted: result.observer.repairAttempted, reason: null, quality: secondAttemptQuality, }); const retryAttempt = buildBenchmarkSecondAttempt(result, { attempted: result.observer.retryAttempted, reason: result.observer.retryReason, quality: secondAttemptQuality, }); runs.push({ iteration, batchId: batch.batchId, sessionId: batch.sessionId, label: batch.label, purpose: batch.purpose, complexity: batch.complexity, scenarioId, expectedTier: batch.expectedTier ?? null, expectedSummaryDisposition: batch.expectedSummaryDisposition, analysis: { eventSpan: result.analysis.eventSpan, promptCount: result.analysis.promptCount, toolCount: result.analysis.toolCount, transcriptLength: result.analysis.transcriptLength, }, status: reconciled.status, reason: reconciled.reason, tier: result.observer.tier ?? "manual", provider: result.observer.provider, model: result.observer.model, transport: result.observer.transport, requestedModel: result.observer.requestedModel, resolvedModel: result.observer.resolvedModel, modelFallbackApplied: result.observer.modelFallbackApplied, modelFallbackReason: result.observer.modelFallbackReason, requestedOutputMode: result.observer.requestedOutputMode, actualOutputMode: result.observer.actualOutputMode, outputValidation: result.observer.outputValidation, openaiUseResponses: result.observer.openaiUseResponses, reasoningEffort: result.observer.reasoningEffort, reasoningSummary: result.observer.reasoningSummary, maxOutputTokens: result.observer.maxOutputTokens, temperature: result.observer.temperature, summaries: result.evaluation.counts.summaries, observations: result.evaluation.counts.observations, repairApplied: result.observer.repairApplied, repairAttempted: result.observer.repairAttempted, retryAttempted: result.observer.retryAttempted, retryReason: result.observer.retryReason, initial: { raw: result.observer.initialRaw, status: result.initialClassification.status, reason: result.initialClassification.reason, pass: result.initialEvaluation.pass, failureReasons: result.initialEvaluation.failureReasons, summaries: result.initialEvaluation.counts.summaries, observations: result.initialEvaluation.counts.observations, diagnostics: result.observer.initialDiagnostics, elapsedMs: result.observer.initialElapsedMs, usage: result.observer.initialUsage, quality: reconciled.initialQuality, }, repair: { applied: result.observer.repairApplied, raw: repairAttempt.raw, status: repairAttempt.status, reason: repairAttempt.resultReason, pass: repairAttempt.pass, failureReasons: repairAttempt.failureReasons, summaries: repairAttempt.summaries, observations: repairAttempt.observations, diagnostics: repairAttempt.diagnostics, elapsedMs: repairAttempt.elapsedMs, usage: repairAttempt.usage, quality: repairAttempt.quality, }, retry: retryAttempt, telemetry: { totalElapsedMs: result.observer.totalElapsedMs, totalUsage: result.observer.totalUsage, }, pricing, cost: { initial: initialCost, repair: repairCost, retry: retryCost, total: totalCost, unavailableReason: costUnavailableReason, }, quality: reconciled.quality, }); } } const reviewedQualityRuns = runs.filter((run) => run.quality?.weightedQualityScore != null); // Context-only skips never enter evaluation, stability, cost or output-rate denominators. const attempts = [...runs, ...outputFailures]; const knownCostRuns = attempts.filter((run) => run.cost.total != null); const attemptSummary = summarizeExtractionBenchmarkAttempts(attempts, benchmark.batches); const knownElapsedRuns = attempts.filter((run) => run.telemetry.totalElapsedMs != null); const summary = { repetitions, scheduledTotal: benchmark.batches.length * repetitions, contextOnlySkipped: contextOnlySkips.length, contextOnlySkips, ...attemptSummary, summaryDispositionTotal: runs.filter((run) => run.quality != null).length, summaryDispositionMatches: runs.filter( (run) => run.quality?.summaryDisposition.score === 1, ).length, reviewedQualityRuns: reviewedQualityRuns.length, knownCostRuns: knownCostRuns.length, unknownCostRuns: attempts.length - knownCostRuns.length, missingUsageRuns: attempts.filter((run) => run.cost.unavailableReason === "missing_usage") .length, unknownPricingRuns: attempts.filter( (run) => run.cost.unavailableReason === "unknown_model_pricing", ).length, fallbackUnresolvedRuns: attempts.filter( (run) => run.cost.unavailableReason === "model_fallback_unresolved", ).length, totalKnownCostUsd: knownCostRuns.reduce( (sum, run) => sum + (run.cost.total?.totalCostUsd ?? 0), 0, ), knownElapsedRuns: knownElapsedRuns.length, totalKnownElapsedMs: knownElapsedRuns.reduce( (sum, run) => sum + (run.telemetry.totalElapsedMs ?? 0), 0, ), output: summarizeExtractionBenchmarkReporting(attempts), outputFailures, }; const uniqueObserverKeys = Array.from( new Set(attempts.map((run) => `${run.provider}::${run.model}::${run.transport}`)), ); const firstObserver = attempts[0]; const benchmarkReasoning = summarizeBenchmarkReasoning(attempts, { reasoningEffort: observer.reasoningEffort, reasoningSummary: observer.reasoningSummary, }); const observerSummary = opts.observerTierRouting === true ? { provider: uniqueObserverKeys.length === 1 ? (firstObserver?.provider ?? observer.provider) : "mixed", model: uniqueObserverKeys.length === 1 ? (firstObserver?.model ?? observer.model) : "mixed", transport: uniqueObserverKeys.length === 1 ? (firstObserver?.transport ?? "unknown") : "mixed", tierRouting: true, openaiUseResponses: uniqueObserverKeys.length === 1 ? (firstObserver?.openaiUseResponses ?? observer.openaiUseResponses) : null, reasoningEffort: benchmarkReasoning.reasoningEffort, reasoningSummary: benchmarkReasoning.reasoningSummary, maxOutputTokens: uniqueObserverKeys.length === 1 ? (firstObserver?.maxOutputTokens ?? null) : null, temperature: uniqueObserverKeys.length === 1 ? (firstObserver?.temperature ?? null) : null, transcriptBudget: transcriptBudget ?? null, selectedObservers: uniqueObserverKeys, } : { provider: observer.provider, model: observer.model, transport: firstObserver?.transport ?? observer.getStatus().runtime, tierRouting: false, openaiUseResponses: observer.openaiUseResponses, reasoningEffort: benchmarkReasoning.reasoningEffort, reasoningSummary: benchmarkReasoning.reasoningSummary, maxOutputTokens: firstObserver?.maxOutputTokens ?? null, temperature: firstObserver?.temperature ?? null, transcriptBudget: transcriptBudget ?? null, selectedObservers: uniqueObserverKeys, }; const output = { benchmark: { id: benchmark.id, title: benchmark.title, scenarioId: benchmark.scenarioId, modelCandidates: benchmark.modelCandidates, }, observer: observerSummary, summary, runs, }; if (opts.json) { console.log(JSON.stringify(output, null, 2)); return; } p.intro("codemem diagnostics extraction benchmark"); p.log.info( [ `Benchmark: ${benchmark.id} — ${benchmark.title}`, `Observer: ${observerSummary.provider}/${observerSummary.model}`, `Transport: ${observerSummary.transport}`, `Tier routing: ${opts.observerTierRouting === true ? "yes" : "no"}`, `OpenAI Responses: ${observerSummary.openaiUseResponses === null ? "mixed" : observerSummary.openaiUseResponses ? "yes" : "no"}`, `Reasoning effort: ${observerSummary.reasoningEffort ?? "not transmitted"}`, `Reasoning summary: ${observerSummary.reasoningSummary ?? "not transmitted"}`, `Max output tokens: ${observerSummary.transport === "codex_consumer" ? "not transmitted" : (observerSummary.maxOutputTokens ?? "mixed")}`, `Temperature: ${observerSummary.transport === "mixed" ? "mixed" : (observerSummary.temperature ?? "not transmitted")}`, `Transcript budget override: ${transcriptBudget ?? "default"}`, `Repetitions: ${summary.repetitions}`, `Context-only skips: ${summary.contextOnlySkipped} (${summary.scheduledTotal} scheduled; ${summary.total} observer attempts)`, `Shape-quality passes: ${summary.shapeQualityPasses}/${summary.shapeQualityTotal}`, `Shape-quality fails: ${summary.shapeQualityFails}`, `Expected-tier matches: ${summary.expectedTierMatches}/${summary.expectedTierTotal}`, `Observer no-output cases: ${summary.robustnessNoOutput}`, `Summary disposition matches: ${summary.summaryDispositionMatches}/${summary.summaryDispositionTotal}`, `Reviewed quality runs: ${summary.reviewedQualityRuns} (compare per-run dimensions; scores are fixture-specific)`, `Known estimated cost: $${summary.totalKnownCostUsd.toFixed(6)} (${summary.knownCostRuns}/${summary.total}; missing usage=${summary.missingUsageRuns}, unknown pricing=${summary.unknownPricingRuns}, unresolved fallback=${summary.fallbackUnresolvedRuns})`, `Known elapsed time: ${summary.totalKnownElapsedMs}ms (${summary.knownElapsedRuns}/${summary.total} run(s))`, `Output modes requested/actual: ${JSON.stringify(summary.output.requestedOutputModes)} / ${JSON.stringify(summary.output.actualOutputModes)}`, `Valid output: ${summary.output.overall.validOutputs}/${summary.output.overall.runs}; repair/retry: ${summary.output.overall.repairAttempts}/${summary.output.overall.retryAttempts}; mean latency: ${summary.output.overall.meanLatencyMs ?? "n/a"}ms`, `Tokens input/output: ${summary.output.overall.inputTokens}/${summary.output.overall.outputTokens} (${summary.output.overall.knownTokenRuns} measured run(s)); retained observations: ${summary.output.overall.retainedObservations}`, `Summary disposition matches: ${summary.output.overall.summaryDispositionMatches}/${summary.output.overall.summaryDispositionEvaluated}; contract integrity/semantic quality: ${summary.output.overall.contractIntegrityRate?.toFixed(3) ?? "n/a"}/${summary.output.overall.meanSemanticQuality?.toFixed(3) ?? "n/a"}`, ].join("\n"), ); for (const skipped of contextOnlySkips) { p.log.message( ` [${skipped.batchId}#${skipped.iteration}] context_only — ${skipped.code}; not evaluated`, ); } for (const run of runs) { const qualityLabel = run.quality?.weightedQualityScore == null ? "n/a" : run.quality.weightedQualityScore.toFixed(3); const costLabel = run.cost.total == null ? "n/a" : `$${run.cost.total.totalCostUsd.toFixed(6)}`; const latencyLabel = run.telemetry.totalElapsedMs == null ? "n/a" : `${run.telemetry.totalElapsedMs}ms`; const missingRequired = run.quality?.requiredRecall.missingLabelIds.join(",") || "none"; p.log.message( ` [${run.batchId}#${run.iteration}] ${run.status.padEnd(18)} ${run.complexity.padEnd(10)} tier=${run.tier.padEnd(6)} expected=${(run.expectedTier ?? "n/a").padEnd(6)} mode=${run.requestedOutputMode}/${run.actualOutputMode} valid=${run.outputValidation} disposition=${run.quality?.summaryDisposition.actual ?? "n/a"}/${run.expectedSummaryDisposition} span=${String(run.analysis.eventSpan).padEnd(3)} prompts=${run.analysis.promptCount} tools=${String(run.analysis.toolCount).padEnd(2)} transcript=${run.analysis.transcriptLength} ${run.provider}/${run.model} [${run.transport}] initial=${run.initial.summaries}s/${run.initial.observations}o final=${run.summaries}s/${run.observations}o quality=${qualityLabel} coverage=${run.quality?.weightedQualityCoverage?.toFixed(3) ?? "n/a"} required_missing=${missingRequired} cost=${costLabel} latency=${latencyLabel} schema_loss=${run.initial.diagnostics?.dataLoss === true ? "yes" : "no"} fallback=${run.modelFallbackApplied ? "yes" : "no"} repair=${run.repairApplied ? "yes" : "no"} retry=${run.retryAttempted ? (run.retryReason ?? "yes") : "no"} — ${run.label}`, ); } for (const failure of outputFailures.toSorted( (left, right) => left.iteration - right.iteration || left.batchId - right.batchId, )) { const costLabel = failure.cost.total == null ? "n/a" : `$${failure.cost.total.totalCostUsd.toFixed(6)}`; const latencyLabel = failure.telemetry.totalElapsedMs == null ? "n/a" : `${failure.telemetry.totalElapsedMs}ms`; p.log.error( ` [${failure.batchId}#${failure.iteration}] output_failure tier=${failure.tier ?? "n/a"} expected=${failure.expectedTier ?? "n/a"} mode=${failure.requestedOutputMode}/${failure.actualOutputMode} model=${failure.model ?? "unresolved"} cost=${costLabel} latency=${latencyLabel} — ${failure.reason}`, ); } p.outro("done"); } catch (error) { const message = error instanceof Error ? error.message : "Extraction benchmark failed"; if (opts.json) { emitJsonError("extraction_benchmark_failed", message); } else { p.log.error(message); process.exitCode = 1; } return; } }, ); return cmd; } function createMemoryRelinkReportCommand(): Command { const cmd = new Command("relink-report") .configureHelp(helpStyle) .description("Report session relinking opportunities without changing data") .option("--project <project>", "project name or path (defaults to git repo root)") .option("--all-projects", "analyze across all projects") .option("--limit <n>", "max groups to print", "25"); addDbOption(cmd); addJsonOption(cmd); cmd.action( ( opts: DbOpts & JsonOpts & { project?: string; allProjects?: boolean; limit?: string; }, ) => { const project = opts.allProjects === true ? null : opts.project?.trim() || process.env.CODEMEM_PROJECT?.trim() || resolveProject(process.cwd(), null); const limit = Number.parseInt(opts.limit ?? "25", 10) || 25; const result = getRawEventRelinkReport(resolveDbOpt(opts), { project, allProjects: opts.allProjects === true, limit, }); if (opts.json) { console.log(JSON.stringify(result, null, 2)); return; } p.intro("codemem diagnostics memory relink"); p.log.info( [ `Recoverable sessions: ${result.totals.recoverable_sessions}`, `Distinct stable ids: ${result.totals.distinct_stable_ids}`, `Groups with multiple sessions: ${result.totals.groups_with_multiple_sessions}`, `Groups with mapped session: ${result.totals.groups_with_mapped_session}`, `Groups without mapped session: ${result.totals.groups_without_mapped_session}`, `Active memories in groups: ${result.totals.active_memories}`, `Repointable active memories: ${result.totals.repointable_active_memories}`, ].join("\n"), ); p.log.info("Top relink groups:"); for (const group of result.groups) { p.log.message( ` ${group.stable_id} -> canonical ${group.canonical_session_id} | local=${group.local_sessions} mapped=${group.mapped_sessions} unmapped=${group.unmapped_sessions} active=${group.active_memories} repointable=${group.repointable_active_memories}`, ); } p.outro("done"); }, ); return cmd; } function createMemoryRelinkPlanCommand(): Command { const cmd = new Command("relink-plan") .configureHelp(helpStyle) .description("Plan session relinking without changing data") .option("--project <project>", "project name or path (defaults to git repo root)") .option("--all-projects", "analyze across all projects") .option("--limit <n>", "max groups to include", "25"); addDbOption(cmd); addJsonOption(cmd); cmd.action( ( opts: DbOpts & JsonOpts & { project?: string; allProjects?: boolean; limit?: string; }, ) => { const project = opts.allProjects === true ? null : opts.project?.trim() || process.env.CODEMEM_PROJECT?.trim() || resolveProject(process.cwd(), null); const limit = Number.parseInt(opts.limit ?? "25", 10) || 25; const result = getRawEventRelinkPlan(resolveDbOpt(opts), { project, allProjects: opts.allProjects === true, limit, }); if (opts.json) { console.log(JSON.stringify(result, null, 2)); return; } p.intro("codemem diagnostics memory relink-plan"); p.log.info( [ `Groups: ${result.totals.groups}`, `Eligible groups: ${result.totals.eligible_groups}`, `Skipped groups: ${result.totals.skipped_groups}`, `Actions: ${result.totals.actions}`, `Bridge creations: ${result.totals.bridge_creations}`, `Memory repoints: ${result.totals.memory_repoints}`, `Session compactions: ${result.totals.session_compactions}`, ].join("\n"), ); p.log.info("Planned actions:"); for (const action of result.actions.slice(0, 15)) { p.log.message( ` ${action.action} ${action.stable_id} -> canonical ${action.canonical_session_id} | sessions=${action.session_ids.join(",") || "-"} memories=${action.memory_count} reason=${action.reason}`, ); } if (result.skipped_groups.length > 0) { p.log.info("Skipped groups:"); for (const group of result.skipped_groups.slice(0, 10)) { p.log.message(` ${group.stable_id} | blockers=${group.blockers.join(",")}`); } } p.outro("done"); }, ); return cmd; } export const showMemoryCommand = createShowMemoryCommand(); export const forgetMemoryCommand = createForgetMemoryCommand(); export const rememberMemoryCommand = createRememberMemoryCommand(); export function createMemoryCommand(): Command { // Stop at the child command so context trace receives its own flags. return new Command("memory") .enablePositionalOptions() .configureHelp(helpStyle) .description("Manage memories and build task context") .addCommand(createRecentCommand().name("list")) .addCommand(createSearchCommand()) .addCommand(createShowMemoryCommand()) .addCommand(createRememberMemoryCommand()) .addCommand(createForgetMemoryCommand()) .addCommand(createContextCommand()) .addCommand(createPackCommand(), { hidden: true }) .addCommand(createDistillCommand()) .addCommand(createStatsCommand()) .addCommand(createExportMemoriesCommand().name("export")) .addCommand(createImportMemoriesCommand().name("import")); } export function createMemoryDiagnosticsCommand(): Command { return new Command("memory") .configureHelp(helpStyle) .description("Inspect memory quality and session links") .addCommand(createMemoryRoleReportCommand().name("roles")) .addCommand(createMemoryRoleCompareCommand().name("compare")) .addCommand(createMemoryArtifactReportCommand().name("artifacts")) .addCommand(createMemoryRelinkReportCommand().name("relink")) .addCommand(createMemoryRelinkPlanCommand()); } export function createExtractionDiagnosticsCommand(): Command { return new Command("extraction") .configureHelp(helpStyle) .description("Evaluate memory extraction") .addCommand(createMemoryExtractionReportCommand().name("report")) .addCommand(createMemoryExtractionReplayCommand().name("replay")) .addCommand(createMemoryExtractionBenchmarkCommand().name("benchmark")); } export const memoryCommand = createMemoryCommand();