// Story-digest evaluation cases, one event per JSONL line, written by hand or exported from the database
// (scripts/eval-story-digests.ts --stories); the site's digest input made from a case; and the comparison
// a run writes, as Markdown for reading.
import { z } from "zod";
import { byReportTime } from "@aihot/backend/events/digest";
import type { DigestReport } from "@aihot/backend/publication/story-evidence";
import { parseCasesJsonl } from "./eval-tools.ts";

const ReportSchema = z.strictObject({
  id: z.string().min(1),
  publishedAt: z.iso.datetime({ offset: true }),
  source: z.string().min(1),
  firstParty: z.boolean().default(false),
  title: z.string().min(1),
  summary: z.string().nullable().default(null),
  fact: z.strictObject({
    id: z.number().int().positive(),
    subject: z.string().nullable().default(null),
    action: z.string().nullable().default(null),
    object: z.string().nullable().default(null),
    conditions: z.string().nullable().default(null),
    evidence: z.string().nullable().default(null),
    /** The analysis's structured fact: its conditions with source quotes, and its evidence. */
    structured: z.unknown().default(null),
  }),
});

const CaseSchema = z.strictObject({
  caseId: z.string().min(1),
  story: z.strictObject({ title: z.string().min(1), previousDigest: z.string().nullable().default(null) }),
  /**
   * incremental: the site's update, with previousDigest (if any) and the reports not in knownArticleIds
   * marked new; with neither, an event's first digest. corrected: rewritten from the reports as they are
   * now, without the previous digest, as after a correction.
   */
  inputMode: z.enum(["incremental", "corrected"]).default("incremental"),
  knownArticleIds: z.array(z.string().min(1)).default([]),
  reports: z.array(ReportSchema).min(1),
});

export type DigestEvalCase = z.infer<typeof CaseSchema>;

export const parseDigestEvalJsonl = (text: string): DigestEvalCase[] => parseCasesJsonl(text, CaseSchema);

/**
 * An event as the site rewrites its digest from its current reports (digestReports): without the
 * previous digest when it has saved one, the way a correction is rewritten; as a first digest otherwise.
 */
export function caseFromStory(story: { publicId: string; title: string; hasDigest: boolean }, reports: DigestReport[]): DigestEvalCase {
  return {
    caseId: story.publicId,
    story: { title: story.title, previousDigest: null },
    inputMode: story.hasDigest ? "corrected" : "incremental",
    knownArticleIds: [],
    reports: [...reports].sort(byReportTime).map((report) => ({
      id: report.id,
      publishedAt: report.at.toISOString(),
      source: report.source_name,
      firstParty: report.first_party,
      title: report.title,
      summary: report.summary,
      fact: {
        id: report.fact_id,
        subject: report.fact_subject,
        action: report.fact_action,
        object: report.fact_object,
        conditions: report.fact_conditions,
        evidence: report.evidence,
        structured: report.structured_fact ?? null,
      },
    })),
  };
}

/** The case as buildStoryDigestInput's arguments. */
export function toDigestInput(row: DigestEvalCase) {
  return {
    story: { title: row.story.title, digest: row.story.previousDigest },
    reports: row.reports.map((report): Omit<DigestReport, "story_id"> => ({
      id: report.id,
      title: report.title,
      summary: report.summary,
      source_name: report.source,
      first_party: report.firstParty,
      at: new Date(report.publishedAt),
      fact_id: report.fact.id,
      fact_subject: report.fact.subject,
      fact_action: report.fact.action,
      fact_object: report.fact.object,
      fact_conditions: report.fact.conditions,
      evidence: report.fact.evidence,
      structured_fact: report.fact.structured,
    })).sort(byReportTime),
    corrected: row.inputMode === "corrected",
    knownArticleIds: row.knownArticleIds,
  };
}

export function inputForm(row: DigestEvalCase): string {
  if (row.inputMode === "corrected") return "更正后重写（不带上一版综述）";
  return row.story.previousDigest ? "增量更新（带上一版综述）" : "首次撰写";
}

export interface DigestCall {
  title: string | null;
  digest: string | null;
  receiptId: number | null;
  reused: boolean;
  tokensIn: number;
  tokensOut: number;
  avgLatencyMs: number;
  error: string | null;
}

export interface DigestPromptSummary {
  calls: number;
  succeeded: number;
  errors: number;
  reused: number;
  tokensIn: number;
  tokensOut: number;
  avgLatencyMs: number;
}

/** What a run writes: per model, each case's call with the site's prompt beside the candidate's. */
export interface DigestEvalReport {
  meta: {
    createdAt: string;
    cases: string;
    calls: number;
    prompts: { live: { version: string }; candidate?: { file: string; version: string } };
  };
  models: Record<string, {
    summary: { live: DigestPromptSummary; candidate?: DigestPromptSummary; wallSeconds: number };
    cases: Array<{ caseId: string; storyTitle: string; input: string; reports: number; live: DigestCall; candidate?: DigestCall }>;
  }>;
}

const PROMPTS = [["live", "线上提示词"], ["candidate", "候选提示词"]] as const;

const usage = (call: { receiptId: number | null; reused: boolean; tokensIn: number; tokensOut: number }) =>
  `${call.receiptId === null ? "无回执" : `回执 #${call.receiptId}`} · 输入 ${call.tokensIn} tokens · 输出 ${call.tokensOut} tokens${call.reused ? " · 复用已收到的结果" : ""}`;

/** The report for reading: every case's answers one after another, line breaks kept. */
export function digestComparisonMarkdown(report: DigestEvalReport): string {
  const { meta } = report;
  const lines = [
    "# 事件综述提示词对比",
    "",
    `- 生成时间：${meta.createdAt}`,
    `- 案例：${meta.cases}`,
    `- 线上提示词：${meta.prompts.live.version}`,
    ...(meta.prompts.candidate ? [`- 候选提示词：${meta.prompts.candidate.file}（${meta.prompts.candidate.version}）`] : []),
    `- 模型调用：${meta.calls} 次`,
  ];
  for (const [model, run] of Object.entries(report.models)) {
    for (const [key, label] of PROMPTS) {
      const summary = run.summary[key];
      if (summary) lines.push(`- ${model} · ${label}：成功 ${summary.succeeded}/${summary.calls}，输入 ${summary.tokensIn} tokens，输出 ${summary.tokensOut} tokens，${summary.reused} 次复用已收到的结果`);
    }
  }
  const runs = Object.entries(report.models);
  for (const [index, item] of (runs[0]?.[1].cases ?? []).entries()) {
    lines.push("", `## ${index + 1}. ${item.storyTitle}`, "", `案例 \`${item.caseId}\` · 报道 ${item.reports} 篇 · ${item.input}`);
    for (const [model, run] of runs) {
      for (const [key, label] of PROMPTS) {
        const call = run.cases[index]![key];
        if (!call) continue;
        lines.push("", `### ${label} · ${model}`, "");
        if (call.error !== null || call.digest === null) lines.push(`生成失败：${call.error}`);
        else lines.push(`**${call.title || "（沿用事件标题）"}**`, "", ...call.digest.split("\n").map((line) => (line ? `> ${line}  ` : ">")));
        lines.push("", usage(call));
      }
    }
  }
  return `${lines.join("\n")}\n`;
}