#!/usr/bin/env bash
# scripts/refresh-hant-data.sh — re-vendor the Traditional Chinese title datasets
# into go-api/data/hant/.
#
# Two upstreams, both fetched pinned to an exact revision so a run is reproducible:
#   1. soruly/anilist-chinese (MIT) — keyed by AniList id, the trunk.
#      https://github.com/soruly/anilist-chinese
#   2. zh.wikipedia Module:CGroup/Anime (CC BY-SA 4.0) — keyed by title strings,
#      the Hong Kong overlay.
#      https://zh.wikipedia.org/wiki/Module:CGroup/Anime
#
# Refs:
#   GitHub commits API — resolve master to a SHA before downloading, so the raw
#     fetch cannot race an upstream push (anilist-chinese rebuilds daily)
#     https://docs.github.com/en/rest/commits/commits
#   MediaWiki revisions API + action=raw&oldid — same trick for the wiki page
#     https://www.mediawiki.org/wiki/API:Revisions
#   Wikimedia User-Agent policy — identify the client on every request
#     https://foundation.wikimedia.org/wiki/Policy:User-Agent_policy
#
# Determinism: two runs against unchanged upstreams produce byte-identical output.
# Everything is sorted, key order is pinned, no timestamps are written into the
# data files, and every file ends with a newline. `--check` proves it in CI.

set -euo pipefail

MODE="write"
for arg in "$@"; do
  case "$arg" in
    --check) MODE="check" ;;
    -h|--help)
      cat <<'USAGE'
Usage: scripts/refresh-hant-data.sh [--check]

  (no args)  Download both upstreams and rewrite go-api/data/hant/*.json.
  --check    Build into a temp dir and diff against the committed files.
             Writes nothing; exits 1 if they differ. Use this in CI.
USAGE
      exit 0 ;;
    *) echo "Unknown arg: $arg" >&2; exit 2 ;;
  esac
done

log() { printf '[refresh-hant] %s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >&2; }
die() { log "FATAL: $*"; exit 1; }

REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
OUT_DIR="$REPO_ROOT/go-api/data/hant"

AC_REPO="soruly/anilist-chinese"
AC_FILE="anilist-chinese.json"
CG_PAGE="Module:CGroup/Anime"
CG_WIKI="zh.wikipedia.org"
UA="animego-refresh-hant-data/1.0 (+https://github.com/lawrenceli0228/animego)"

# Truncation floors. Upstream sizes at time of writing: 854 KB and 110 KB.
# A half-written download or an error page lands well under these.
AC_MIN_BYTES=400000
CG_MIN_BYTES=50000

command -v curl    >/dev/null || die "curl not found"
command -v python3 >/dev/null || die "python3 not found"

TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/refresh-hant.XXXXXX")"
trap 'rm -rf "$TMP_DIR"' EXIT

# fetch <url> <dest> <min-bytes> [extra curl args...]
# Downloads to <dest> only if the request returns 200 AND the body clears the
# floor. Nothing is written to the repo here, so a bad fetch can never truncate
# a committed file.
fetch() {
  local url="$1" dest="$2" min="$3"; shift 3
  local code size
  code="$(curl -sS -L --max-time 120 --retry 3 --retry-delay 2 \
            -A "$UA" -o "$dest" -w '%{http_code}' "$@" "$url")" \
    || die "curl failed for $url"
  [[ "$code" == "200" ]] || die "$url returned HTTP $code (expected 200)"
  size="$(wc -c < "$dest" | tr -d ' ')"
  [[ "$size" -ge "$min" ]] \
    || die "$url returned $size bytes, below the $min-byte floor — refusing to use a truncated download"
  log "fetched $url ($size bytes)"
}

# --- 1. anilist-chinese: resolve master -> SHA, then fetch that SHA -----------
log "resolving $AC_REPO master"
fetch "https://api.github.com/repos/$AC_REPO/commits/master" "$TMP_DIR/ac-commit.json" 100 \
  -H 'Accept: application/vnd.github+json'
AC_SHA="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["sha"])' "$TMP_DIR/ac-commit.json")"
AC_DATE="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["commit"]["committer"]["date"])' "$TMP_DIR/ac-commit.json")"
[[ "$AC_SHA" =~ ^[0-9a-f]{40}$ ]] || die "bad commit sha from GitHub: $AC_SHA"
fetch "https://raw.githubusercontent.com/$AC_REPO/$AC_SHA/$AC_FILE" "$TMP_DIR/ac.json" "$AC_MIN_BYTES"

# --- 2. CGroup/Anime: resolve latest revid, then fetch that oldid -------------
log "resolving $CG_PAGE latest revision"
fetch "https://$CG_WIKI/w/api.php?action=query&prop=revisions&titles=$(printf '%s' "$CG_PAGE" | sed 's|:|%3A|; s|/|%2F|')&rvprop=ids%7Ctimestamp&rvlimit=1&format=json&formatversion=2" \
  "$TMP_DIR/cg-rev.json" 50
CG_REVID="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["query"]["pages"][0]["revisions"][0]["revid"])' "$TMP_DIR/cg-rev.json")"
CG_DATE="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["query"]["pages"][0]["revisions"][0]["timestamp"])' "$TMP_DIR/cg-rev.json")"
[[ "$CG_REVID" =~ ^[0-9]+$ ]] || die "bad revid from MediaWiki: $CG_REVID"
fetch "https://$CG_WIKI/w/index.php?title=Module%3ACGroup%2FAnime&action=raw&oldid=$CG_REVID" \
  "$TMP_DIR/cg.lua" "$CG_MIN_BYTES"

# --- 3. Normalise into the two vendored JSON files ---------------------------
python3 - "$TMP_DIR/ac.json" "$TMP_DIR/cg.lua" \
           "$TMP_DIR/anilist-chinese.json" "$TMP_DIR/cgroup-hk.json" <<'PY'
# -*- coding: utf-8 -*-
"""Normalise both upstreams into the shapes go-api/internal consumes.

Output contract (both files):
  * one JSON object per line inside a JSON array — compact, and git diffs stay
    line-level instead of reflowing the whole file;
  * key order pinned by construction order, never sorted alphabetically;
  * UTF-8, LF, trailing newline, no BOM.
"""
import io
import json
import re
import sys
from collections import Counter, defaultdict

HAN = re.compile("[㐀-䶿一-鿿豈-﫿]")

# Locale tags used by MediaWiki's Chinese conversion tables.
LANGS = frozenset(("zh", "zh-hans", "zh-hant", "zh-cn", "zh-tw",
                   "zh-hk", "zh-mo", "zh-sg", "zh-my"))
# Real entries always begin at column 0. Anchoring here is what skips the
# usage comment on line 1 -- "-- 用法： Item('原文', '轉換規則')" -- which a
# naive scan for "Item(" picks up as a phantom 792nd entry.
ITEM_RE = re.compile(r"^Item\(\s*(nil|'[^']*'|\"[^\"]*\")\s*,\s*('[^']*'|\"[^\"]*\")\s*\)")
H2_RE = re.compile(r"^==\s*(.+?)\s*==$")
H3_RE = re.compile(r"^===\s*(.+?)\s*===$")
WORKS_SECTION = "作品名"  # 作品名 -- the work-titles section

MIN_ANILIST_ROWS = 5000
MIN_CGROUP_ITEMS = 500


def write_json_lines(path, records):
    out = io.StringIO()
    out.write("[\n")
    last = len(records) - 1
    for i, rec in enumerate(records):
        out.write(json.dumps(rec, ensure_ascii=False, separators=(",", ":")))
        out.write("\n" if i == last else ",\n")
    out.write("]\n")
    with io.open(path, "w", encoding="utf-8", newline="\n") as fh:
        fh.write(out.getvalue())


def build_anilist(src, dst):
    """{id, title, synonyms} -> sorted by id, trimmed, deduped, blanks dropped.

    synonyms are kept: 776 rows have a Latin-only `title` whose only Chinese
    name lives in `synonyms`, so dropping the field would silently delete those
    rows' entire contribution. Upstream synonym order is preserved rather than
    sorted -- it encodes the curator's preference, and the file order is already
    stable input, so determinism costs nothing here.
    """
    rows = json.load(io.open(src, encoding="utf-8"))
    if not isinstance(rows, list) or len(rows) < MIN_ANILIST_ROWS:
        raise SystemExit("anilist-chinese.json failed its sanity check")

    out = []
    trimmed = blank = dupe = same_as_title = dropped = 0
    for row in rows:
        original = row.get("title") or ""
        title = original.strip()
        if title != original:
            trimmed += 1
        seen, synonyms = set(), []
        for value in row.get("synonyms") or []:
            value = value.strip()
            if not value:
                blank += 1
            elif value == title:
                same_as_title += 1
            elif value in seen:
                dupe += 1
            else:
                seen.add(value)
                synonyms.append(value)
        if not title and not synonyms:
            dropped += 1        # an id with no string at all: nothing to backfill
            continue
        record = {"id": row["id"], "title": title}
        if synonyms:
            record["synonyms"] = synonyms
        out.append(record)

    out.sort(key=lambda r: r["id"])
    write_json_lines(dst, out)

    han = sum(1 for r in out if HAN.search(r["title"]))
    rescued = sum(1 for r in out if not HAN.search(r["title"])
                  and any(HAN.search(s) for s in r.get("synonyms", ())))
    print("[anilist] in=%d out=%d (dropped %d id-only rows)" % (len(rows), len(out), dropped))
    print("[anilist] CJK title=%d  Latin-only title=%d (%d rescued by a CJK synonym, %d with none)"
          % (han, len(out) - han, rescued, len(out) - han - rescued))
    print("[anilist] rows carrying synonyms=%d  synonyms=%d"
          % (sum(1 for r in out if "synonyms" in r),
             sum(len(r.get("synonyms", ())) for r in out)))
    print("[anilist] normalised: %d titles trimmed, %d blank / %d duplicate / "
          "%d title-equal synonyms dropped" % (trimmed, blank, dupe, same_as_title))


def parse_cgroup(src):
    """Item('原文', 'zh-hk:值;原詞=>zh-tw:值;...') -> {zh_hk, keys, section}.

    Two things a regex-only scan gets wrong and this does not: the `原詞=>`
    prefix carries a *source word* that is itself a legitimate join key, and
    MediaWiki leftovers such as `}--{-|TOUCH` are not lang:value pairs and must
    be discarded rather than merged into the previous locale.
    """
    items, section = [], None
    for line in io.open(src, encoding="utf-8"):
        line = line.rstrip("\n")
        stripped = line.strip()
        if H3_RE.match(stripped):
            continue                        # subsection; the level-2 heading still governs
        heading = H2_RE.match(stripped)
        if heading:
            section = heading.group(1)
            continue
        if not line.startswith("Item("):
            continue
        matched = ITEM_RE.match(line)
        if not matched:
            raise SystemExit("unparsed Item(): %s" % line[:120])

        raw = None if matched.group(1) == "nil" else matched.group(1)[1:-1]
        keys, hk_plain, hk_conditional = set(), None, []
        for segment in matched.group(2)[1:-1].split(";"):
            segment = segment.strip()
            if not segment:
                continue
            conditional = "=>" in segment
            if conditional:
                source_word, segment = segment.split("=>", 1)
                source_word = source_word.strip()
                if source_word:
                    keys.add(source_word)
            if ":" not in segment:
                continue                    # e.g. }--{-|TOUCH
            lang, value = segment.split(":", 1)
            lang, value = lang.strip().lower(), value.strip()
            if lang not in LANGS or not value:
                continue
            keys.add(value)
            if lang == "zh-hk":
                if conditional:
                    hk_conditional.append(value)
                elif hk_plain is None:
                    hk_plain = value
        if raw:
            keys.add(raw)

        zh_hk = hk_plain
        if zh_hk is None and hk_conditional:
            zh_hk = hk_conditional[0]
        items.append((zh_hk, section, keys))
    return items


def build_cgroup(src, dst):
    """Keep work titles that carry a zh-hk value; emit every possible join key."""
    items = parse_cgroup(src)
    if len(items) < MIN_CGROUP_ITEMS:
        raise SystemExit("CGroup parse produced only %d items" % len(items))

    kept, seen = [], set()
    no_hk = off_section = duplicate = 0
    for zh_hk, section, keys in items:
        if not zh_hk:
            no_hk += 1              # zh-tw / zh-hant only: nothing for an HK overlay
        elif section != WORKS_SECTION:
            off_section += 1        # 術語 / 人名: not titles, joining them invents matches
        else:
            signature = (zh_hk, tuple(sorted(keys)))
            if signature in seen:
                duplicate += 1
            else:
                seen.add(signature)
                kept.append({"zh_hk": zh_hk, "keys": sorted(keys)})

    kept.sort(key=lambda r: (r["zh_hk"], r["keys"]))
    write_json_lines(dst, kept)

    per_key = defaultdict(set)
    for record in kept:
        for key in record["keys"]:
            per_key[key].add(record["zh_hk"])
    ambiguous = sorted(k for k, v in per_key.items() if len(v) > 1)
    sections = Counter(section for _, section, _ in items)
    print("[cgroup] Item() calls=%d  per section=%s"
          % (len(items), dict(sorted(sections.items(), key=lambda kv: -kv[1]))))
    print("[cgroup] kept=%d  dropped: %d without zh-hk, %d outside %s, %d identical duplicates"
          % (len(kept), no_hk, off_section, WORKS_SECTION, duplicate))
    print("[cgroup] distinct join keys=%d  ambiguous (one key, several zh-hk)=%d"
          % (len(per_key), len(ambiguous)))
    if ambiguous:
        print("[cgroup] ambiguous keys: %s" % ", ".join(ambiguous))


build_anilist(sys.argv[1], sys.argv[3])
build_cgroup(sys.argv[2], sys.argv[4])
PY

# --- 4. Install, or diff in --check mode -------------------------------------
if [[ "$MODE" == "check" ]]; then
  status=0
  for name in anilist-chinese.json cgroup-hk.json; do
    if diff -q "$OUT_DIR/$name" "$TMP_DIR/$name" >/dev/null 2>&1; then
      log "OK    $name matches a fresh build"
    else
      log "DRIFT $name differs from a fresh build"
      status=1
    fi
  done
  [[ "$status" -eq 0 ]] || die "committed data is stale — re-run without --check"
  log "check passed"
  exit 0
fi

mkdir -p "$OUT_DIR"
for name in anilist-chinese.json cgroup-hk.json; do
  mv "$TMP_DIR/$name" "$OUT_DIR/$name"
  log "wrote $OUT_DIR/$name ($(wc -c < "$OUT_DIR/$name" | tr -d ' ') bytes)"
done

# Provenance is not written into the data files — a timestamp in there would
# break byte-for-byte reproducibility. Paste this block into the README instead.
cat >&2 <<EOF

  Paste into go-api/data/hant/README.md:

  | anilist-chinese.json | $AC_REPO@\`${AC_SHA:0:12}\` | committed $AC_DATE |
  | cgroup-hk.json       | $CG_PAGE oldid=$CG_REVID | revised $CG_DATE |

  fetched $(date -u +%Y-%m-%d)
EOF
log "done"