// The importer (server/deckImport.ts) and the ZIP reader under it
// (server/zip.ts): an Anki package BUILT HERE — a real SQLite collection
// written with node:sqlite, zipped with headers written by hand — goes in,
// and the deck notes the spec describes come out, byte for byte.
//
// The archive is written by this file and not by shared/zip.ts on purpose:
// a reader tested only against our own writer would prove the two agree,
// not that either matches the format. The hand-written builder deflates
// its entries (Anki's do) and puts every size in the local header (Anki's
// do); shared/zip.ts stores and uses a data descriptor, and a second test
// feeds the reader that shape too, so both halves of the format get read.
import assert from "node:assert/strict";
import { existsSync, readFileSync } from "node:fs";
import path from "node:path";
import { after, before, describe, it } from "node:test";
import { deflateRawSync } from "node:zlib";
import { crc32, zipSync } from "../shared/zip.ts";
import { scanCards } from "../shared/cards.ts";
import { listZip, readZipEntry, zipIndex, ZipError } from "../server/zip.ts";
import {
ankiSchedule,
clozeAsQa,
clozeLine,
clozeOrdinals,
clozeReadable,
csvDeck,
fieldText,
looksLikeHeader,
mediaNamesFromProto,
noteBaseName,
parseDelimited,
readApkg,
serialiseImportedDeck,
Skips,
writeDecks,
type ImportDeck,
} from "../server/deckImport.ts";
import { initIndexer } from "../server/indexer.ts";
import { initSite } from "../server/site.ts";
import { initVault } from "../server/vault.ts";
import { makeDir, makeVault, removeVault } from "./helpers/vault.ts";
const enc = new TextEncoder();
const dec = new TextDecoder();
// ── A ZIP writer of the OTHER kind: deflate, sizes up front ─────────────
function le16(n: number): number[] {
return [n & 0xff, (n >>> 8) & 0xff];
}
function le32(n: number): number[] {
return [n & 0xff, (n >>> 8) & 0xff, (n >>> 16) & 0xff, (n >>> 24) & 0xff];
}
/** An archive with deflated entries and no data descriptor — the shape
* Anki (and `zip`) write. Names are UTF-8 with bit 11 set. */
function zipDeflated(entries: Array<{ name: string; data: Uint8Array; store?: boolean }>): Uint8Array {
const parts: number[] = [];
const central: number[] = [];
for (const entry of entries) {
const name = [...enc.encode(entry.name)];
const packed = entry.store ? entry.data : new Uint8Array(deflateRawSync(entry.data));
const crc = crc32(entry.data);
const offset = parts.length;
const method = entry.store ? 0 : 8;
parts.push(
...le32(0x04034b50), ...le16(20), ...le16(1 << 11), ...le16(method), ...le16(0), ...le16(0x21),
...le32(crc), ...le32(packed.length), ...le32(entry.data.length), ...le16(name.length), ...le16(0),
...name, ...packed,
);
central.push(
...le32(0x02014b50), ...le16(20), ...le16(20), ...le16(1 << 11), ...le16(method), ...le16(0), ...le16(0x21),
...le32(crc), ...le32(packed.length), ...le32(entry.data.length), ...le16(name.length), ...le16(0), ...le16(0),
...le16(0), ...le16(0), ...le32(0), ...le32(offset), ...name,
);
}
const dirAt = parts.length;
const end = [
...le32(0x06054b50), ...le16(0), ...le16(0), ...le16(entries.length), ...le16(entries.length),
...le32(central.length), ...le32(dirAt), ...le16(0),
];
return new Uint8Array([...parts, ...central, ...end]);
}
describe("server/zip.ts — the reader", () => {
it("reads deflated entries written with sizes in the local header", () => {
const text = enc.encode("سلام ".repeat(200));
const bytes = zipDeflated([
{ name: "ملاحظة.txt", data: text },
{ name: "raw.bin", data: new Uint8Array([1, 2, 3, 4]), store: true },
]);
const entries = listZip(bytes);
assert.deepEqual(entries.map((e) => [e.name, e.method, e.size]), [["ملاحظة.txt", 8, text.length], ["raw.bin", 0, 4]]);
assert.ok(entries[0].compressedSize < text.length, "the text was actually deflated");
assert.equal(dec.decode(readZipEntry(bytes, entries[0])), "سلام ".repeat(200));
assert.deepEqual([...readZipEntry(bytes, entries[1])], [1, 2, 3, 4]);
});
it("reads our own writer's stored entries, whose local header carries no CRC", () => {
const bytes = zipSync([{ name: "a/b.md", data: enc.encode("# hi\n") }]);
const index = zipIndex(bytes);
assert.equal(dec.decode(readZipEntry(bytes, index.get("a/b.md")!)), "# hi\n");
});
it("finds the end record behind an archive comment and refuses what is not a zip", () => {
const plain = zipDeflated([{ name: "x", data: enc.encode("x") }]);
const commented = new Uint8Array([...plain.subarray(0, plain.length - 2), ...le16(9), ...enc.encode("PK\x05\x06xxxxx")]);
assert.equal(listZip(commented).length, 1);
assert.throws(() => listZip(enc.encode("not a zip at all, but long enough to search")), ZipError);
});
it("names an unsupported method rather than returning garbage", () => {
const bytes = zipDeflated([{ name: "x", data: enc.encode("x") }]);
const entry = { ...listZip(bytes)[0], method: 12 };
assert.throws(() => readZipEntry(bytes, entry), /method 12/);
});
});
// ── Field text, cloze, schedule: the pure pieces ─────────────────────────
describe("fieldText", () => {
it("turns Anki's HTML into one line of the vault's Markdown", () => {
assert.equal(fieldText("cat
& kitten
a cat [sound:meow.mp3]'), "![[cat.png]] a cat ![[meow.mp3]]");
assert.equal(fieldText("bold <tag> 🐱 A"), "bold
'), "");
});
});
describe("cloze", () => {
const text = "The speed of light is {{c1::299,792 km/s}} in {{c2::vacuum::medium}}.";
it("lists the deletions once each, in order", () => {
assert.deepEqual(clozeOrdinals(text), [1, 2]);
assert.deepEqual(clozeOrdinals("{{c2::a}} {{c1::b}} {{c2::c}}"), [2, 1]);
});
it("marks one deletion as a highlight and shows the rest plain, hints dropped", () => {
assert.equal(clozeLine(text, 1), "The speed of light is ==299,792 km/s== in vacuum.");
assert.equal(clozeLine(text, 2), "The speed of light is 299,792 km/s in ==vacuum==.");
});
it("keeps a `::` in the text from becoming a card separator", () => {
assert.equal(clozeLine("{{c1::x}} in std::vector", 1), "==x== in std: :vector");
// Every colon that touches another, not every pair: `:::` softened
// pairwise left `: ::`, and the scanner split the line there.
assert.equal(clozeLine("{{c1::x}} a:::b", 1), "==x== a: : :b");
assert.deepEqual(scanCards("==x== a: : :b\n").map((c) => c.kind), ["cloze"]);
});
it("turns a deletion the vault cannot highlight into a plain card, so nothing is lost", () => {
const eq = "Einstein: {{c1::E = mc²}} in {{c2::1905}}";
assert.ok(!clozeReadable(eq, 1), "an `=` inside `==…==` is no highlight to shared/cards.ts");
assert.ok(clozeReadable(eq, 2));
assert.deepEqual(clozeAsQa(eq, 1), { front: "Einstein: **[…]** in 1905", back: "E = mc²" });
assert.ok(!clozeReadable(`{{c1::${"x".repeat(201)}}}`, 1), "a highlight is at most 200 characters");
assert.ok(!clozeReadable("{{c1::}} empty", 1));
// Two deletions with one number are one card with two answers.
assert.deepEqual(clozeAsQa("{{c1::a=1}} and {{c1::b=2}}", 1), { front: "**[…]** and **[…]**", back: "a=1 · b=2" });
});
});
describe("ankiSchedule", () => {
const crt = Math.floor(new Date(2024, 0, 1, 4).getTime() / 1000);
const base = { queue: 2, odue: 0, odid: 0, ivl: 4, factor: 2500 };
it("gives a new card no schedule", () => {
assert.equal(ankiSchedule({ ...base, type: 0, queue: 0, due: 3 }, { crt }), null);
});
it("counts a review card's due in days from the collection's creation day", () => {
assert.deepEqual(ankiSchedule({ ...base, type: 2, due: 10 }, { crt }), { due: "2024-01-11", interval: 4, ease: 2500 });
});
it("reads a learning card's due as seconds and gives it a day", () => {
const due = Math.floor(new Date(2024, 1, 1, 10).getTime() / 1000);
assert.deepEqual(ankiSchedule({ ...base, type: 1, queue: 1, due, ivl: 0, factor: 0 }, { crt }), { due: "2024-02-01", interval: 1, ease: 2500 });
});
it("uses the original due of a card parked in a filtered deck, and floors the ease", () => {
assert.deepEqual(ankiSchedule({ ...base, type: 2, due: -100000, odue: 2, odid: 99, factor: 1100 }, { crt }), { due: "2024-01-03", interval: 4, ease: 1300 });
});
});
describe("mediaNamesFromProto", () => {
it("reads the names out of a recent export's media index", () => {
// MediaEntries{ entries: [ {name:"a.png", size: 3}, {name:"b.mp3"} ] }
const entry = (name: string, extra: number[] = []): number[] => {
const n = [...enc.encode(name)];
const body = [0x0a, n.length, ...n, ...extra];
return [0x0a, body.length, ...body];
};
const bytes = new Uint8Array([...entry("a.png", [0x10, 3]), ...entry("b.mp3")]);
assert.deepEqual(mediaNamesFromProto(bytes), ["a.png", "b.mp3"]);
});
});
// ── CSV ──────────────────────────────────────────────────────────────────
describe("CSV / TSV", () => {
it("parses quoted fields, embedded newlines and a BOM, and picks the delimiter", () => {
const { rows, delimiter } = parseDelimited('a,"b, c","say ""hi""\nthere"\n\nd,e,f\r\n');
assert.equal(delimiter, ",");
assert.deepEqual(rows, [["a", "b, c", 'say "hi"\nthere'], ["d", "e", "f"]]);
assert.deepEqual(parseDelimited("x\ty\nz\tw").rows, [["x", "y"], ["z", "w"]]);
});
it("honours Anki's #separator directive and skips the others", () => {
const { rows, delimiter } = parseDelimited("#separator:semicolon\n#html:true\n#tags column:3\na;b;t1 t2\n");
assert.equal(delimiter, ";");
assert.deepEqual(rows, [["a", "b", "t1 t2"]]);
});
it("tells a header row from a first card", () => {
assert.ok(looksLikeHeader(["Front", "Back", "Tags"]));
assert.ok(!looksLikeHeader(["cat", "gato"]));
assert.ok(!looksLikeHeader(["What is 2+2?", "4"]));
});
it("keeps a front that begins like Markdown structure a card, and reads every line back", () => {
const skips = new Skips();
const csv = [
"# of legs on a spider,8",
"```js,a fence would swallow every card after it",
"> quoted,q",
"| pipe,p",
"- [ ] task,t",
"a:::b,c",
"東京:首都,Tokyo",
"plain,card",
"x,y,",
].join("\r\n");
const deck = csvDeck(csv, { title: "Edges", front: 0, back: 1, extra: 2, tags: null, header: null }, skips);
const text = serialiseImportedDeck(deck);
const cards = scanCards(text);
assert.equal(cards.length, 9, text);
assert.deepEqual(cards.map((c) => c.front), ["\\# of legs on a spider", "\\```js", "\\> quoted", "\\| pipe", "\\- [ ] task", "a: : :b", "東京:首都", "plain", "x"]);
assert.deepEqual(cards.map((c) => c.back), ["8", "a fence would swallow every card after it", "q", "p", "t", "c", "Tokyo", "card", "y"]);
assert.deepEqual(skips.list(), []);
// An empty third column is no `::extra`.
assert.ok(text.endsWith("\nx::y\n"), text);
});
it("maps columns by index, drops empty rows, keeps tags", () => {
const skips = new Skips();
const deck = csvDeck("Term,Definition,Example,Tags\ncat,gato,el gato,animals\n,empty,,\ndog,perro,,animals pets", { title: "Spanish", front: 0, back: 1, extra: 2, tags: 3, header: null }, skips);
assert.deepEqual(
deck.cards.map((c) => [c.front, c.back, c.extra, c.tags]),
[["cat", "gato", "el gato", ["animals"]], ["dog", "perro", null, ["animals", "pets"]]],
);
assert.deepEqual(skips.list(), [{ reason: "empty", count: 1 }]);
assert.equal(
serialiseImportedDeck(deck),
"---\ntitle: Spanish\n---\n\n```deck\ntitle: Spanish\nkind: basic\n```\n\ncat::gato::el gato #animals\ndog::perro #animals #pets\n",
);
});
});
describe("serialiseImportedDeck", () => {
it("writes sections as headings and quotes a title YAML would misread", () => {
const deck: ImportDeck = {
title: "Physics: Light",
cards: [
{ kind: "qa", front: "c", back: "speed of light", extra: null, tags: [], section: null, schedule: null, reverse: null },
{ kind: "reversed", front: "ν", back: "frequency", extra: "Hz", tags: [], section: "Waves", schedule: { due: "2024-01-05", interval: 2, ease: 2400 }, reverse: null },
{ kind: "cloze", front: "==Light== is a wave", back: "", extra: null, tags: ["opt"], section: "Waves", schedule: null, reverse: null },
],
};
assert.equal(
serialiseImportedDeck(deck),
[
"---", 'title: "Physics: Light"', "---", "", "```deck", "title: Physics: Light", "kind: basic", "```", "",
"c::speed of light", "", "## Waves", "",
"ν:::frequency::Hz ",
"==Light== is a wave #opt", "",
].join("\n"),
);
// A title the caller typed with a newline and a fence in it stays one line, and the fence stays whole.
// A newline folds; the backticks stay (one line cannot open a fence),
// and YAML gets them quoted.
const odd = serialiseImportedDeck({ title: "Two\nlines ```", cards: [] });
assert.ok(odd.startsWith("---\ntitle: \"Two lines ```\"\n---\n\n```deck\ntitle: Two lines ```\nkind: basic\n```\n"), odd);
assert.equal(noteBaseName("A/B: C?"), "A B C");
assert.equal(noteBaseName("///"), "Imported deck");
});
});
// ── The whole thing: an .apkg built here, notes written there ────────────
const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3, 4]);
const MP3 = new Uint8Array([0x49, 0x44, 0x33, 4, 0, 0, 0, 0, 0, 0]);
/** The collection as SQLite bytes: schema 11, the shape every export with
* "support older Anki versions" carries. */
async function buildCollection(): Promise
\x1f[sound:meow.mp3] a cat\x1fExample: neko');
note.run(5, "g5", 1, "", "hidden\x1fsuspended");
note.run(6, "g6", 1, "", "学ぶ\x1fto learn");
note.run(7, "g7", 5, "", "鳥\x1fbird\x1fy");
note.run(8, "g8", 3, "", "Einstein: {{c1::E = mc²}} in {{c2::1905}}\x1f");
note.run(9, "g9", 1, "", "# of legs on a spider\x1f8");
// id, nid, did, ord, type, queue, due, ivl, factor, odue, odid
card.run(101, 1, 10, 0, 2, 2, 10, 4, 2500, 0, 0);
card.run(102, 2, 10, 0, 0, 0, 5, 0, 0, 0, 0);
card.run(103, 2, 10, 1, 2, 2, 3, 1, 2300, 0, 0);
card.run(104, 3, 12, 0, 0, 0, 6, 0, 0, 0, 0);
card.run(105, 3, 12, 1, 2, 2, 20, 7, 2650, 0, 0);
card.run(106, 4, 11, 0, 0, 0, 7, 0, 0, 0, 0);
card.run(107, 5, 10, 0, 2, -1, 2, 3, 2500, 0, 0);
card.run(108, 6, 10, 0, 1, 1, learningDue, 0, 0, 0, 0);
card.run(109, 7, 10, 0, 0, 0, 8, 0, 0, 0, 0);
card.run(110, 7, 10, 1, 0, 0, 8, 0, 0, 0, 0);
card.run(111, 8, 12, 0, 0, 0, 9, 0, 0, 0, 0);
card.run(112, 8, 12, 1, 0, 0, 9, 0, 0, 0, 0);
card.run(113, 9, 12, 0, 0, 0, 10, 0, 0, 0, 0);
db.close();
return new Uint8Array(readFileSync(file));
}
/** A real SQLite file with none of Anki's tables. */
async function emptyDb(): Promise