import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; /** * Tokens are the unit the operator is billed in, and the unit nothing counted. * * `auto_events` has had a `tokens` column since it was created; every caller * wrote 0 into it, so the weekly token cap was unreachable and a 24-slide dense * deck with deep research cost exactly what a 10-slide sparse one did — one * "request". These tests hold the two halves of the fix: that a run's real * token spend is measured and recorded, and that a reservation taken before the * run is replaced by what the run actually cost. */ const scratch = await mkdtemp(path.join(tmpdir(), "forge-meter-")); process.env.FORGE_DB_PATH = path.join(scratch, "forge.db"); process.env.FORGE_CONFIG_DIR = scratch; process.env.FORGE_AUTO_WINDOW_REQUESTS = "100"; process.env.FORGE_AUTO_WEEKLY_REQUESTS = "100"; process.env.FORGE_AUTO_WINDOW_SLIDES = "1000"; process.env.FORGE_AUTO_WEEKLY_SLIDES = "1000"; process.env.FORGE_AUTO_WEEKLY_TOKENS = "100000"; // This file is about the meter and the rolling caps. The lifetime trial would // bind first at any realistic size, so it is lifted here and has its own file. process.env.FORGE_AUTO_TRIAL_TOKENS = String(10 ** 9); const { reserveAuto, settleAuto, getUsage, checkAutoLimits, clearAutoEvents, limitConfig, planFor, setPlan, PLANS, DEFAULT_PLAN, isPlan, } = await import("../src/limits.js"); const { newMeter, withMeter, recordModelUsage, currentMeter, meterTotal, meterSummary, estimateTokens, } = await import("../src/usage.js"); const auth = await import("../src/auth.js"); const email = "meter@example.test"; await auth.register({ name: "M", email, password: "test-password-123" }); const userId = auth.getUserId(email); test.after(() => rm(scratch, { recursive: true, force: true })); const reset = () => clearAutoEvents({ userId }); /* ------------------------------------------------------------- the meter */ test("the meter adds up what the model client reports, per role", () => { const m = newMeter(); withMeter(m, () => { // Exactly the shape `chat()` returns. recordModelUsage({ role: "author", model: "qwen3.6", promptCount: 1200, evalCount: 300 }); recordModelUsage({ role: "author", model: "qwen3.6", promptCount: 800, evalCount: 200 }); recordModelUsage({ role: "research", model: "qwen3.6", promptCount: 500, evalCount: 50 }); }); assert.equal(m.calls, 3); assert.equal(m.promptTokens, 2500); assert.equal(m.completionTokens, 550); assert.equal(meterTotal(m), 3050); assert.equal(m.byRole.author.calls, 2); assert.equal(m.byRole.research.promptTokens, 500); assert.deepEqual(meterSummary(m).models, ["qwen3.6"]); }); test("the meter survives a client that reports nothing", () => { // A transport that omits usage must not poison the total with NaN — that // would settle a reservation to garbage rather than to a number. const m = newMeter(); withMeter(m, () => { recordModelUsage({ role: "author" }); recordModelUsage({ role: "author", promptCount: undefined, evalCount: null }); recordModelUsage(null); }); assert.equal(meterTotal(m), 0); assert.equal(Number.isFinite(meterTotal(m)), true); }); test("outside a metered scope every call is a no-op — the CLI is unaffected", () => { assert.equal(currentMeter(), null); assert.doesNotThrow(() => recordModelUsage({ role: "author", promptCount: 10, evalCount: 5 })); }); test("the meter follows async work, which is the only reason it can be ambient", async () => { // The model client sits five or six awaits below the request. const m = newMeter(); await withMeter(m, async () => { await Promise.resolve(); await new Promise((r) => setTimeout(r, 1)); await (async () => { await Promise.resolve(); recordModelUsage({ role: "author", promptCount: 7, evalCount: 3 }); })(); }); assert.equal(meterTotal(m), 10); }); test("two concurrent operations do not pool their spend", async () => { const a = newMeter(); const b = newMeter(); await Promise.all([ withMeter(a, async () => { await new Promise((r) => setTimeout(r, 2)); recordModelUsage({ role: "author", promptCount: 100, evalCount: 0 }); }), withMeter(b, async () => { recordModelUsage({ role: "author", promptCount: 5, evalCount: 0 }); await new Promise((r) => setTimeout(r, 4)); recordModelUsage({ role: "author", promptCount: 5, evalCount: 0 }); }), ]); assert.equal(meterTotal(a), 100); assert.equal(meterTotal(b), 10); }); /* ------------------------------------------------- reserve, then settle */ test("a reservation is replaced by what the run actually cost", () => { reset(); const r = reserveAuto({ userId, upcomingSlides: 10, upcomingTokens: 50000 }); assert.equal(r.allowed, true); assert.ok(r.eventId, "a reservation must be addressable to be settled"); assert.equal(getUsage({ userId }).week.tokens, 50000); settleAuto({ eventId: r.eventId, tokens: 12345 }); assert.equal(getUsage({ userId }).week.tokens, 12345, "the estimate should be gone, not added to"); }); test("a run that cost more than its estimate is charged the truth", () => { reset(); const r = reserveAuto({ userId, upcomingTokens: 1000 }); settleAuto({ eventId: r.eventId, tokens: 40000 }); assert.equal(getUsage({ userId }).week.tokens, 40000); }); test("settling a failed run charges what it spent, not the estimate", () => { // A generation that died in its third slide still spent those three slides' // tokens; charging it a full deck is the difference between fair use and a // refund request. reset(); const r = reserveAuto({ userId, upcomingSlides: 22, upcomingTokens: 90000 }); settleAuto({ eventId: r.eventId, tokens: 8000 }); const usage = getUsage({ userId }); assert.equal(usage.week.tokens, 8000); assert.equal(usage.week.requests, 1, "the run still happened, so it still counts as a request"); }); test("a refused reservation has nothing to settle", () => { reset(); // Blow the weekly token budget outright. const first = reserveAuto({ userId, upcomingTokens: 99000 }); assert.equal(first.allowed, true); const second = reserveAuto({ userId, upcomingTokens: 99000 }); assert.equal(second.allowed, false); assert.equal(second.eventId, null, "a refusal must not write a row it would then settle"); assert.equal(settleAuto({ eventId: second.eventId, tokens: 5 }), false); assert.equal(getUsage({ userId }).week.tokens, 99000, "the refused run cost nothing"); }); test("the token cap refuses a run that would cross it, and says by how much", () => { reset(); reserveAuto({ userId, upcomingTokens: 95000 }); const chk = checkAutoLimits({ userId, upcomingTokens: 20000 }); assert.equal(chk.allowed, false); assert.ok(chk.errors.some((e) => /token/i.test(e)), chk.errors.join("; ")); assert.ok(chk.errors.some((e) => e.includes("20,000")), "the refusal should name the run's own cost"); }); test("settleAuto ignores a settle with nothing in it", () => { reset(); const r = reserveAuto({ userId, upcomingTokens: 100 }); assert.equal(settleAuto({ eventId: r.eventId }), false); assert.equal(getUsage({ userId }).week.tokens, 100); }); /* ------------------------------------------------------------------ tiers */ test("every account is free until it is not, and free is what it always was", () => { assert.equal(planFor(userId), DEFAULT_PLAN); assert.equal(DEFAULT_PLAN, "free"); const free = limitConfig("free"); const unspecified = limitConfig(); assert.deepEqual(free, unspecified, "the default tier must be the settings as configured"); }); test("a paid tier scales the budgets and nothing else", () => { const free = limitConfig("free"); const plus = limitConfig("plus"); for (const k of ["windowRequests", "weeklyRequests", "windowSlides", "weeklySlides", "weeklyTokens"]) { assert.equal(plus[k], free[k] * PLANS.plus.multiplier, `${k} should scale`); } // The shape of one piece of work is not a budget: paying more does not make // an hour longer, and deck length is its own admin setting. assert.equal(plus.windowHours, free.windowHours); assert.equal(plus.maxSlidesPerDeck, free.maxSlidesPerDeck); }); test("unlimited is uncapped rather than very large", () => { const u = limitConfig("unlimited"); assert.equal(u.unlimited, true); assert.equal(u.weeklyTokens, Infinity); assert.equal(u.weeklyRequests, Infinity); }); test("an account's tier decides which caps it is held to", () => { reset(); reserveAuto({ userId, upcomingTokens: 95000 }); assert.equal(checkAutoLimits({ userId, upcomingTokens: 20000 }).allowed, false); setPlan(userId, "plus"); assert.equal(planFor(userId), "plus"); assert.equal( checkAutoLimits({ userId, upcomingTokens: 20000 }).allowed, true, "the same spend, on a bigger tier", ); setPlan(userId, "unlimited"); assert.equal(checkAutoLimits({ userId, upcomingTokens: 10 ** 9 }).allowed, true); setPlan(userId, "free"); }); test("an unknown tier is refused rather than silently applied", () => { assert.throws(() => setPlan(userId, "enterprise"), /unknown plan/); assert.equal(planFor(userId), "free"); assert.equal(isPlan("free"), true); assert.equal(isPlan("nonsense"), false); // A row carrying a tier that no longer exists falls back rather than // uncapping somebody by accident. assert.equal(limitConfig("nonsense").plan, "free"); }); test("a check reports the tier it judged against", () => { reset(); assert.equal(checkAutoLimits({ userId }).plan, "free"); assert.equal(checkAutoLimits({ userId, plan: "plus" }).plan, "plus"); }); /* -------------------------------------------------------------- estimates */ test("the estimate grows with the work, which is the whole point", () => { const small = estimateTokens({ slides: 10 }); const big = estimateTokens({ slides: 24 }); assert.ok(big > small * 2, "a deck twice the size should not cost the same"); assert.ok(estimateTokens({ slides: 10, research: true }) > small, "research costs tokens too"); assert.ok(estimateTokens({}) > 0, "even a single call is not free"); }); test("the free tier's SHIPPED token budget agrees with its slide budget", async () => { // The two caps are stated in different units and must not contradict each // other — a token budget under one deck would refuse everybody while the // slide budget said four. That is exactly what shipped: 80,000 tokens against // a 90-slide week, which was harmless only because nothing counted a token. // // Read the DEFAULTS, not this box's configuration: the env overrides above // are the test's own, and asserting against them would check nothing. const { settingDefault } = await import("../src/runtime.js"); const perDeck = estimateTokens({ slides: 22, research: true }); const decksByTokens = settingDefault("autoWeeklyTokens") / perDeck; const decksBySlides = settingDefault("autoWeeklySlides") / 22; assert.ok(decksByTokens >= 2, `token budget allows only ${decksByTokens.toFixed(1)} decks`); assert.ok( Math.abs(decksByTokens - decksBySlides) < decksBySlides, `the two budgets disagree: ${decksByTokens.toFixed(1)} decks by tokens, ${decksBySlides.toFixed(1)} by slides`, ); }); test("the estimator's research size tracks what a call actually carries", async () => { // A spend model reading a stale number over-reserves, refusing people for // tokens that cannot be spent. Before retrieval this had to track // `excerpt_chars` in models.yaml; now that only bounds the POOL read off // disk, and what is billed is the retrieval budget. const { RESEARCH_EXCERPT_CHARS } = await import("../src/usage.js"); const { CALL_RESEARCH_CHARS } = await import("../src/ai/retrieve.js"); assert.equal(RESEARCH_EXCERPT_CHARS, CALL_RESEARCH_CHARS); }); test("retrieval is what makes the free tier affordable, and the budget says so", async () => { const { settingDefault } = await import("../src/runtime.js"); const { estimateTokens } = await import("../src/usage.js"); const perDeck = estimateTokens({ slides: 22, research: true }); // Before per-slide retrieval a deck was ~1.6M tokens and no student price // point worked at all (docs/ECONOMICS.md). assert.ok(perDeck < 400_000, `a deck should not cost ${perDeck.toLocaleString()} tokens`); const decks = settingDefault("autoWeeklyTokens") / perDeck; assert.ok(decks >= 3, `the weekly budget buys only ${decks.toFixed(1)} decks`); });