// The math-editing figures (scripts/gen_mathedit.py makes their data): //
// one line per benchmark: its score for the base (unedited) model and for the edits made with 10, 25 and 50 forward // passes, its name and last score at its end. One is in front with its scores written at its points: MATH-500, or // the line (or name) pointed at. The lines draw themselves the first time the figure is on screen. //
// a run turn by turn, played like a training run: the accuracy Astra measured on practice problems for the // original weights and its current edit (after each paired generation pass), and under it a tick for every turn, // red where it changed the weights. Beside it, that turn: what it did in a line (for an edit, what Astra had seen // and what it changed), then what Astra wrote, cut short ("more" shows all of it). Tabs: the 50-pass run (shown // first), the 25- and 10-pass runs. It plays while on screen, stopping a moment at the key edits (the turns where the // line of attack changed) and passing quickly over the rest; pointing at the chart (or the arrow keys, or the list // of edits) stops it at the turn there, and leaving the figure plays on from that turn: every turn has a place on // the x axis, the turns between two forward passes side by side. (() => { const NS = "http://www.w3.org/2000/svg"; const GREEN = "#118659", ORANGE = "#c25a00", GREY = "#8b929b", RED = "#d93a4b", INK = "#333333"; const GRID = "#e6e8eb", AXIS = "#b9bec6"; const esc = (s) => String(s).replace(/&/g, "&").replace(//g, ">"); function svg(tag, attrs = {}, parent) { const el = document.createElementNS(NS, tag); for (const [k, v] of Object.entries(attrs)) el.setAttribute(k, v); if (parent) parent.appendChild(el); return el; } function label(parent, x, y, text, attrs = {}) { const t = svg("text", { x, y, "font-size": 14, fill: INK, ...attrs }, parent); t.textContent = text; return t; } // ---------- benchmarks ---------- function bench(root, data) { // drawn at the width it is shown at, so that its text is the page's size; drawn again when that width changes. // One benchmark is in front (MATH-500, or the one pointed at): thicker, its scores written at its points, a wash // under it; the lines draw themselves the first time the figure is on screen. const color = (b) => (b.group === "math" ? GREEN : ORANGE); const DEFAULT = 0; // MATH-500 root.innerHTML = `
math benchmarks other benchmarks
`; const box = root.querySelector(".me-box"); let drawnAt = 0, drawn = false, focusOn = null; function render() { const W = Math.max(330, Math.round(box.clientWidth)); if (Math.abs(W - drawnAt) < 2) return; drawnAt = W; const H = Math.round(Math.min(440, Math.max(320, W * 0.46))); const M = { l: 52, r: W < 560 ? 150 : 178, t: 24, b: 60 }; const X = (p) => M.l + 22 + (p / 50) * (W - M.l - M.r - 22); const YMAX = 75, Y = (v) => M.t + (1 - v / YMAX) * (H - M.t - M.b); box.replaceChildren(); const s = svg("svg", { viewBox: `0 0 ${W} ${H}`, width: W, height: H, class: "me-svg", role: "img", "aria-label": "Accuracy on eight benchmarks for the base model and the edits made with 10, 25 and 50 forward passes." }, box); const defs = svg("defs", {}, s); for (const [id, c] of [["me-wash-math", GREEN], ["me-wash-other", ORANGE]]) { const g = svg("linearGradient", { id, x1: 0, x2: 0, y1: 0, y2: 1 }, defs); svg("stop", { offset: "0%", "stop-color": c, "stop-opacity": 0.16 }, g); svg("stop", { offset: "100%", "stop-color": c, "stop-opacity": 0 }, g); } for (const v of [0, 20, 40, 60]) { svg("line", { x1: M.l, x2: W - M.r + 8, y1: Y(v), y2: Y(v), stroke: v ? GRID : AXIS }, s); label(s, M.l - 10, Y(v) + 5, `${v}%`, { "text-anchor": "end" }); } for (const p of [0, 10, 25, 50]) { svg("line", { x1: X(p), x2: X(p), y1: M.t, y2: H - M.b, stroke: GRID }, s); label(s, X(p), H - M.b + 22, p ? `${p}` : "base", { "text-anchor": "middle", "font-weight": p ? 400 : 600 }); } label(s, (X(0) + X(50)) / 2, H - 8, "forward passes Astra could use", { "text-anchor": "middle", "font-weight": 600 }); const wash = svg("path", { "pointer-events": "none", class: "me-wash" }, s); const series = data.benchmarks.map((b, i) => { const pts = b.points.map((p) => [X(p.passes), Y(p.score)]); const g = svg("g", { class: "me-series" }, s); const line = svg("polyline", { points: pts.map((p) => p.join(",")).join(" "), fill: "none", stroke: color(b), "stroke-width": 2, "stroke-linejoin": "round", "stroke-linecap": "round" }, g); const dots = pts.map(([cx, cy]) => svg("circle", { cx, cy, r: 4, fill: color(b), stroke: "#ffffff", "stroke-width": 2 }, g)); return { b, i, pts, g, line, dots }; }); // the names at the right end of the lines, with their last score, spread so that they do not overlap const ends = series.map((d) => ({ d, y: d.pts[3][1] })).sort((a, b) => a.y - b.y); for (let k = 1; k < ends.length; k++) ends[k].y = Math.max(ends[k].y, ends[k - 1].y + 18); const over = ends[ends.length - 1].y - (H - M.b); if (over > 0) ends.forEach((e) => (e.y -= over)); for (const e of ends) { const x0 = e.d.pts[3][0]; svg("line", { x1: x0 + 7, y1: e.d.pts[3][1], x2: x0 + 15, y2: e.y, stroke: AXIS, class: "me-lead" }, e.d.g); const t = label(e.d.g, x0 + 19, e.y + 5, "", { class: "me-name" }); const name = svg("tspan", {}, t); name.textContent = e.d.b.name; const val = svg("tspan", { dx: 6, class: "me-val" }, t); val.textContent = e.d.b.points[3].score.toFixed(1); e.d.name = t; } const marks = svg("g", { "pointer-events": "none" }, s); // the scores of the benchmark in front function focus(d) { if (d === focusOn && drawn) return; focusOn = d; for (const o of series) { const on = o === d; o.g.classList.toggle("me-dim", !on); o.g.classList.toggle("me-on", on); o.line.setAttribute("stroke-width", on ? 3 : 2); o.dots.forEach((c) => c.setAttribute("r", on ? 5 : 3.5)); } d.g.parentNode.insertBefore(d.g, marks); // in front of the others, under its scores const base = H - M.b; wash.setAttribute("d", `M${d.pts.map((p) => p.join(",")).join(" L")} L${d.pts[3][0]},${base} L${d.pts[0][0]},${base} Z`); wash.setAttribute("fill", `url(#${d.b.group === "math" ? "me-wash-math" : "me-wash-other"})`); marks.replaceChildren(); d.b.points.slice(0, 3).forEach((p, k) => { const [x, y] = d.pts[k]; label(marks, x, y - 12, `${p.score.toFixed(1)}${p.note ? "*" : ""}`, { "text-anchor": "middle", "font-weight": 700, class: "me-halo" }); }); } function nearest(evt) { const pt = s.createSVGPoint(); pt.x = evt.clientX; pt.y = evt.clientY; const p = pt.matrixTransform(s.getScreenCTM().inverse()); let best = null, dist = 22; for (const d of series) { for (let k = 0; k < 3; k++) { const [x1, y1] = d.pts[k], [x2, y2] = d.pts[k + 1]; const t = Math.max(0, Math.min(1, ((p.x - x1) * (x2 - x1) + (p.y - y1) * (y2 - y1)) / ((x2 - x1) ** 2 + (y2 - y1) ** 2))); const dd = Math.hypot(p.x - (x1 + t * (x2 - x1)), p.y - (y1 + t * (y2 - y1))); if (dd < dist) { dist = dd; best = d; } } const bb = d.name.getBBox(); if (p.x >= bb.x - 4 && p.x <= bb.x + bb.width + 4 && p.y >= bb.y - 2 && p.y <= bb.y + bb.height + 2) return d; } return best; } s.addEventListener("pointermove", (e) => { const d = nearest(e); if (d) focus(d); }); s.addEventListener("pointerleave", () => focus(series[DEFAULT])); focus(focusOn ? series[focusOn.i] : series[DEFAULT]); if (!drawn) { // drawn in, once, when it is first on screen s.classList.add("me-hidden"); new IntersectionObserver((es, obs) => { if (!es[0].isIntersecting) return; obs.disconnect(); drawn = true; s.classList.remove("me-hidden"); series.forEach((d, k) => { const len = d.line.getTotalLength(); d.line.style.strokeDasharray = `${len}`; d.line.style.strokeDashoffset = `${len}`; d.line.getBoundingClientRect(); d.line.style.transition = `stroke-dashoffset 1.1s cubic-bezier(.3,.7,.3,1) ${k * 70}ms`; d.line.style.strokeDashoffset = "0"; d.g.style.setProperty("--delay", `${k * 70}ms`); }); s.classList.add("me-drawing"); setTimeout(() => s.classList.remove("me-drawing"), 2600); // later scores appear at once }, { threshold: 0.4 }).observe(s); } } render(); new ResizeObserver(render).observe(box); } // ---------- a run, turn by turn ---------- function md(text) { // the final answer: headings, paragraphs, lists, tables, bold and code const inline = (s) => esc(s).replace(/\*\*(.+?)\*\*/g, "$1").replace(/`([^`]+)`/g, "$1"); const out = []; let table = null, list = false; const close = () => { if (table) { out.push(`${table.join("")}
`); table = null; } if (list) { out.push(""); list = false; } }; for (const raw of text.split("\n")) { const line = raw.trim(); if (line.startsWith("|")) { if (/^[|\s:-]+$/.test(line)) continue; if (list) { out.push(""); list = false; } table = table || []; const cells = line.replace(/^\||\|$/g, "").split("|").map((c) => inline(c.trim())); table.push(`${cells.map((c) => `${c}`).join("")}`); continue; } if (!line) { close(); continue; } const h = line.match(/^#+\s*(.*)$/); if (h) { close(); out.push(`
${inline(h[1])}
`); continue; } if (/^[-*]\s+/.test(line)) { if (table) { out.push(`${table.join("")}
`); table = null; } if (!list) { out.push("