// AI-MONITOR · Training observability section (Phase 1 — obs-only substrate). // 数据来自 /api/training(DCGM L1 + node-exporter, 经 obs-prometheus 只读)→ live.training。 // 覆盖 Layer C(GPU 健康)/D(RoCE 互联)/E(静默 stall 检测)。loss·grad-norm·step· // ETA·MFU 需训练框架信号源(Phase 2),此处诚实标"未接入",不伪造。 // 注:本文件独立 babel 作用域——全局词法环境已占用 live/_live/_NODES,故用 _tLive 等独立名; // SmallStat/DetailMetric 未导出到 window,故内联 _TStat。可用 window 全局:Ring/Sparkline/useLive/useLang。 const _tLive = window.AIData.live; function _fmtBw(mbps) { if (mbps >= 1000) return (mbps / 1000).toFixed(1) + " GB/s"; return Math.round(mbps) + " MB/s"; } function _fmtEta(sec) { if (sec == null) return "—"; const h = Math.floor(sec / 3600), m = Math.round((sec % 3600) / 60); return h > 0 ? `${h}h ${m}m` : `${m}m`; } // 训练框架信号(Phase 2):loss·grad-norm·step/ETA·acceptance。来自 leader tfevents, // 缺失时(当前轮 / 文件不存在)整块不渲染,由 TrainingSection 退回 "awaiting" 提示。 function _SignalBlock({ sig }) { const { t } = useLang(); const loss = sig.lossSeries || []; const acc = sig.accSeries || []; const macc = sig.meanAcceptSeries || []; const lossDelta = loss.length >= 2 ? loss[loss.length - 1] - loss[0] : 0; // mean accept length 是 EAGLE/投机解码核心 KPI:≥2.3 达标(对应单流 ≥50 t/s),1.0=无效 const ma = sig.meanAccept; const maTone = ma == null ? "var(--ink-3)" : ma >= 2.3 ? "var(--ok)" : ma >= 1.6 ? "var(--warn)" : "var(--bad)"; const stale = sig.staleSec != null && sig.staleSec > 600; // >10min 无写入 = 疑似停 const pct = Math.round((sig.progress || 0) * 100); return (
{/* loss — hero, 自动缩放 sparkline 显降势 */}
{t("train loss")} {lossDelta < 0 ? "↓" : ""}{lossDelta ? Math.abs(lossDelta).toFixed(3) : ""}
{sig.loss != null ? sig.loss.toFixed(3) : "—"}
{/* step + epoch + progress + ETA */}
{t("step")}{sig.epoch != null ? ` · ep ${sig.epoch}` : ""}
{sig.step}{sig.totalSteps ? ` / ${sig.totalSteps}` : ""}
{t("ETA")} {_fmtEta(sig.etaSec)} · {sig.perStepSec != null ? sig.perStepSec.toFixed(2) + "s/it" : "—"}
{/* mean accept length — EAGLE/投机解码核心 KPI(≥2.3 达标);acceptance% 作辅 */}
{t("mean accept")} {t("≥2.3")}
{ma != null ? ma.toFixed(2) + "×" : "—"} {sig.acceptance != null && · {(sig.acceptance * 100).toFixed(0)}% acc}
{/* grad-norm + lr + 状态 */}
{t("grad norm")} {sig.gradNorm != null ? sig.gradNorm.toFixed(2) : "—"}
{t("lr")} {sig.lr != null ? sig.lr.toExponential(1) : "—"}
{stale ? <_TPill label={t("write stalled?")} tone="bad" /> : <_TPill label={t("logging")} tone="ok" />}
); } function _TStat({ label, value, unit, sub, color }) { return (
{label}
{value}{unit && {unit}}
{sub &&
{sub}
}
); } function _TPill({ label, tone }) { const c = { ok: "var(--ok)", warn: "var(--warn)", bad: "var(--bad)", idle: "var(--ink-3)" }[tone] || "var(--ink-3)"; return ( {label} ); } function _RankCard({ n, slowest }) { const { t } = useLang(); const fault = n.xid > 0 || n.eccDbe > 0 || n.ibLinkDowned > 0; const isSlow = slowest && n.id === slowest; const memTone = n.memPct >= 90 ? "var(--bad)" : n.memPct >= 80 ? "var(--hot)" : "var(--ok)"; const ringColor = n.stallSuspect ? "var(--bad)" : isSlow ? "var(--hot)" : "var(--accent)"; const border = (n.stallSuspect || fault) ? "var(--bad)" : isSlow ? "var(--hot)" : undefined; return (
{n.id} {n.stallSuspect ? {t("stall?")} : isSlow ? {t("straggler")} : null}
{/* unified-memory headroom — OOM-critical on GB10 */}
{t("mem")}{n.memPct.toFixed(0)}%
{n.tempGpu.toFixed(0)}°C {n.power.toFixed(0)} W {_fmtBw(n.roceRxMBps + n.roceTxMBps)}
{(fault || n.stallSuspect) && (
{n.xid > 0 ? `Xid ${n.xid} ` : ""}{n.eccDbe > 0 ? `ECC-DBE ${n.eccDbe} ` : ""} {n.ibLinkDowned > 0 ? `link-down ${n.ibLinkDowned} ` : ""}{n.stallSuspect ? t("comm stalled?") : ""}
)}
); } function TrainingSection() { useLive(); const { t } = useLang(); const T = _tLive.training || { nodes: [], summary: {} }; const S = T.summary || {}; const nodes = T.nodes || []; const sig = T.signal || { present: false }; const hist = _tLive.trainingHist || { util: [], roce: [] }; if (!nodes.length) return null; // 无 GB10 训练节点 → 不渲染 const active = S.active || 0; const idle = active === 0; // straggler:活跃节点里 util 最低的(且偏斜显著 ≥15%) let slowest = null; if (active > 1 && (S.utilSkew || 0) >= 15) { const sorted = nodes.filter((n) => n.util > 0).sort((a, b) => a.util - b.util); slowest = sorted.length ? sorted[0].id : null; } const health = S.stallSuspect ? { tone: "bad", label: t("stall suspected") } : (S.anyEccDbe || S.anyXid) ? { tone: "bad", label: t("GPU fault") } : idle ? { tone: "idle", label: t("idle") } : { tone: "ok", label: t("healthy") }; return (
07{t("Training · distributed")}

{t("The long ")}{t("run.")}

{t("Substrate health for the 4×GB10 training cluster — per-rank GPU, unified-memory headroom, CX-7 RoCE collective bandwidth, and a silent-stall heuristic. Read-only from obs; zero training impact.")}

{/* Phase 2 训练框架信号(loss/step/ETA/acceptance)—— 有 tfevents 才显 */} {sig.present && <_SignalBlock sig={sig} />} {/* status strip(底座 — 始终显)*/}
<_TStat label={t("Ranks active")} value={`${active}/${S.nodes || nodes.length}`} sub={t("util ≥ 50%")} />
{t("RoCE Σ rx+tx")}
{_fmtBw((S.roceRxMBps || 0) + (S.roceTxMBps || 0))}
<_TStat label={t("GPU power Σ")} value={(S.powerTotal || 0).toFixed(0)} unit=" W" />
<_TPill label={health.label} tone={health.tone} /> {(S.utilSkew || 0) >= 15 && active > 1 && <_TPill label={`${t("skew")} ${S.utilSkew.toFixed(0)}%`} tone="warn" />}
{/* per-rank small multiples */}
{nodes.map((n) => <_RankCard key={n.id} n={n} slowest={slowest} />)}
{/* signal 缺失时的诚实提示;有信号时标注来源 + MFU 仍待接 */} {!sig.present ? (
{t("Loss · step / ETA · accept rate — awaiting a training signal source (auto-detects JSON / Prometheus / TensorBoard from the leader). Not fabricated. See docs/training-observability.md.")}
) : (
{t("source")}: {sig.source} · {t("refresh ~12s")} · {t("MFU pending GB10 bf16 peak FLOPs.")}
)}
); } Object.assign(window, { TrainingSection });