"""Train TINY version 2. python3 -I train2.py bench speed test of model sizes python3 -I train2.py train MINUTES OUT.pt [--from TINY.BIN] python3 -I train2.py eval MODEL.pt|TINY.BIN score on held-back questions python3 -I train2.py export MODEL.pt TINY.BIN write the DOS weight file The model reads and writes one letter at a time (ASCII 32..126 plus newline).""" import sys, os, time, math, random, struct, re, collections import numpy as np import torch, torch.nn as nn, torch.nn.functional as F sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import gen2 VOCAB, CTX = 97, 128 torch.set_num_threads(2) def encode(s): return [1 if c == '\n' else ord(c) - 30 for c in s] def decode(ids): return ''.join('\n' if i == 1 else chr(i + 30) for i in ids if i > 0) class Cfg: def __init__(self, D=96, L=4, H=4, HID=256): self.D, self.L, self.H, self.HID = D, L, H, HID def __repr__(self): return 'D=%d L=%d H=%d HID=%d' % (self.D, self.L, self.H, self.HID) class Norm(nn.Module): def __init__(self, d): super().__init__() self.g = nn.Parameter(torch.ones(d)) def forward(self, x): return x * torch.rsqrt(x.float().pow(2).mean(-1, keepdim=True) + 1e-5).to(x.dtype) * self.g class Block(nn.Module): def __init__(self, c): super().__init__() self.c = c self.n1, self.n2 = Norm(c.D), Norm(c.D) self.q = nn.Linear(c.D, c.D, bias=False) self.k = nn.Linear(c.D, c.D, bias=False) self.v = nn.Linear(c.D, c.D, bias=False) self.o = nn.Linear(c.D, c.D, bias=False) self.f1 = nn.Linear(c.D, c.HID, bias=False) self.f2 = nn.Linear(c.HID, c.D, bias=False) def forward(self, x): B, T, D = x.shape H = self.c.H h = self.n1(x) sp = lambda t: t.view(B, T, H, D // H).transpose(1, 2) a = F.scaled_dot_product_attention(sp(self.q(h)), sp(self.k(h)), sp(self.v(h)), is_causal=True) x = x + self.o(a.transpose(1, 2).reshape(B, T, D)) return x + self.f2(F.relu(self.f1(self.n2(x)))) class Model(nn.Module): def __init__(self, c): super().__init__() self.c = c self.emb = nn.Embedding(VOCAB, c.D) self.pos = nn.Embedding(CTX, c.D) self.blocks = nn.ModuleList([Block(c) for _ in range(c.L)]) self.nf = Norm(c.D) nn.init.normal_(self.emb.weight, std=0.05) nn.init.normal_(self.pos.weight, std=0.05) for b in self.blocks: for m in (b.q, b.k, b.v, b.o, b.f1, b.f2): nn.init.normal_(m.weight, std=0.5 / math.sqrt(m.in_features)) def forward(self, idx): x = self.emb(idx) + self.pos(torch.arange(idx.shape[1])) for b in self.blocks: x = b(x) return self.nf(x) @ self.emb.weight.t() # ---------------------------------------------------------------- old TINY.BIN def load_old_bin(path, D=96, L=3, HID=192): """Read the version-1 weight file (one scale per table) into a model.""" b = open(path, 'rb').read() off = [0] def table(r, c): s = struct.unpack_from(' %s (want: %s)' % b) return tot, line # ---------------------------------------------------------------- export def export(model, path): """Version-2 weight file. Header: 'TNY2', D, L, H, HID (int16 each). Each table: rows x (float32 scale + cols int8) (one scale per row). Gains: D float32.""" c = model.c sd = {k: v.float() for k, v in model.state_dict().items()} with open(path, 'wb') as f: f.write(b'TNY2' + struct.pack('<4h', c.D, c.L, c.H, c.HID)) def table(w): w = w.numpy() for row in w: s = max(float(np.abs(row).max()) / 127.0, 1e-8) f.write(struct.pack(' total: break warm = min(1.0, (step + 1) / 200) if decay_frac is None: lr = peak * warm * (0.03 + 0.97 * 0.5 * (1 + math.cos(math.pi * el / total))) else: left = (total - el) / (decay_frac * total) lr = peak * warm * min(1.0, max(0.0, left)) if eval_every and el > (eval_no + 1) * eval_every * 60: eval_no += 1 save_model(model, out) tot, line = evaluate(quantized_copy(model)) print(' EVAL at %.0f min: %.1f%% %s' % (el / 60, 100 * tot, line), flush=True) for g in opt.param_groups: g['lr'] = lr x, m = batch(rng, bs) loss = loss_fn(model, x, m) opt.zero_grad(set_to_none=True) loss.backward() nn.utils.clip_grad_norm_(model.parameters(), 1.0) opt.step() step += 1 run = loss.item() if run is None else 0.98 * run + 0.02 * loss.item() if step % log_every == 0: print('step %5d %4.0fs lr %.5f loss %.3f' % (step, el, lr, run), flush=True) save_model(model, out) print('saved', out, 'steps', step, 'time %.0fs' % (time.time() - t0), flush=True) return model if __name__ == '__main__': cmd = sys.argv[1] if cmd == 'bench': rng = random.Random(0) for c in [Cfg(96, 3, 4, 192), Cfg(96, 4, 4, 256), Cfg(112, 4, 4, 224)]: m = Model(c) opt = torch.optim.AdamW(m.parameters(), lr=1e-3) for i in range(13): if i == 3: t = time.time() x, k = batch(rng, 96) l = loss_fn(m, x, k); opt.zero_grad(); l.backward(); opt.step() print(c, sum(p.numel() for p in m.parameters()), '%.3f s/step' % ((time.time() - t) / 10), flush=True) elif cmd == 'train': arg = lambda k, d=None: sys.argv[sys.argv.index(k) + 1] if k in sys.argv else d m = train(float(sys.argv[2]), sys.argv[3], arg('--from'), resume=arg('--resume'), peak=float(arg('--lr', 2e-3)), decay_frac=float(arg('--decay')) if arg('--decay') else None, eval_every=float(arg('--eval-every', 0))) tot, line = evaluate(m, show=0) print('eval (float) %.1f%% %s' % (100 * tot, line)) tot, line = evaluate(quantized_copy(m), show=25) print('eval (8-bit) %.1f%% %s' % (100 * tot, line)) elif cmd == 'eval': m = load_model(sys.argv[2]) tot, line = evaluate(m, show=int(sys.argv[3]) if len(sys.argv) > 3 else 20) print('eval %.1f%% %s' % (100 * tot, line)) elif cmd == 'export': print('wrote', export(load_model(sys.argv[2]), sys.argv[3]), 'bytes')