{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "course-introduction",
   "metadata": {},
   "source": [
    "# Course 2 — Pretraining the Foundation\n",
    "Deterministic teaching miniatures for Kimi K3 §3.1–§3.4. **Paper-scale corpus, scaling coefficients, training outcomes, and 1M-token capability are not reproduced here.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "curation-miniature",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-08-11T00:09:21.800343Z",
     "iopub.status.busy": "2026-08-11T00:09:21.799863Z",
     "iopub.status.idle": "2026-08-11T00:09:23.036983Z",
     "shell.execute_reply": "2026-08-11T00:09:23.035963Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accepted: ['w1', 'v1']\n",
      "reasons: ['accepted', 'exact_duplicate', 'near_duplicate', 'too_short', 'low_quality', 'accepted']\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import subprocess, sys, tempfile\n",
    "repo = next((p for p in (Path.cwd(), *Path.cwd().parents) if (p / 'src').is_dir()), None)\n",
    "if repo is None:\n",
    "    repo = (Path('/content') if Path('/content').is_dir() else Path(tempfile.gettempdir())) / 'build-Kimi-K3-architecture'\n",
    "    if not (repo / 'src').is_dir():\n",
    "        subprocess.run(['git', 'clone', '--depth', '1', 'https://github.com/mailtotanvir/build-Kimi-K3-architecture.git', str(repo)], check=True)\n",
    "sys.path.insert(0, str(repo))\n",
    "\n",
    "from src.training.data_curation import Document, PretrainingDataCurator\n",
    "docs = [Document('w1','web','A careful explanation of recurrent memory and retrieval.',.92), Document('w2','web','A careful explanation of recurrent memory and retrieval.',.90), Document('m1','math','A careful explanation of recurrent memory and retrieval!',.88), Document('c1','code','x=1',.95), Document('k1','knowledge','A long but unreliable generated fragment with broken provenance.',.2), Document('v1','vision','Rendered SVG paired with the source program and labels.',.86,'image-text')]\n",
    "accepted, decisions = PretrainingDataCurator().curate(docs)\n",
    "print('accepted:', [d.identifier for d in accepted])\n",
    "print('reasons:', [d.reason for d in decisions])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "mixture-policy",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-08-11T00:09:23.038837Z",
     "iopub.status.busy": "2026-08-11T00:09:23.038599Z",
     "iopub.status.idle": "2026-08-11T00:09:23.044196Z",
     "shell.execute_reply": "2026-08-11T00:09:23.043178Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deterministic sampled domains: {'web': 6, 'math': 3, 'vision': 3}\n",
      "sample order: ['w0', 'm0', 'v0', 'w1', 'm1', 'm2', 'w2', 'w3', 'w4', 'v1', 'w5', 'v2']\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "from src.training.data_curation import sample_mixture\n",
    "pools = {\n",
    "    'web': [Document(f'w{i}', 'web', f'Web teaching record number {i} with sufficient detail.', .9) for i in range(6)],\n",
    "    'math': [Document(f'm{i}', 'math', f'Mathematics teaching record number {i} with sufficient detail.', .9) for i in range(3)],\n",
    "    'vision': [Document(f'v{i}', 'vision', f'Visual teaching record number {i} with sufficient detail.', .9) for i in range(3)],\n",
    "}\n",
    "mixture = sample_mixture(pools, {'web': .5, 'math': .3, 'vision': .2}, 12, seed=11)\n",
    "print('deterministic sampled domains:', dict(Counter(doc.domain for doc in mixture)))\n",
    "print('sample order:', [doc.identifier for doc in mixture])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "scaling-miniature",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-08-11T00:09:23.045902Z",
     "iopub.status.busy": "2026-08-11T00:09:23.045756Z",
     "iopub.status.idle": "2026-08-11T00:09:23.049832Z",
     "shell.execute_reply": "2026-08-11T00:09:23.048601Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best synthetic allocation: N=1000, D=1000, loss=1.868804\n"
     ]
    }
   ],
   "source": [
    "from src.training.long_context import iso_compute_candidates\n",
    "runs = iso_compute_candidates(6e6, [10, 100, 1000])\n",
    "best = min(runs, key=lambda row: row[2])\n",
    "print(f'best synthetic allocation: N={best[0]}, D={best[1]:.0f}, loss={best[2]:.6f}')\n",
    "# Synthetic coefficients teach the method; they are not fitted from K3 runs.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "schedule-shapes",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-08-11T00:09:23.051363Z",
     "iopub.status.busy": "2026-08-11T00:09:23.051221Z",
     "iopub.status.idle": "2026-08-11T00:09:23.054790Z",
     "shell.execute_reply": "2026-08-11T00:09:23.053919Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cosine [(0, '0.000000'), (200, '0.000913'), (400, '0.000667'), (600, '0.000358'), (800, '0.000106'), (1000, '0.000010')]\n",
      "wsd [(0, '0.000000'), (200, '0.001000'), (400, '0.001000'), (600, '0.001000'), (800, '0.001000'), (1000, '0.000010')]\n",
      "Shape comparison only: the paper independently tunes each schedule before comparing loss.\n"
     ]
    }
   ],
   "source": [
    "from src.training.long_context import schedule_profiles\n",
    "profiles = schedule_profiles(1000, 1e-3, 1e-5, samples=6)\n",
    "for name, points in profiles.items():\n",
    "    print(name, [(step, f'{rate:.6f}') for step, rate in points])\n",
    "print('Shape comparison only: the paper independently tunes each schedule before comparing loss.')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "schedule-miniature",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-08-11T00:09:23.056483Z",
     "iopub.status.busy": "2026-08-11T00:09:23.056322Z",
     "iopub.status.idle": "2026-08-11T00:09:23.060318Z",
     "shell.execute_reply": "2026-08-11T00:09:23.059392Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "learning rates: ['0.000000', '0.001000', '0.000980', '0.000513', '0.000010']\n",
      "context stages: ['8K', '64K', '256K', '1M']\n"
     ]
    }
   ],
   "source": [
    "from src.training.long_context import cosine_learning_rate, context_curriculum\n",
    "print('learning rates:', [f'{cosine_learning_rate(s,1000,1e-3,1e-5):.6f}' for s in [0,10,100,500,1000]])\n",
    "print('context stages:', [f'{context_curriculum(p)//1024}K' if context_curriculum(p)<1_000_000 else '1M' for p in [.1,.8,.94,.99]])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "evidence-boundary",
   "metadata": {},
   "source": [
    "## Evidence boundary\n",
    "Executed here: filtering/dedup decisions, synthetic iso-compute arithmetic, cosine schedule arithmetic, curriculum stage selection, and tensor chunk preservation.\n",
    "\n",
    "Paper reported: corpus construction, independent schedule searches, 2.5× scaling efficiency, full training stability, and 1M-token results."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}