import regex
from abc import ABC
from dataclasses import dataclass
from collections import defaultdict
import random
from execute_util import link, image, text
from lecture_util import article_link, x_link, youtube_link
from references import gpt_3, gpt4, shannon1950, bengio2003, susketver2014, \
bahdanau2015_attention, transformer_2017, gpt2, t5, kaplan_scaling_laws_2020, \
the_pile, gpt_j, opt_175b, bloom, palm, chinchilla, llama, mistral_7b, \
instruct_gpt, dpo, adamw2017, lima, deepseek_v3, adam2014, grpo, ppo2017, muon, \
large_batch_training_2018, wsd_2024, cosine_learning_rate_2017, olmo_7b, moe_2017, \
megatron_lm_2019, shazeer_2020, elmo, bert, qwen_2_5, deepseek_r1, moe_2017, \
rms_norm_2019, rope_2021, soap, gqa, mla, deepseek_67b, deepseek_v2, brants2007, \
layernorm_2016, pre_post_norm_2020, llama2, llama3, olmo2, \
megabyte, byt5, blt, tfree, sennrich_2016, zero_2019, gpipe_2018
from data import get_common_crawl_urls, read_common_crawl, write_documents, markdownify_documents
from model_util import query_gpt4o
import tiktoken
def main():
welcome()
why_this_course_exists()
current_landscape()
what_is_this_program()
course_logistics()
course_components()
tokenization()
text("Next time: PyTorch building blocks, resource accounting")
def welcome():
text("## CS336: Language Models From Scratch (Spring 2025)"),
image("images/course-staff.png", width=600)
text("This is the second offering of CS336.")
text("Stanford edition has grown by 50%.")
text("Lectures will be posted on YouTube and be made available to the whole world.")
def why_this_course_exists():
text("## Why did we make this course?")
text("Let's ask GPT-4 "), link(gpt4)
response = query_gpt4o(prompt="Why teach a course on building language models from scratch? Answer in one sentence.") # @inspect response
text("Problem: researchers are becoming **disconnected** from the underlying technology.")
text("8 years ago, researchers would implement and train their own models.")
text("6 years ago, researchers would download a model (e.g., BERT) and fine-tune it.")
text("Today, researchers just prompt a proprietary model (e.g., GPT-4/Claude/Gemini).")
text("Moving up levels of abstractions boosts productivity, but")
text("- These abstractions are leaky (in contrast to programming languages or operating systems).")
text("- There is still fundamental research to be done that require tearing up the stack.")
text("**Full understanding** of this technology is necessary for **fundamental research**.")
text("This course: **understanding via building**")
text("But there's one small problem...")
text("## The industrialization of language models")
image("https://upload.wikimedia.org/wikipedia/commons/thumb/c/cc/Industrialisation.jpg/440px-Industrialisation.jpg", width=400)
text("GPT-4 supposedly has 1.8T parameters. "), article_link("https://www.hpcwire.com/2024/03/19/the-generative-ai-future-is-now-nvidias-huang-says")
text("GPT-4 supposedly cost $100M to train. "), article_link("https://www.wired.com/story/openai-ceo-sam-altman-the-age-of-giant-ai-models-is-already-over/")
text("xAI builds cluster with 200,000 H100s to train Grok. "), article_link("https://www.tomshardware.com/pc-components/gpus/elon-musk-is-doubling-the-worlds-largest-ai-gpu-cluster-expanding-colossus-gpu-cluster-to-200-000-soon-has-floated-300-000-in-the-past")
text("Stargate (OpenAI, NVIDIA, Oracle) invests $500B over 4 years. "), article_link("https://openai.com/index/announcing-the-stargate-project/")
text("Also, there are no public details on how frontier models are built.")
text("From the GPT-4 technical report "), link(gpt4), text(":")
image("images/gpt4-no-details.png", width=600)
text("## More is different")
text("Frontier models are out of reach for us.")
text("But building small language models (<1B parameters in this class) might not be representative of large language models.")
text("Example 1: fraction of FLOPs spent in attention versus MLP changes with scale. "), x_link("https://x.com/stephenroller/status/1579993017234382849")
image("images/roller-flops.png", width=400)
text("Example 2: emergence of behavior with scale "), link("https://arxiv.org/pdf/2206.07682")
image("images/wei-emergence-plot.png", width=600)
text("## What can we learn in this class that transfers to frontier models?")
text("There are three types of knowledge:")
text("- **Mechanics**: how things work (what a Transformer is, how model parallelism leverages GPUs)")
text("- **Mindset**: squeezing the most out of the hardware, taking scale seriously (scaling laws)")
text("- **Intuitions**: which data and modeling decisions yield good accuracy")
text("We can teach mechanics and mindset (these do transfer).")
text("We can only partially teach intuitions (do not necessarily transfer across scales).")
text("## Intuitions? 🤷")
text("Some design decisions are simply not (yet) justifiable and just come from experimentation.")
text("Example: Noam Shazeer paper that introduced SwiGLU "), link(shazeer_2020)
image("images/divine-benevolence.png", width=600)
text("## The bitter lesson")
text("Wrong interpretation: scale is all that matters, algorithms don't matter.")
text("Right interpretation: algorithms that scale is what matters.")
text("### accuracy = efficiency x resources")
text("In fact, efficiency is way more important at larger scale (can't afford to be wasteful).")
link("https://arxiv.org/abs/2005.04305"), text(" showed 44x algorithmic efficiency on ImageNet between 2012 and 2019")
text("Framing: what is the best model one can build given a certain compute and data budget?")
text("In other words, **maximize efficiency**!")
def current_landscape():
text("## Pre-neural (before 2010s)")
text("- Language model to measure the entropy of English "), link(shannon1950)
text("- Lots of work on n-gram language models (for machine translation, speech recognition) "), link(brants2007)
text("## Neural ingredients (2010s)")
text("- First neural language model "), link(bengio2003)
text("- Sequence-to-sequence modeling (for machine translation) "), link(susketver2014)
text("- Adam optimizer "), link(adam2014)
text("- Attention mechanism (for machine translation) "), link(bahdanau2015_attention)
text("- Transformer architecture (for machine translation) "), link(transformer_2017)
text("- Mixture of experts "), link(moe_2017)
text("- Model parallelism "), link(gpipe_2018), link(zero_2019), link(megatron_lm_2019)
text("## Early foundation models (late 2010s)")
text("- ELMo: pretraining with LSTMs, fine-tuning helps tasks "), link(elmo)
text("- BERT: pretraining with Transformer, fine-tuning helps tasks "), link(bert)
text("- Google's T5 (11B): cast everything as text-to-text "), link(t5)
text("## Embracing scaling, more closed")
text("- OpenAI's GPT-2 (1.5B): fluent text, first signs of zero-shot, staged release "), link(gpt2)
text("- Scaling laws: provide hope / predictability for scaling "), link(kaplan_scaling_laws_2020)
text("- OpenAI's GPT-3 (175B): in-context learning, closed "), link(gpt_3)
text("- Google's PaLM (540B): massive scale, undertrained "), link(palm)
text("- DeepMind's Chinchilla (70B): compute-optimal scaling laws "), link(chinchilla)
text("## Open models")
text("- EleutherAI's open datasets (The Pile) and models (GPT-J) "), link(the_pile), link(gpt_j)
text("- Meta's OPT (175B): GPT-3 replication, lots of hardware issues "), link(opt_175b)
text("- Hugging Face / BigScience's BLOOM: focused on data sourcing "), link(bloom)
text("- Meta's Llama models "), link(llama), link(llama2), link(llama3)
text("- Alibaba\'s Qwen models "), link(qwen_2_5)
text("- DeepSeek\'s models "), link(deepseek_67b), link(deepseek_v2), link(deepseek_v3)
text("- AI2's OLMo 2 "), link(olmo_7b), link(olmo2),
text("## Levels of openness")
text("- Closed models (e.g., GPT-4o): API access only "), link(gpt4)
text("- Open-weight models (e.g., DeepSeek): weights available, paper with architecture details, some training details, no data details "), link(deepseek_v3)
text("- Open-source models (e.g., OLMo): weights and data available, paper with most details (but not necessarily the rationale, failed experiments) "), link(olmo_7b)
text("## Today's frontier models")
text("- OpenAI's o3 "), link("https://openai.com/index/openai-o3-mini/")
text("- Anthropic's Claude Sonnet 3.7 "), link("https://www.anthropic.com/news/claude-3-7-sonnet")
text("- xAI's Grok 3 "), link("https://x.ai/news/grok-3")
text("- Google's Gemini 2.5 "), link("https://blog.google/technology/google-deepmind/gemini-model-thinking-updates-march-2025/")
text("- Meta's Llama 3.3 "), link("https://ai.meta.com/blog/meta-llama-3/")
text("- DeepSeek's r1 "), link(deepseek_r1)
text("- Alibaba's Qwen 2.5 Max "), link("https://qwenlm.github.io/blog/qwen2.5-max/")
text("- Tencent's Hunyuan-T1 "), link("https://tencent.github.io/llm.hunyuan.T1/README_EN.html")
def what_is_this_program():
text("This is an *executable lecture*, a program whose execution delivers the content of a lecture.")
text("Executable lectures make it possible to:")
text("- view and run code (since everything is code!),")
total = 0 # @inspect total
for x in [1, 2, 3]: # @inspect x
total += x # @inspect total
text("- see the hierarchical structure of the lecture, and")
text("- jump to definitions and concepts: "), link(supervised_finetuning)
def course_logistics():
text("All information online: "), link("https://stanford-cs336.github.io/spring2025/")
text("This is a 5-unit class.")
text("Comment from Spring 2024 course evaluation: *The entire assignment was approximately the same amount of work as all 5 assignments from CS 224n plus the final project. And that's just the first homework assignment.*")
text("## Why you should take this course")
text("- You have an obsessive need to understand how things work.")
text("- You want to build up your research engineering muscles.")
text("## Why you should not take this course")
text("- You actually want to get research done this quarter.
(Talk to your advisor.)")
text("- You are interested in learning about the hottest new techniques in AI (e.g., multimodality, RAG, etc.).
(You should take a seminar class for that.)")
text("- You want to get good results on your own application domain.
(You should just prompt or fine-tune an existing model.)")
text("## How you can follow along at home")
text("- All lecture materials and assignments will be posted online, so feel free to follow on your own.")
text("- Lectures are recorded via [CGOE, formally SCPD](https://cgoe.stanford.edu/) and be made available on YouTube (with some lag).")
text("- We plan to offer this class again next year.")
text("## Assignments")
text("- 5 assignments (basics, systems, scaling laws, data, alignment).")
text("- No scaffolding code, but we provide unit tests and adapter interfaces to help you check correctness.")
text("- Implement locally to test for correctness, then run on cluster for benchmarking (accuracy and speed).")
text("- Leaderboard for some assignments (minimize perplexity given training budget).")
text("- AI tools (e.g., CoPilot, Cursor) can take away from learning, so use at your own risk.")
text("## Cluster")
text("- Thanks to Together AI for providing a compute cluster. 🙏")
text("- Please read [the guide](https://docs.google.com/document/d/1BSSig7zInyjDKcbNGftVxubiHlwJ-ZqahQewIzBmBOo/edit) on how to use the cluster.")
text("- Start your assignments early, since the cluster will fill up close to the deadline!")
def course_components():
text("## It's all about efficiency")
text("Resources: data + hardware (compute, memory, communication bandwidth)")
text("How do you train the best model given a fixed set of resources?")
text("Example: given a Common Crawl dump and 32 H100s for 2 weeks, what should you do?")
text("Design decisions:")
image("images/design-decisions.png", width=800)
text("## Overview of the course")
basics()
systems()
scaling_laws()
data()
alignment()
text("## Efficiency drives design decisions")
text("Today, we are compute-constrained, so design decisions will reflect squeezing the most out of given hardware.")
text("- Data processing: avoid wasting precious compute updating on bad / irrelevant data")
text("- Tokenization: working with raw bytes is elegant, but compute-inefficient with today's model architectures.")
text("- Model architecture: many changes motivated by reducing memory or FLOPs (e.g., sharing KV caches, sliding window attention)")
text("- Training: we can get away with a single epoch!")
text("- Scaling laws: use less compute on smaller models to do hyperparameter tuning")
text("- Alignment: if tune model more to desired use cases, require smaller base models")
text("Tomorrow, we will become data-constrained...")
class Tokenizer(ABC):
"""Abstract interface for a tokenizer."""
def encode(self, string: str) -> list[int]:
raise NotImplementedError
def decode(self, indices: list[int]) -> str:
raise NotImplementedError
def basics():
text("Goal: get a basic version of the full pipeline working")
text("Components: tokenization, model architecture, training")
text("## Tokenization")
text("Tokenizers convert between strings and sequences of integers (tokens)")
image("images/tokenized-example.png", width=600)
text("Intuition: break up string into popular segments")
text("This course: Byte-Pair Encoding (BPE) tokenizer "), link(sennrich_2016)
text("Tokenizer-free approaches: "), link(byt5), link(megabyte), link(blt), link(tfree)
text("Use bytes directly, promising, but have not yet been scaled up to the frontier.")
text("## Architecture")
text("Starting point: original Transformer "), link(transformer_2017)
image("images/transformer-architecture.png", width=500)
text("Variants:")
text("- Activation functions: ReLU, SwiGLU "), link(shazeer_2020)
text("- Positional encodings: sinusoidal, RoPE "), link(rope_2021)
text("- Normalization: LayerNorm, RMSNorm "), link(layernorm_2016), link(rms_norm_2019)
text("- Placement of normalization: pre-norm versus post-norm "), link(pre_post_norm_2020)
text("- MLP: dense, mixture of experts "), link(moe_2017)
text("- Attention: full, sliding window, linear "), link(mistral_7b), link("https://arxiv.org/abs/2006.16236")
text("- Lower-dimensional attention: group-query attention (GQA), multi-head latent attention (MLA) "), link(gqa), link(mla)
text("- State-space models: Hyena "), link("https://arxiv.org/abs/2302.10866")
text("## Training")
text("- Optimizer (e.g., AdamW, Muon, SOAP) "), link(adam2014), link(adamw2017), link(muon), link(soap)
text("- Learning rate schedule (e.g., cosine, WSD) "), link(cosine_learning_rate_2017), link(wsd_2024)
text("- Batch size (e..g, critical batch size) "), link(large_batch_training_2018)
text("- Regularization (e.g., dropout, weight decay)")
text("- Hyperparameters (number of heads, hidden dimension): grid search")
text("## Assignment 1")
link(title="[GitHub]", url="https://github.com/stanford-cs336/assignment1-basics"), link(title="[PDF]", url="https://github.com/stanford-cs336/assignment1-basics/blob/main/cs336_spring2025_assignment1_basics.pdf")
text("- Implement BPE tokenizer")
text("- Implement Transformer, cross-entropy loss, AdamW optimizer, training loop")
text("- Train on TinyStories and OpenWebText")
text("- Leaderboard: minimize OpenWebText perplexity given 90 minutes on a H100 "), link(title="[last year's leaderboard]", url="https://github.com/stanford-cs336/spring2024-assignment1-basics-leaderboard")
def systems():
text("Goal: squeeze the most out of the hardware")
text("Components: kernels, parallelism, inference")
text("## Kernels")
text("What a GPU (A100) looks like:")
image("https://miro.medium.com/v2/resize:fit:2000/format:webp/1*6xoBKi5kL2dZpivFe1-zgw.jpeg", width=800)
text("Analogy: warehouse : DRAM :: factory : SRAM")
image("https://horace.io/img/perf_intro/factory_bandwidth.png", width=400)
text("Trick: organize computation to maximize utilization of GPUs by minimizing data movement")
text("Write kernels in CUDA/**Triton**/CUTLASS/ThunderKittens")
text("## Parallelism")
text("What if we have multiple GPUs (8 A100s)?")
image("https://www.fibermall.com/blog/wp-content/uploads/2024/09/the-hardware-topology-of-a-typical-8xA100-GPU-host.png", width=500)
text("Data movement between GPUs is even slower, but same 'minimize data movement' principle holds")
text("Use collective operations (e.g., gather, reduce, all-reduce)")
text("Shard (parameters, activations, gradients, optimizer states) across GPUs")
text("How to split computation: {data,tensor,pipeline,sequence} parallelism")
text("## Inference")
text("Goal: generate tokens given a prompt (needed to actually use models!)")
text("Inference is also needed for reinforcement learning, test-time compute, evaluation")
text("Globally, inference compute (every use) exceeds training compute (one-time cost)")
text("Two phases: prefill and decode")
image("images/prefill-decode.png", width=500)
text("Prefill (similar to training): tokens are given, can process all at once (compute-bound)")
text("Decode: need to generate one token at a time (memory-bound)")
text("Methods to speed up decoding:")
text("- Use cheaper model (via model pruning, quantization, distillation)")
text("- Speculative decoding: use a cheaper \"draft\" model to generate multiple tokens, then use the full model to score in parallel (exact decoding!)")
text("- Systems optimizations: KV caching, batching")
text("## Assignment 2")
link(title="[GitHub from 2024]", url="https://github.com/stanford-cs336/spring2024-assignment2-systems"), link(title="[PDF from 2024]", url="https://github.com/stanford-cs336/spring2024-assignment2-systems/blob/master/cs336_spring2024_assignment2_systems.pdf")
text("- Implement a fused RMSNorm kernel in Triton")
text("- Implement distributed data parallel training")
text("- Implement optimizer state sharding")
text("- Benchmark and profile the implementations")
def scaling_laws():
text("Goal: do experiments at small scale, predict hyperparameters/loss at large scale")
text("Question: given a FLOPs budget ($C$), use a bigger model ($N$) or train on more tokens ($D$)?")
text("Compute-optimal scaling laws: "), link(kaplan_scaling_laws_2020), link(chinchilla)
image("images/chinchilla-isoflop.png", width=800)
text("TL;DR: $D^* = 20 N^*$ (e.g., 1.4B parameter model should be trained on 28B tokens)")
text("But this doesn't take into account inference costs!")
text("## Assignment 3")
link(title="[GitHub from 2024]", url="https://github.com/stanford-cs336/spring2024-assignment3-scaling"), link(title="[PDF from 2024]", url="https://github.com/stanford-cs336/spring2024-assignment3-scaling/blob/master/cs336_spring2024_assignment3_scaling.pdf")
text("- We define a training API (hyperparameters -> loss) based on previous runs")
text("- Submit \"training jobs\" (under a FLOPs budget) and gather data points")
text("- Fit a scaling law to the data points")
text("- Submit predictions for scaled up hyperparameters")
text("- Leaderboard: minimize loss given FLOPs budget")
def data():
text("Question: What capabilities do we want the model to have?")
text("Multilingual? Code? Math?")
image("https://ar5iv.labs.arxiv.org/html/2101.00027/assets/pile_chart2.png", width=600)
text("## Evaluation")
text("- Perplexity: textbook evaluation for language models")
text("- Standardized testing (e.g., MMLU, HellaSwag, GSM8K)")
text("- Instruction following (e.g., AlpacaEval, IFEval, WildBench)")
text("- Scaling test-time compute: chain-of-thought, ensembling")
text("- LM-as-a-judge: evaluate generative tasks")
text("- Full system: RAG, agents")
text("## Data curation")
text("- Data does not just fall from the sky.")
look_at_web_data()
text("- Sources: webpages crawled from the Internet, books, arXiv papers, GitHub code, etc.")
text("- Appeal to fair use to train on copyright data? "), link("https://arxiv.org/pdf/2303.15715.pdf")
text("- Might have to license data (e.g., Google with Reddit data) "), article_link("https://www.reuters.com/technology/reddit-ai-content-licensing-deal-with-google-sources-say-2024-02-22/")
text("- Formats: HTML, PDF, directories (not text!)")
text("## Data processing")
text("- Transformation: convert HTML/PDF to text (preserve content, some structure, rewriting)")
text("- Filtering: keep high quality data, remove harmful content (via classifiers)")
text("- Deduplication: save compute, avoid memorization; use Bloom filters or MinHash")
text("## Assignment 4")
link(title="[GitHub from 2024]", url="https://github.com/stanford-cs336/spring2024-assignment4-data"), link(title="[PDF from 2024]", url="https://github.com/stanford-cs336/spring2024-assignment4-data/blob/master/cs336_spring2024_assignment4_data.pdf")
text("- Convert Common Crawl HTML to text")
text("- Train classifiers to filter for quality and harmful content")
text("- Deduplication using MinHash")
text("- Leaderboard: minimize perplexity given token budget")
def look_at_web_data():
urls = get_common_crawl_urls()[:3] # @inspect urls
documents = list(read_common_crawl(urls[1], limit=300))
random.seed(40)
random.shuffle(documents)
documents = markdownify_documents(documents[:10])
write_documents(documents, "var/sample-documents.txt")
link(title="[sample documents]", url="var/sample-documents.txt")
text("It's a wasteland out there! Need to really process the data.")
def alignment():
text("So far, a **base model** is raw potential, very good at completing the next token.")
text("Alignment makes the model actually useful.")
text("Goals of alignment:")
text("- Get the language model to follow instructions")
text("- Tune the style (format, length, tone, etc.)")
text("- Incorporate safety (e.g., refusals to answer harmful questions)")
text("Two phases:")
supervised_finetuning()
learning_from_feedback()
text("## Assignment 5")
link(title="[GitHub from 2024]", url="https://github.com/stanford-cs336/spring2024-assignment5-alignment"), link(title="[PDF from 2024]", url="https://github.com/stanford-cs336/spring2024-assignment5-alignment/blob/master/cs336_spring2024_assignment5_alignment.pdf")
text("- Implement supervised fine-tuning")
text("- Implement Direct Preference Optimization (DPO)")
text("- Implement Group Relative Preference Optimization (GRPO)")
@dataclass(frozen=True)
class Turn:
role: str
content: str
@dataclass(frozen=True)
class ChatExample:
turns: list[Turn]
@dataclass(frozen=True)
class PreferenceExample:
history: list[Turn]
response_a: str
response_b: str
chosen: str
def supervised_finetuning():
text("## Supervised finetuning (SFT)")
text("Instruction data: (prompt, response) pairs")
sft_data: list[ChatExample] = [
ChatExample(
turns=[
Turn(role="system", content="You are a helpful assistant."),
Turn(role="user", content="What is 1 + 1?"),
Turn(role="assistant", content="The answer is 2."),
],
),
]
text("Data often involves human annotation.")
text("Intuition: base model already has the skills, just need few examples to surface them. "), link(lima)
text("Supervised learning: fine-tune model to maximize p(response | prompt).")
def learning_from_feedback():
text("Now we have a preliminary instruction following model.")
text("Let's make it better without expensive annotation.")
text("## Preference data")
text("Data: generate multiple responses using model (e.g., [A, B]) to a given prompt.")
text("User provides preferences (e.g., A < B or A > B).")
preference_data: list[PreferenceExample] = [
PreferenceExample(
history=[
Turn(role="system", content="You are a helpful assistant."),
Turn(role="user", content="What is the best way to train a language model?"),
],
response_a="You should use a large dataset and train for a long time.",
response_b="You should use a small dataset and train for a short time.",
chosen="a",
)
]
text("## Verifiers")
text("- Formal verifiers (e.g., for code, math)")
text("- Learned verifiers: train against an LM-as-a-judge")
text("## Algorithms")
text("- Proximal Policy Optimization (PPO) from reinforcement learning "), link(ppo2017), link(instruct_gpt)
text("- Direct Policy Optimization (DPO): for preference data, simpler "), link(dpo)
text("- Group Relative Preference Optimization (GRPO): remove value function "), link(grpo)
############################################################
# Tokenization
# https://github.com/openai/tiktoken/blob/main/tiktoken_ext/openai_public.py#L23
GPT2_TOKENIZER_REGEX = \
r"""'(?:[sdmt]|ll|ve|re)| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
def tokenization():
text("This unit was inspired by Andrej Karpathy's video on tokenization; check it out! "), youtube_link("https://www.youtube.com/watch?v=zduSFxRajkE")
intro_to_tokenization()
tokenization_examples()
character_tokenizer()
byte_tokenizer()
word_tokenizer()
bpe_tokenizer()
text("## Summary")
text("- Tokenizer: strings <-> tokens (indices)")
text("- Character-based, byte-based, word-based tokenization highly suboptimal")
text("- BPE is an effective heuristic that looks at corpus statistics")
text("- Tokenization is a necessary evil, maybe one day we'll just do it from bytes...")
@dataclass(frozen=True)
class BPETokenizerParams:
"""All you need to specify a BPETokenizer."""
vocab: dict[int, bytes] # index -> bytes
merges: dict[tuple[int, int], int] # index1,index2 -> new_index
class CharacterTokenizer(Tokenizer):
"""Represent a string as a sequence of Unicode code points."""
def encode(self, string: str) -> list[int]:
return list(map(ord, string))
def decode(self, indices: list[int]) -> str:
return "".join(map(chr, indices))
class ByteTokenizer(Tokenizer):
"""Represent a string as a sequence of bytes."""
def encode(self, string: str) -> list[int]:
string_bytes = string.encode("utf-8") # @inspect string_bytes
indices = list(map(int, string_bytes)) # @inspect indices
return indices
def decode(self, indices: list[int]) -> str:
string_bytes = bytes(indices) # @inspect string_bytes
string = string_bytes.decode("utf-8") # @inspect string
return string
def merge(indices: list[int], pair: tuple[int, int], new_index: int) -> list[int]: # @inspect indices, @inspect pair, @inspect new_index
"""Return `indices`, but with all instances of `pair` replaced with `new_index`."""
new_indices = [] # @inspect new_indices
i = 0 # @inspect i
while i < len(indices):
if i + 1 < len(indices) and indices[i] == pair[0] and indices[i + 1] == pair[1]:
new_indices.append(new_index)
i += 2
else:
new_indices.append(indices[i])
i += 1
return new_indices
class BPETokenizer(Tokenizer):
"""BPE tokenizer given a set of merges and a vocabulary."""
def __init__(self, params: BPETokenizerParams):
self.params = params
def encode(self, string: str) -> list[int]:
indices = list(map(int, string.encode("utf-8"))) # @inspect indices
# Note: this is a very slow implementation
for pair, new_index in self.params.merges.items(): # @inspect pair, @inspect new_index
indices = merge(indices, pair, new_index)
return indices
def decode(self, indices: list[int]) -> str:
bytes_list = list(map(self.params.vocab.get, indices)) # @inspect bytes_list
string = b"".join(bytes_list).decode("utf-8") # @inspect string
return string
def get_compression_ratio(string: str, indices: list[int]) -> float:
"""Given `string` that has been tokenized into `indices`, ."""
num_bytes = len(bytes(string, encoding="utf-8")) # @inspect num_bytes
num_tokens = len(indices) # @inspect num_tokens
return num_bytes / num_tokens
def get_gpt2_tokenizer():
# Code: https://github.com/openai/tiktoken
# You can use cl100k_base for the gpt3.5-turbo or gpt4 tokenizer
return tiktoken.get_encoding("gpt2")
def intro_to_tokenization():
text("Raw text is generally represented as Unicode strings.")
string = "Hello, 🌍! 你好!"
text("A language model places a probability distribution over sequences of tokens (usually represented by integer indices).")
indices = [15496, 11, 995, 0]
text("So we need a procedure that *encodes* strings into tokens.")
text("We also need a procedure that *decodes* tokens back into strings.")
text("A "), link(Tokenizer), text(" is a class that implements the encode and decode methods.")
text("The **vocabulary size** is number of possible tokens (integers).")
def tokenization_examples():
text("To get a feel for how tokenizers work, play with this "), link(title="interactive site", url="https://tiktokenizer.vercel.app/?encoder=gpt2")
text("## Observations")
text("- A word and its preceding space are part of the same token (e.g., \" world\").")
text("- A word at the beginning and in the middle are represented differently (e.g., \"hello hello\").")
text("- Numbers are tokenized into every few digits.")
text("Here's the GPT-2 tokenizer from OpenAI (tiktoken) in action.")
tokenizer = get_gpt2_tokenizer()
string = "Hello, 🌍! 你好!" # @inspect string
text("Check that encode() and decode() roundtrip:")
indices = tokenizer.encode(string) # @inspect indices
reconstructed_string = tokenizer.decode(indices) # @inspect reconstructed_string
assert string == reconstructed_string
compression_ratio = get_compression_ratio(string, indices) # @inspect compression_ratio
def character_tokenizer():
text("## Character-based tokenization")
text("A Unicode string is a sequence of Unicode characters.")
text("Each character can be converted into a code point (integer) via `ord`.")
assert ord("a") == 97
assert ord("🌍") == 127757
text("It can be converted back via `chr`.")
assert chr(97) == "a"
assert chr(127757) == "🌍"
text("Now let's build a `Tokenizer` and make sure it round-trips:")
tokenizer = CharacterTokenizer()
string = "Hello, 🌍! 你好!" # @inspect string
indices = tokenizer.encode(string) # @inspect indices
reconstructed_string = tokenizer.decode(indices) # @inspect reconstructed_string
assert string == reconstructed_string
text("There are approximately 150K Unicode characters. "), link(title="[Wikipedia]", url="https://en.wikipedia.org/wiki/List_of_Unicode_characters")
vocabulary_size = max(indices) + 1 # This is a lower bound @inspect vocabulary_size
text("Problem 1: this is a very large vocabulary.")
text("Problem 2: many characters are quite rare (e.g., 🌍), which is inefficient use of the vocabulary.")
compression_ratio = get_compression_ratio(string, indices) # @inspect compression_ratio
def byte_tokenizer():
text("## Byte-based tokenization")
text("Unicode strings can be represented as a sequence of bytes, which can be represented by integers between 0 and 255.")
text("The most common Unicode encoding is "), link(title="UTF-8", url="https://en.wikipedia.org/wiki/UTF-8")
text("Some Unicode characters are represented by one byte:")
assert bytes("a", encoding="utf-8") == b"a"
text("Others take multiple bytes:")
assert bytes("🌍", encoding="utf-8") == b"\xf0\x9f\x8c\x8d"
text("Now let's build a `Tokenizer` and make sure it round-trips:")
tokenizer = ByteTokenizer()
string = "Hello, 🌍! 你好!" # @inspect string
indices = tokenizer.encode(string) # @inspect indices
reconstructed_string = tokenizer.decode(indices) # @inspect reconstructed_string
assert string == reconstructed_string
text("The vocabulary is nice and small: a byte can represent 256 values.")
vocabulary_size = 256 # @inspect vocabulary_size
text("What about the compression rate?")
compression_ratio = get_compression_ratio(string, indices) # @inspect compression_ratio
assert compression_ratio == 1
text("The compression ratio is terrible, which means the sequences will be too long.")
text("Given that the context length of a Transformer is limited (since attention is quadratic), this is not looking great...")
def word_tokenizer():
text("## Word-based tokenization")
text("Another approach (closer to what was done classically in NLP) is to split strings into words.")
string = "I'll say supercalifragilisticexpialidocious!"
segments = regex.findall(r"\w+|.", string) # @inspect segments
text("This regular expression keeps all alphanumeric characters together (words).")
text("Here is a fancier version:")
pattern = GPT2_TOKENIZER_REGEX # @inspect pattern
segments = regex.findall(pattern, string) # @inspect segments
text("To turn this into a `Tokenizer`, we need to map these segments into integers.")
text("Then, we can build a mapping from each segment into an integer.")
text("But there are problems:")
text("- The number of words is huge (like for Unicode characters).")
text("- Many words are rare and the model won't learn much about them.")
text("- This doesn't obviously provide a fixed vocabulary size.")
text("New words we haven't seen during training get a special UNK token, which is ugly and can mess up perplexity calculations.")
vocabulary_size = "Number of distinct segments in the training data"
compression_ratio = get_compression_ratio(string, segments) # @inspect compression_ratio
def bpe_tokenizer():
text("## Byte Pair Encoding (BPE)")
link(title="[Wikipedia]", url="https://en.wikipedia.org/wiki/Byte_pair_encoding")
text("The BPE algorithm was introduced by Philip Gage in 1994 for data compression. "), article_link("http://www.pennelynn.com/Documents/CUJ/HTML/94HTML/19940045.HTM")
text("It was adapted to NLP for neural machine translation. "), link(sennrich_2016)
text("(Previously, papers had been using word-based tokenization.)")
text("BPE was then used by GPT-2. "), link(gpt2)
text("Basic idea: *train* the tokenizer on raw text to automatically determine the vocabulary.")
text("Intuition: common sequences of characters are represented by a single token, rare sequences are represented by many tokens.")
text("The GPT-2 paper used word-based tokenization to break up the text into inital segments and run the original BPE algorithm on each segment.")
text("Sketch: start with each byte as a token, and successively merge the most common pair of adjacent tokens.")
text("## Training the tokenizer")
string = "the cat in the hat" # @inspect string
params = train_bpe(string, num_merges=3)
text("## Using the tokenizer")
text("Now, given a new text, we can encode it.")
tokenizer = BPETokenizer(params)
string = "the quick brown fox" # @inspect string
indices = tokenizer.encode(string) # @inspect indices
reconstructed_string = tokenizer.decode(indices) # @inspect reconstructed_string
assert string == reconstructed_string
text("In Assignment 1, you will go beyond this in the following ways:")
text("- encode() currently loops over all merges. Only loop over merges that matter.")
text("- Detect and preserve special tokens (e.g., <|endoftext|>).")
text("- Use pre-tokenization (e.g., the GPT-2 tokenizer regex).")
text("- Try to make the implementation as fast as possible.")
def train_bpe(string: str, num_merges: int) -> BPETokenizerParams: # @inspect string, @inspect num_merges
text("Start with the list of bytes of `string`.")
indices = list(map(int, string.encode("utf-8"))) # @inspect indices
merges: dict[tuple[int, int], int] = {} # index1, index2 => merged index
vocab: dict[int, bytes] = {x: bytes([x]) for x in range(256)} # index -> bytes
for i in range(num_merges):
text("Count the number of occurrences of each pair of tokens")
counts = defaultdict(int)
for index1, index2 in zip(indices, indices[1:]): # For each adjacent pair
counts[(index1, index2)] += 1 # @inspect counts
text("Find the most common pair.")
pair = max(counts, key=counts.get) # @inspect pair
index1, index2 = pair
text("Merge that pair.")
new_index = 256 + i # @inspect new_index
merges[pair] = new_index # @inspect merges
vocab[new_index] = vocab[index1] + vocab[index2] # @inspect vocab
indices = merge(indices, pair, new_index) # @inspect indices
return BPETokenizerParams(vocab=vocab, merges=merges)
if __name__ == "__main__":
main()