{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Uncomment and run this cell if you're on Colab or Kaggle\n",
"# !git clone https://github.com/nlp-with-transformers/notebooks.git\n",
"# %cd notebooks\n",
"# from install import *\n",
"# install_requirements()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using transformers v4.11.3\n",
"Using datasets v1.13.0\n",
"Using accelerate v0.5.1\n"
]
}
],
"source": [
"#hide\n",
"from utils import *\n",
"setup_chapter()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Transformer Anatomy"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## The Transformer Architecture"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## The Encoder"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Self-Attention"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Scaled dot-product attention"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#hide\n",
"\n",
"Copy and execute the following cell magic in a new cell to use `bertviz` in JupyterLab:\n",
"\n",
"```python\n",
"%%javascript\n",
"require.config({\n",
" paths: {\n",
" d3: '//cdnjs.cloudflare.com/ajax/libs/d3/3.4.8/d3.min',\n",
" jquery: '//ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min',\n",
" }\n",
"});\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 433/433 [00:00<00:00, 95270.09B/s]\n",
"100%|██████████| 440473133/440473133 [01:00<00:00, 7224426.06B/s] \n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
" \n",
"
\n",
" Layer: \n",
" Head: \n",
" \n",
"
\n",
"
\n",
"
\n",
" Layer: \n",
" Attention: \n",
" \n",
"
\n",
"