{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "#hide\n", "from utils import *" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Making our RNN state of the art" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "#hide\n", "from fastai2.text.all import *\n", "path = untar_data(URLs.HUMAN_NUMBERS)\n", "lines = L()\n", "with open(path/'train.txt') as f: lines += L(*f.readlines())\n", "with open(path/'valid.txt') as f: lines += L(*f.readlines())\n", "text = ' . '.join([l.strip() for l in lines])\n", "tokens = text.split(' ')\n", "vocab = L(*tokens).unique()\n", "word2idx = {w:i for i,w in enumerate(vocab)}\n", "nums = L(word2idx[i] for i in tokens)\n", "\n", "def group_chunks(ds, bs):\n", " m = len(ds) // bs\n", " new_ds = L()\n", " for i in range(m): new_ds += L(ds[i + m*j] for j in range(bs))\n", " return new_ds" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "sl,bs = 16,64\n", "seqs = L((tensor(nums[i:i+sl]), tensor(nums[i+1:i+sl+1])) for i in range(0,len(nums)-sl-1,sl))\n", "cut = int(len(seqs) * 0.8)\n", "dls = DataLoaders.from_dsets(group_chunks(seqs[:cut], bs), group_chunks(seqs[cut:], bs), bs=bs, drop_last=True, shuffle=False)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Multilayer RNNs" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### The model" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "class LMModel5(Module):\n", " def __init__(self, vocab_sz, n_hidden, n_layers):\n", " self.i_h = nn.Embedding(vocab_sz, n_hidden)\n", " self.rnn = nn.RNN(n_hidden, n_hidden, n_layers, batch_first=True)\n", " self.h_o = nn.Linear(n_hidden, vocab_sz)\n", " self.h = torch.zeros(n_layers, bs, n_hidden)\n", " \n", " def forward(self, x):\n", " res,h = self.rnn(self.i_h(x), self.h)\n", " self.h = h.detach()\n", " return self.h_o(res)\n", " \n", " def reset(self): self.h.zero_()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
epochtrain_lossvalid_lossaccuracytime
03.0481152.6223840.43400100:02
12.1363881.7639670.47119100:02
21.6892461.8987180.36474600:02
31.4435451.7474400.48038700:01
41.2710231.8709390.47998000:02
51.1012591.7944280.49536100:02
60.9483801.7696440.51114900:02
70.8223731.8004060.53540000:01
80.7311881.9140650.52246100:01
90.6626591.9875470.52579800:02
100.6130532.0221020.52775100:02
110.5770072.0684720.52653000:02
120.5511442.1135330.52156600:02
130.5353562.1230890.52360000:02
140.5267832.1224130.52433300:02
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "learn = Learner(dls, LMModel5(len(vocab), 64, 2), loss_func=CrossEntropyLossFlat(), metrics=accuracy, cbs=ModelReseter)\n", "learn.fit_one_cycle(15, 3e-3)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Handling exploding or disappearing activations" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## LSTM" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Building an LSTM from scratch" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "class LSTMCell(Module):\n", " def __init__(self, ni, nh):\n", " self.forget_gate = nn.Linear(ni + nh, nh)\n", " self.input_gate = nn.Linear(ni + nh, nh)\n", " self.cell_gate = nn.Linear(ni + nh, nh)\n", " self.output_gate = nn.Linear(ni + nh, nh)\n", "\n", " def forward(self, input, state):\n", " h,c = state\n", " h = torch.stack([x, input], dim=1)\n", " forget = torch.sigmoid(self.forget_gate(h))\n", " c = c * forget\n", " inp = torch.sigmoid(self.input_gate(h))\n", " cell = torch.tanh(self.cell_gate(h))\n", " c = c + inp * cell\n", " out = torch.sigmoid(self.output_gate(h))\n", " h = outgate * torch.tanh(c)\n", " return h, (h,c)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "class LSTMCell(Module):\n", " def __init__(self, ni, nh):\n", " self.ih = nn.Linear(ni,4*nh)\n", " self.hh = nn.Linear(nh,4*nh)\n", "\n", " def forward(self, input, state):\n", " h,c = state\n", " #One big multiplication for all the gates is better than 4 smaller ones\n", " gates = (self.ih(input) + self.hh(h)).chunk(4, 1)\n", " ingate,forgetgate,outgate = map(torch.sigmoid, gates[:3])\n", " cellgate = gates[3].tanh()\n", "\n", " c = (forgetgate*c) + (ingate*cellgate)\n", " h = outgate * c.tanh()\n", " return h, (h,c)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Training a language model using LSTMs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "class LMModel6(Module):\n", " def __init__(self, vocab_sz, n_hidden, n_layers):\n", " self.i_h = nn.Embedding(vocab_sz, n_hidden)\n", " self.rnn = nn.LSTM(n_hidden, n_hidden, n_layers, batch_first=True)\n", " self.h_o = nn.Linear(n_hidden, vocab_sz)\n", " self.h = [torch.zeros(2, bs, n_hidden) for _ in range(n_layers)]\n", " \n", " def forward(self, x):\n", " res,h = self.rnn(self.i_h(x), self.h)\n", " self.h = [h_.detach() for h_ in h]\n", " return self.h_o(res)\n", " \n", " def reset(self): \n", " for h in self.h: h.zero_()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
epochtrain_lossvalid_lossaccuracytime
03.0313462.7493810.27921500:03
12.2196512.0844500.20475300:03
21.6595181.6856390.47957400:03
31.4105501.6666630.50944000:03
41.2040621.6064850.54182900:03
51.0214591.5291090.59244800:03
60.7858711.3402800.64200800:03
70.5475191.2717100.68880200:03
80.3397751.2166050.75382500:03
90.1975501.2185570.74365200:02
100.1142971.2535710.75113900:03
110.0713011.3148270.75268600:03
120.0495071.3073750.76546200:03
130.0388101.2877790.76774100:03
140.0337381.2929510.76798500:03
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "learn = Learner(dls, LMModel6(len(vocab), 64, 2), loss_func=CrossEntropyLossFlat(), metrics=accuracy, cbs=ModelReseter)\n", "learn.fit_one_cycle(15, 1e-2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Regularizing an LSTM" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Dropout" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "class Dropout(Module):\n", " def __init__(self, p): self.p = p\n", " def forward(self, x):\n", " if self.training: return x\n", " mask = x.new(*x.shape).bernoulli_(1-p)\n", " return x * mask.div_(1-p)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### AR and TAR regularization" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Training a regularized LSTM" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "class LMModel7(Module):\n", " def __init__(self, vocab_sz, n_hidden, n_layers, p):\n", " self.i_h = nn.Embedding(vocab_sz, n_hidden)\n", " self.rnn = nn.LSTM(n_hidden, n_hidden, n_layers, batch_first=True)\n", " self.drop = nn.Dropout(p)\n", " self.h_o = nn.Linear(n_hidden, vocab_sz)\n", " self.h = [torch.zeros(2, bs, n_hidden) for _ in range(n_layers)]\n", " \n", " def forward(self, x):\n", " raw,h = self.rnn(self.i_h(x), self.h)\n", " out = self.drop(raw)\n", " self.h = [h_.detach() for h_ in h]\n", " return self.h_o(out),raw,out\n", " \n", " def reset(self): \n", " for h in self.h: h.zero_()" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "learn = Learner(dls, LMModel7(len(vocab), 64, 2, 0.4), loss_func=CrossEntropyLossFlat(), \n", " metrics=accuracy, cbs=[ModelReseter, RNNRegularizer(alpha=2, beta=1)])" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
epochtrain_lossvalid_lossaccuracytime
03.1455532.4959940.43758100:03
12.3331891.6744630.49186200:03
21.6787531.5005360.55395500:03
31.1119041.0401090.74877900:03
40.7078290.7733690.80769900:02
50.4658990.6211590.82934600:03
60.3352490.6499260.83919300:03
70.2544180.5869890.84106400:03
80.2051910.5272880.85017900:02
90.1728760.4600110.86865200:02
100.1514520.5006040.86067700:03
110.1368720.4803420.86352500:03
120.1275760.4965340.85839800:03
130.1221870.4759310.86702500:03
140.1195380.4903660.86116500:03
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "learn = TextLearner(dls, LMModel7(len(vocab), 64, 2, 0.4), loss_func=CrossEntropyLossFlat(), \n", " metrics=accuracy)\n", "learn.fit_one_cycle(15, 1e-2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Conclusion" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "jupytext": { "split_at_heading": true }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4" } }, "nbformat": 4, "nbformat_minor": 2 }