{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
"- Author: Sebastian Raschka\n",
"- GitHub Repository: https://github.com/rasbt/deeplearning-models"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Sebastian Raschka \n",
"\n",
"CPython 3.7.3\n",
"IPython 7.6.1\n",
"\n",
"torch 1.1.0\n"
]
}
],
"source": [
"%load_ext watermark\n",
"%watermark -a 'Sebastian Raschka' -v -p torch"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"- Runs on CPU or GPU (if available)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Model Zoo -- Convolutional Neural Network"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Imports"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"import numpy as np\n",
"import torch\n",
"import torch.nn.functional as F\n",
"from torchvision import datasets\n",
"from torchvision import transforms\n",
"from torch.utils.data import DataLoader\n",
"\n",
"\n",
"if torch.cuda.is_available():\n",
" torch.backends.cudnn.deterministic = True"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Settings and Dataset"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Image batch dimensions: torch.Size([128, 1, 28, 28])\n",
"Image label dimensions: torch.Size([128])\n"
]
}
],
"source": [
"##########################\n",
"### SETTINGS\n",
"##########################\n",
"\n",
"# Device\n",
"device = torch.device(\"cuda:3\" if torch.cuda.is_available() else \"cpu\")\n",
"\n",
"# Hyperparameters\n",
"random_seed = 1\n",
"learning_rate = 0.05\n",
"num_epochs = 10\n",
"batch_size = 128\n",
"\n",
"# Architecture\n",
"num_classes = 10\n",
"\n",
"\n",
"##########################\n",
"### MNIST DATASET\n",
"##########################\n",
"\n",
"# Note transforms.ToTensor() scales input images\n",
"# to 0-1 range\n",
"train_dataset = datasets.MNIST(root='data', \n",
" train=True, \n",
" transform=transforms.ToTensor(),\n",
" download=True)\n",
"\n",
"test_dataset = datasets.MNIST(root='data', \n",
" train=False, \n",
" transform=transforms.ToTensor())\n",
"\n",
"\n",
"train_loader = DataLoader(dataset=train_dataset, \n",
" batch_size=batch_size, \n",
" shuffle=True)\n",
"\n",
"test_loader = DataLoader(dataset=test_dataset, \n",
" batch_size=batch_size, \n",
" shuffle=False)\n",
"\n",
"# Checking the dataset\n",
"for images, labels in train_loader: \n",
" print('Image batch dimensions:', images.shape)\n",
" print('Image label dimensions:', labels.shape)\n",
" break"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Model"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"##########################\n",
"### MODEL\n",
"##########################\n",
"\n",
"\n",
"class ConvNet(torch.nn.Module):\n",
"\n",
" def __init__(self, num_classes):\n",
" super(ConvNet, self).__init__()\n",
" \n",
" # calculate same padding:\n",
" # (w - k + 2*p)/s + 1 = o\n",
" # => p = (s(o-1) - w + k)/2\n",
" \n",
" # 28x28x1 => 28x28x8\n",
" self.conv_1 = torch.nn.Conv2d(in_channels=1,\n",
" out_channels=8,\n",
" kernel_size=(3, 3),\n",
" stride=(1, 1),\n",
" padding=1) # (1(28-1) - 28 + 3) / 2 = 1\n",
" # 28x28x8 => 14x14x8\n",
" self.pool_1 = torch.nn.MaxPool2d(kernel_size=(2, 2),\n",
" stride=(2, 2),\n",
" padding=0) # (2(14-1) - 28 + 2) = 0 \n",
" # 14x14x8 => 14x14x16\n",
" self.conv_2 = torch.nn.Conv2d(in_channels=8,\n",
" out_channels=16,\n",
" kernel_size=(3, 3),\n",
" stride=(1, 1),\n",
" padding=1) # (1(14-1) - 14 + 3) / 2 = 1 \n",
" # 14x14x16 => 7x7x16 \n",
" self.pool_2 = torch.nn.MaxPool2d(kernel_size=(2, 2),\n",
" stride=(2, 2),\n",
" padding=0) # (2(7-1) - 14 + 2) = 0\n",
"\n",
" self.linear_1 = torch.nn.Linear(7*7*16, num_classes)\n",
"\n",
" # optionally initialize weights from Gaussian;\n",
" # Guassian weight init is not recommended and only for demonstration purposes\n",
" for m in self.modules():\n",
" if isinstance(m, torch.nn.Conv2d) or isinstance(m, torch.nn.Linear):\n",
" m.weight.data.normal_(0.0, 0.01)\n",
" m.bias.data.zero_()\n",
" if m.bias is not None:\n",
" m.bias.detach().zero_()\n",
" \n",
" \n",
" def forward(self, x):\n",
" out = self.conv_1(x)\n",
" out = F.relu(out)\n",
" out = self.pool_1(out)\n",
"\n",
" out = self.conv_2(out)\n",
" out = F.relu(out)\n",
" out = self.pool_2(out)\n",
" \n",
" logits = self.linear_1(out.view(-1, 7*7*16))\n",
" probas = F.softmax(logits, dim=1)\n",
" return logits, probas\n",
"\n",
" \n",
"torch.manual_seed(random_seed)\n",
"model = ConvNet(num_classes=num_classes)\n",
"\n",
"model = model.to(device)\n",
"\n",
"optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Training"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch: 001/010 | Batch 000/469 | Cost: 2.3026\n",
"Epoch: 001/010 | Batch 050/469 | Cost: 2.3036\n",
"Epoch: 001/010 | Batch 100/469 | Cost: 2.3001\n",
"Epoch: 001/010 | Batch 150/469 | Cost: 2.3050\n",
"Epoch: 001/010 | Batch 200/469 | Cost: 2.2984\n",
"Epoch: 001/010 | Batch 250/469 | Cost: 2.2986\n",
"Epoch: 001/010 | Batch 300/469 | Cost: 2.2983\n",
"Epoch: 001/010 | Batch 350/469 | Cost: 2.2941\n",
"Epoch: 001/010 | Batch 400/469 | Cost: 2.2962\n",
"Epoch: 001/010 | Batch 450/469 | Cost: 2.2265\n",
"Epoch: 001/010 training accuracy: 65.38%\n",
"Time elapsed: 0.24 min\n",
"Epoch: 002/010 | Batch 000/469 | Cost: 1.8989\n",
"Epoch: 002/010 | Batch 050/469 | Cost: 0.6029\n",
"Epoch: 002/010 | Batch 100/469 | Cost: 0.6099\n",
"Epoch: 002/010 | Batch 150/469 | Cost: 0.4786\n",
"Epoch: 002/010 | Batch 200/469 | Cost: 0.4518\n",
"Epoch: 002/010 | Batch 250/469 | Cost: 0.3553\n",
"Epoch: 002/010 | Batch 300/469 | Cost: 0.3167\n",
"Epoch: 002/010 | Batch 350/469 | Cost: 0.2241\n",
"Epoch: 002/010 | Batch 400/469 | Cost: 0.2259\n",
"Epoch: 002/010 | Batch 450/469 | Cost: 0.3056\n",
"Epoch: 002/010 training accuracy: 93.11%\n",
"Time elapsed: 0.47 min\n",
"Epoch: 003/010 | Batch 000/469 | Cost: 0.3313\n",
"Epoch: 003/010 | Batch 050/469 | Cost: 0.1042\n",
"Epoch: 003/010 | Batch 100/469 | Cost: 0.1328\n",
"Epoch: 003/010 | Batch 150/469 | Cost: 0.2803\n",
"Epoch: 003/010 | Batch 200/469 | Cost: 0.0975\n",
"Epoch: 003/010 | Batch 250/469 | Cost: 0.1839\n",
"Epoch: 003/010 | Batch 300/469 | Cost: 0.1774\n",
"Epoch: 003/010 | Batch 350/469 | Cost: 0.1143\n",
"Epoch: 003/010 | Batch 400/469 | Cost: 0.1753\n",
"Epoch: 003/010 | Batch 450/469 | Cost: 0.1543\n",
"Epoch: 003/010 training accuracy: 95.68%\n",
"Time elapsed: 0.70 min\n",
"Epoch: 004/010 | Batch 000/469 | Cost: 0.1057\n",
"Epoch: 004/010 | Batch 050/469 | Cost: 0.1035\n",
"Epoch: 004/010 | Batch 100/469 | Cost: 0.1851\n",
"Epoch: 004/010 | Batch 150/469 | Cost: 0.1608\n",
"Epoch: 004/010 | Batch 200/469 | Cost: 0.1458\n",
"Epoch: 004/010 | Batch 250/469 | Cost: 0.1913\n",
"Epoch: 004/010 | Batch 300/469 | Cost: 0.1295\n",
"Epoch: 004/010 | Batch 350/469 | Cost: 0.1518\n",
"Epoch: 004/010 | Batch 400/469 | Cost: 0.1717\n",
"Epoch: 004/010 | Batch 450/469 | Cost: 0.0792\n",
"Epoch: 004/010 training accuracy: 96.46%\n",
"Time elapsed: 0.93 min\n",
"Epoch: 005/010 | Batch 000/469 | Cost: 0.0905\n",
"Epoch: 005/010 | Batch 050/469 | Cost: 0.1622\n",
"Epoch: 005/010 | Batch 100/469 | Cost: 0.1934\n",
"Epoch: 005/010 | Batch 150/469 | Cost: 0.1874\n",
"Epoch: 005/010 | Batch 200/469 | Cost: 0.0742\n",
"Epoch: 005/010 | Batch 250/469 | Cost: 0.1056\n",
"Epoch: 005/010 | Batch 300/469 | Cost: 0.0997\n",
"Epoch: 005/010 | Batch 350/469 | Cost: 0.0948\n",
"Epoch: 005/010 | Batch 400/469 | Cost: 0.0575\n",
"Epoch: 005/010 | Batch 450/469 | Cost: 0.1157\n",
"Epoch: 005/010 training accuracy: 96.97%\n",
"Time elapsed: 1.16 min\n",
"Epoch: 006/010 | Batch 000/469 | Cost: 0.1326\n",
"Epoch: 006/010 | Batch 050/469 | Cost: 0.1549\n",
"Epoch: 006/010 | Batch 100/469 | Cost: 0.0784\n",
"Epoch: 006/010 | Batch 150/469 | Cost: 0.0898\n",
"Epoch: 006/010 | Batch 200/469 | Cost: 0.0991\n",
"Epoch: 006/010 | Batch 250/469 | Cost: 0.0965\n",
"Epoch: 006/010 | Batch 300/469 | Cost: 0.0477\n",
"Epoch: 006/010 | Batch 350/469 | Cost: 0.0712\n",
"Epoch: 006/010 | Batch 400/469 | Cost: 0.1109\n",
"Epoch: 006/010 | Batch 450/469 | Cost: 0.0325\n",
"Epoch: 006/010 training accuracy: 97.60%\n",
"Time elapsed: 1.39 min\n",
"Epoch: 007/010 | Batch 000/469 | Cost: 0.0665\n",
"Epoch: 007/010 | Batch 050/469 | Cost: 0.0868\n",
"Epoch: 007/010 | Batch 100/469 | Cost: 0.0427\n",
"Epoch: 007/010 | Batch 150/469 | Cost: 0.0385\n",
"Epoch: 007/010 | Batch 200/469 | Cost: 0.0611\n",
"Epoch: 007/010 | Batch 250/469 | Cost: 0.0484\n",
"Epoch: 007/010 | Batch 300/469 | Cost: 0.1288\n",
"Epoch: 007/010 | Batch 350/469 | Cost: 0.0309\n",
"Epoch: 007/010 | Batch 400/469 | Cost: 0.0359\n",
"Epoch: 007/010 | Batch 450/469 | Cost: 0.0139\n",
"Epoch: 007/010 training accuracy: 97.64%\n",
"Time elapsed: 1.62 min\n",
"Epoch: 008/010 | Batch 000/469 | Cost: 0.0939\n",
"Epoch: 008/010 | Batch 050/469 | Cost: 0.1478\n",
"Epoch: 008/010 | Batch 100/469 | Cost: 0.0769\n",
"Epoch: 008/010 | Batch 150/469 | Cost: 0.0713\n",
"Epoch: 008/010 | Batch 200/469 | Cost: 0.1272\n",
"Epoch: 008/010 | Batch 250/469 | Cost: 0.0446\n",
"Epoch: 008/010 | Batch 300/469 | Cost: 0.0525\n",
"Epoch: 008/010 | Batch 350/469 | Cost: 0.1729\n",
"Epoch: 008/010 | Batch 400/469 | Cost: 0.0672\n",
"Epoch: 008/010 | Batch 450/469 | Cost: 0.0754\n",
"Epoch: 008/010 training accuracy: 96.67%\n",
"Time elapsed: 1.85 min\n",
"Epoch: 009/010 | Batch 000/469 | Cost: 0.0988\n",
"Epoch: 009/010 | Batch 050/469 | Cost: 0.0409\n",
"Epoch: 009/010 | Batch 100/469 | Cost: 0.1046\n",
"Epoch: 009/010 | Batch 150/469 | Cost: 0.0523\n",
"Epoch: 009/010 | Batch 200/469 | Cost: 0.0815\n",
"Epoch: 009/010 | Batch 250/469 | Cost: 0.0811\n",
"Epoch: 009/010 | Batch 300/469 | Cost: 0.0416\n",
"Epoch: 009/010 | Batch 350/469 | Cost: 0.0747\n",
"Epoch: 009/010 | Batch 400/469 | Cost: 0.0467\n",
"Epoch: 009/010 | Batch 450/469 | Cost: 0.0669\n",
"Epoch: 009/010 training accuracy: 97.90%\n",
"Time elapsed: 2.08 min\n",
"Epoch: 010/010 | Batch 000/469 | Cost: 0.0257\n",
"Epoch: 010/010 | Batch 050/469 | Cost: 0.0357\n",
"Epoch: 010/010 | Batch 100/469 | Cost: 0.1469\n",
"Epoch: 010/010 | Batch 150/469 | Cost: 0.0170\n",
"Epoch: 010/010 | Batch 200/469 | Cost: 0.0493\n",
"Epoch: 010/010 | Batch 250/469 | Cost: 0.0489\n",
"Epoch: 010/010 | Batch 300/469 | Cost: 0.1348\n",
"Epoch: 010/010 | Batch 350/469 | Cost: 0.0815\n",
"Epoch: 010/010 | Batch 400/469 | Cost: 0.0552\n",
"Epoch: 010/010 | Batch 450/469 | Cost: 0.0422\n",
"Epoch: 010/010 training accuracy: 97.99%\n",
"Time elapsed: 2.31 min\n",
"Total Training Time: 2.31 min\n"
]
}
],
"source": [
"def compute_accuracy(model, data_loader):\n",
" correct_pred, num_examples = 0, 0\n",
" for features, targets in data_loader:\n",
" features = features.to(device)\n",
" targets = targets.to(device)\n",
" logits, probas = model(features)\n",
" _, predicted_labels = torch.max(probas, 1)\n",
" num_examples += targets.size(0)\n",
" correct_pred += (predicted_labels == targets).sum()\n",
" return correct_pred.float()/num_examples * 100\n",
" \n",
"\n",
"start_time = time.time() \n",
"for epoch in range(num_epochs):\n",
" model = model.train()\n",
" for batch_idx, (features, targets) in enumerate(train_loader):\n",
" \n",
" features = features.to(device)\n",
" targets = targets.to(device)\n",
"\n",
" ### FORWARD AND BACK PROP\n",
" logits, probas = model(features)\n",
" cost = F.cross_entropy(logits, targets)\n",
" optimizer.zero_grad()\n",
" \n",
" cost.backward()\n",
" \n",
" ### UPDATE MODEL PARAMETERS\n",
" optimizer.step()\n",
" \n",
" ### LOGGING\n",
" if not batch_idx % 50:\n",
" print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' \n",
" %(epoch+1, num_epochs, batch_idx, \n",
" len(train_loader), cost))\n",
" \n",
" model = model.eval()\n",
" print('Epoch: %03d/%03d training accuracy: %.2f%%' % (\n",
" epoch+1, num_epochs, \n",
" compute_accuracy(model, train_loader)))\n",
"\n",
" print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
" \n",
"print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Evaluation"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Test accuracy: 97.97%\n"
]
}
],
"source": [
"with torch.set_grad_enabled(False): # save memory during inference\n",
" print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"torch 1.1.0\n",
"numpy 1.16.4\n",
"torchvision 0.3.0\n",
"\n"
]
}
],
"source": [
"%watermark -iv"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
},
"toc": {
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 4
}