{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n", "- Author: Sebastian Raschka\n", "- GitHub Repository: https://github.com/rasbt/deeplearning-models" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Sebastian Raschka \n", "\n", "CPython 3.6.8\n", "IPython 7.2.0\n", "\n", "torch 1.0.0\n" ] } ], "source": [ "%load_ext watermark\n", "%watermark -a 'Sebastian Raschka' -v -p torch" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- Runs on CPU or GPU (if available)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Model Zoo -- Softmax Regression" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Implementation of softmax regression (multinomial logistic regression)." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Imports" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "from torchvision import datasets\n", "from torchvision import transforms\n", "from torch.utils.data import DataLoader\n", "import torch.nn.functional as F\n", "import torch" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Settings and Dataset" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Image batch dimensions: torch.Size([256, 1, 28, 28])\n", "Image label dimensions: torch.Size([256])\n" ] } ], "source": [ "##########################\n", "### SETTINGS\n", "##########################\n", "\n", "# Device\n", "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", "\n", "# Hyperparameters\n", "random_seed = 123\n", "learning_rate = 0.1\n", "num_epochs = 10\n", "batch_size = 256\n", "\n", "# Architecture\n", "num_features = 784\n", "num_classes = 10\n", "\n", "\n", "##########################\n", "### MNIST DATASET\n", "##########################\n", "\n", "train_dataset = datasets.MNIST(root='data', \n", " train=True, \n", " transform=transforms.ToTensor(), \n", " download=True)\n", "\n", "test_dataset = datasets.MNIST(root='data', \n", " train=False, \n", " transform=transforms.ToTensor())\n", "\n", "\n", "train_loader = DataLoader(dataset=train_dataset, \n", " batch_size=batch_size, \n", " shuffle=True)\n", "\n", "test_loader = DataLoader(dataset=test_dataset, \n", " batch_size=batch_size, \n", " shuffle=False)\n", "\n", "\n", "# Checking the dataset\n", "for images, labels in train_loader: \n", " print('Image batch dimensions:', images.shape)\n", " print('Image label dimensions:', labels.shape)\n", " break" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "##########################\n", "### MODEL\n", "##########################\n", "\n", "class SoftmaxRegression(torch.nn.Module):\n", "\n", " def __init__(self, num_features, num_classes):\n", " super(SoftmaxRegression, self).__init__()\n", " self.linear = torch.nn.Linear(num_features, num_classes)\n", " \n", " self.linear.weight.detach().zero_()\n", " self.linear.bias.detach().zero_()\n", " \n", " def forward(self, x):\n", " logits = self.linear(x)\n", " probas = F.softmax(logits, dim=1)\n", " return logits, probas\n", "\n", "model = SoftmaxRegression(num_features=num_features,\n", " num_classes=num_classes)\n", "\n", "model.to(device)\n", "\n", "##########################\n", "### COST AND OPTIMIZER\n", "##########################\n", "\n", "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate) " ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch: 001/010 | Batch 000/234 | Cost: 2.3026\n", "Epoch: 001/010 | Batch 050/234 | Cost: 0.7941\n", "Epoch: 001/010 | Batch 100/234 | Cost: 0.5651\n", "Epoch: 001/010 | Batch 150/234 | Cost: 0.4603\n", "Epoch: 001/010 | Batch 200/234 | Cost: 0.4822\n", "Epoch: 001/010 training accuracy: 88.04%\n", "Epoch: 002/010 | Batch 000/234 | Cost: 0.4105\n", "Epoch: 002/010 | Batch 050/234 | Cost: 0.4415\n", "Epoch: 002/010 | Batch 100/234 | Cost: 0.4367\n", "Epoch: 002/010 | Batch 150/234 | Cost: 0.4289\n", "Epoch: 002/010 | Batch 200/234 | Cost: 0.3926\n", "Epoch: 002/010 training accuracy: 89.37%\n", "Epoch: 003/010 | Batch 000/234 | Cost: 0.4112\n", "Epoch: 003/010 | Batch 050/234 | Cost: 0.3579\n", "Epoch: 003/010 | Batch 100/234 | Cost: 0.3013\n", "Epoch: 003/010 | Batch 150/234 | Cost: 0.3258\n", "Epoch: 003/010 | Batch 200/234 | Cost: 0.4254\n", "Epoch: 003/010 training accuracy: 89.98%\n", "Epoch: 004/010 | Batch 000/234 | Cost: 0.3988\n", "Epoch: 004/010 | Batch 050/234 | Cost: 0.3690\n", "Epoch: 004/010 | Batch 100/234 | Cost: 0.3459\n", "Epoch: 004/010 | Batch 150/234 | Cost: 0.4030\n", "Epoch: 004/010 | Batch 200/234 | Cost: 0.3240\n", "Epoch: 004/010 training accuracy: 90.35%\n", "Epoch: 005/010 | Batch 000/234 | Cost: 0.3265\n", "Epoch: 005/010 | Batch 050/234 | Cost: 0.3673\n", "Epoch: 005/010 | Batch 100/234 | Cost: 0.3085\n", "Epoch: 005/010 | Batch 150/234 | Cost: 0.3183\n", "Epoch: 005/010 | Batch 200/234 | Cost: 0.3316\n", "Epoch: 005/010 training accuracy: 90.64%\n", "Epoch: 006/010 | Batch 000/234 | Cost: 0.4518\n", "Epoch: 006/010 | Batch 050/234 | Cost: 0.3863\n", "Epoch: 006/010 | Batch 100/234 | Cost: 0.3620\n", "Epoch: 006/010 | Batch 150/234 | Cost: 0.3733\n", "Epoch: 006/010 | Batch 200/234 | Cost: 0.3289\n", "Epoch: 006/010 training accuracy: 90.86%\n", "Epoch: 007/010 | Batch 000/234 | Cost: 0.3450\n", "Epoch: 007/010 | Batch 050/234 | Cost: 0.2289\n", "Epoch: 007/010 | Batch 100/234 | Cost: 0.3073\n", "Epoch: 007/010 | Batch 150/234 | Cost: 0.2750\n", "Epoch: 007/010 | Batch 200/234 | Cost: 0.3456\n", "Epoch: 007/010 training accuracy: 91.00%\n", "Epoch: 008/010 | Batch 000/234 | Cost: 0.4900\n", "Epoch: 008/010 | Batch 050/234 | Cost: 0.3479\n", "Epoch: 008/010 | Batch 100/234 | Cost: 0.2343\n", "Epoch: 008/010 | Batch 150/234 | Cost: 0.3059\n", "Epoch: 008/010 | Batch 200/234 | Cost: 0.3684\n", "Epoch: 008/010 training accuracy: 91.22%\n", "Epoch: 009/010 | Batch 000/234 | Cost: 0.3762\n", "Epoch: 009/010 | Batch 050/234 | Cost: 0.2976\n", "Epoch: 009/010 | Batch 100/234 | Cost: 0.2690\n", "Epoch: 009/010 | Batch 150/234 | Cost: 0.2610\n", "Epoch: 009/010 | Batch 200/234 | Cost: 0.3140\n", "Epoch: 009/010 training accuracy: 91.34%\n", "Epoch: 010/010 | Batch 000/234 | Cost: 0.2790\n", "Epoch: 010/010 | Batch 050/234 | Cost: 0.3070\n", "Epoch: 010/010 | Batch 100/234 | Cost: 0.3300\n", "Epoch: 010/010 | Batch 150/234 | Cost: 0.2520\n", "Epoch: 010/010 | Batch 200/234 | Cost: 0.3301\n", "Epoch: 010/010 training accuracy: 91.40%\n" ] } ], "source": [ "# Manual seed for deterministic data loader\n", "torch.manual_seed(random_seed)\n", "\n", "\n", "def compute_accuracy(model, data_loader):\n", " correct_pred, num_examples = 0, 0\n", " \n", " for features, targets in data_loader:\n", " features = features.view(-1, 28*28).to(device)\n", " targets = targets.to(device)\n", " logits, probas = model(features)\n", " _, predicted_labels = torch.max(probas, 1)\n", " num_examples += targets.size(0)\n", " correct_pred += (predicted_labels == targets).sum()\n", " \n", " return correct_pred.float() / num_examples * 100\n", " \n", "\n", "for epoch in range(num_epochs):\n", " for batch_idx, (features, targets) in enumerate(train_loader):\n", " \n", " features = features.view(-1, 28*28).to(device)\n", " targets = targets.to(device)\n", " \n", " ### FORWARD AND BACK PROP\n", " logits, probas = model(features)\n", " \n", " # note that the PyTorch implementation of\n", " # CrossEntropyLoss works with logits, not\n", " # probabilities\n", " cost = F.cross_entropy(logits, targets)\n", " optimizer.zero_grad()\n", " cost.backward()\n", " \n", " ### UPDATE MODEL PARAMETERS\n", " optimizer.step()\n", " \n", " ### LOGGING\n", " if not batch_idx % 50:\n", " print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' \n", " %(epoch+1, num_epochs, batch_idx, \n", " len(train_dataset)//batch_size, cost))\n", " \n", " with torch.set_grad_enabled(False):\n", " print('Epoch: %03d/%03d training accuracy: %.2f%%' % (\n", " epoch+1, num_epochs, \n", " compute_accuracy(model, train_loader)))" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Test accuracy: 91.77%\n" ] } ], "source": [ "print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "torch 1.0.0\n", "\n" ] } ], "source": [ "%watermark -iv" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.1" }, "toc": { "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 2 }