{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Pytorch Cheatsheet" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Imports" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": false }, "outputs": [], "source": [ "import torch\n", "import torch.nn as nn\n", "import torch.nn.init as init\n", "import torch.optim as optim\n", "import torch.nn.functional as F\n", "from torch.autograd import Variable\n", "import torchvision.datasets as datasets\n", "import torchvision.transforms as transforms\n", "import torchvision.utils as tv_utils\n", "from torch.utils.data import DataLoader\n", "import torchvision.models as models\n", "import torch.backends.cudnn as cudnn\n", "import torchvision\n", "import torch.autograd as autograd\n", "from PIL import Image\n", "import imp\n", "import os\n", "import sys\n", "import math\n", "import time\n", "import random\n", "import shutil\n", "import cv2\n", "import scipy.misc\n", "from glob import glob\n", "import sklearn\n", "import logging\n", "\n", "from tqdm import tqdm\n", "import numpy as np\n", "import matplotlib as mpl\n", "mpl.use('Agg')\n", "import matplotlib.pyplot as plt\n", "plt.style.use('bmh')\n", "\n", "%matplotlib inline" ] }, { "cell_type": "markdown", "metadata": { "collapsed": true, "heading_collapsed": true }, "source": [ "# Basics" ] }, { "cell_type": "markdown", "metadata": { "hidden": true }, "source": [ "* http://pytorch.org/tutorials/beginner/pytorch_with_examples.html\n", "* http://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Datasets" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "## File Management" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "random.seed(1)\n", "torch.manual_seed(1)\n", "DATA_PATH='/media/bfortuner/bigguy/data/'\n", "\n", "# Built-in\n", "MNIST_PATH=DATA_PATH+'mnist/'\n", "MNIST_WEIGHTS_PATH=MNIST_PATH+'weights/'\n", "\n", "# Built-in\n", "CIFAR10_PATH=DATA_PATH+'cifar10/'\n", "CIFAR10_IMGS_PATH=CIFAR10_PATH+'images/'\n", "\n", "# Download from http://www.vision.caltech.edu/visipedia/CUB-200.html\n", "CUB_PATH=DATA_PATH+'cub/'\n", "CUB_IMGS_PATH=CUB_PATH+'images/'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "hidden": true }, "outputs": [], "source": [ "def get_paths_to_files(dir_path):\n", " filepaths = []\n", " fnames = []\n", " for (dirpath, dirnames, filenames) in os.walk(dir_path):\n", " filepaths.extend(os.path.join(dirpath, f) for f in filenames if not f[0] == '.')\n", " fnames.extend([f for f in filenames if not f[0] == '.'])\n", " return filepaths, fnames\n", "\n", "def get_random_image_path(dir_path):\n", " filepaths = get_paths_to_files(dir_path)[0]\n", " return filepaths[random.randrange(len(filepaths))]" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "## MNIST" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "MNIST_BATCH_SIZE = 128\n", "MNIST_MEAN = np.array([0.1307,])\n", "MNIST_STD = np.array([0.3081,])\n", "mnist_train_loader = torch.utils.data.DataLoader(\n", " datasets.MNIST(MNIST_PATH, train=True, download=True,\n", " transform=transforms.Compose([\n", " transforms.ToTensor(),\n", " transforms.Normalize(MNIST_MEAN, MNIST_STD)\n", " ])),\n", " batch_size=MNIST_BATCH_SIZE, shuffle=True)\n", "mnist_test_loader = torch.utils.data.DataLoader(\n", " datasets.MNIST(MNIST_PATH, train=False, \n", " transform=transforms.Compose([\n", " transforms.ToTensor(),\n", " transforms.Normalize(MNIST_MEAN, MNIST_STD)\n", " ])),\n", " batch_size=MNIST_BATCH_SIZE*8, shuffle=True)\n", "\n", "def get_mnist_batch(batch_size):\n", " dataloader = torch.utils.data.DataLoader(\n", " datasets.MNIST(MNIST_PATH, train=False, download=False,\n", " transform=transforms.Compose([\n", " transforms.ToTensor(),\n", " transforms.Normalize(MNIST_MEAN, MNIST_STD)\n", " ])),\n", " batch_size=batch_size, shuffle=True)\n", " inputs, targets = next(iter(dataloader))\n", " return inputs, targets\n", "\n", "mnist_train_labels = mnist_train_loader.dataset.train_labels\n", "MNIST_CLASSES = np.unique(mnist_train_labels.numpy())\n", "mnist_batch = get_mnist_batch(100)\n", "print(\"MNIST Train Samples:\", len(mnist_train_loader.dataset))\n", "print(\"MNIST Test Samples:\", len(mnist_test_loader.dataset))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## CIFAR10" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "CIFAR_BATCH_SIZE = 64\n", "CIFAR_MEAN = np.array([0.49139968, 0.48215827, 0.44653124])\n", "CIFAR_STD = np.array([0.24703233, 0.24348505, 0.26158768])\n", "normTransform = transforms.Normalize(CIFAR_MEAN, CIFAR_STD)\n", "\n", "trainTransform = transforms.Compose([\n", " transforms.RandomCrop(32, padding=4),\n", " transforms.RandomHorizontalFlip(),\n", " transforms.ToTensor(),\n", " normTransform\n", "])\n", "testTransform = transforms.Compose([\n", " transforms.ToTensor(),\n", " normTransform\n", "])\n", "\n", "cifar_train_loader = torch.utils.data.DataLoader(\n", " datasets.CIFAR10(CIFAR10_IMGS_PATH, train=True, download=True,\n", " transform=trainTransform),\n", " batch_size=CIFAR_BATCH_SIZE, shuffle=True)\n", "cifar_test_loader = torch.utils.data.DataLoader(\n", " datasets.CIFAR10(CIFAR10_IMGS_PATH, train=False, download=True,\n", " transform=testTransform),\n", " batch_size=CIFAR_BATCH_SIZE*16, shuffle=False)\n", "\n", "def get_cifar_batch(batch_size):\n", " dataloader = torch.utils.data.DataLoader(\n", " datasets.CIFAR10(CIFAR10_IMGS_PATH, train=False, \n", " transform=testTransform), batch_size=batch_size, shuffle=True)\n", " inputs, targets = next(iter(dataloader))\n", " return inputs, targets\n", "\n", "CIFAR_CLASSES = np.unique(np.array(cifar_train_loader.dataset.train_labels))\n", "CIFAR_CLASS_NAMES = np.array(['airplane','automobile','bird','cat',\n", " 'deer','dog','frog', 'horse','ship','truck'])\n", "cifar_batch = get_cifar_batch(100)\n", "print(\"CIFAR Train Samples:\", len(cifar_train_loader.dataset))\n", "print(\"CIFAR Test Samples:\", len(cifar_test_loader.dataset))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Image Folder" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "def get_mean_std_of_dataset(dir_path, sample_size=5):\n", " fpaths, fnames = get_paths_to_files(dir_path)\n", " random.shuffle(fpaths)\n", " total_mean = np.array([0.,0.,0.])\n", " total_std = np.array([0.,0.,0.]) \n", " for f in fpaths[:sample_size]:\n", " img_arr = load_img_as_np_arr(f)\n", " mean = np.mean(img_arr, axis=(0,1))\n", " std = np.std(img_arr, axis=(0,1))\n", " total_mean += mean\n", " total_std += std\n", " avg_mean = total_mean / sample_size\n", " avg_std = total_std / sample_size\n", " print(\"mean: {}\".format(avg_mean), \"stdev: {}\".format(avg_std))\n", " return avg_mean, avg_std\n", "\n", "CUB_BATCH_SIZE = 32\n", "CUB_MEAN, CUB_STD = get_mean_std_of_dataset(CUB_IMAGES_PATH, 1000)\n", "\n", "cub_dataset = datasets.ImageFolder(root=CUB_IMAGES_PATH, \n", " transform=transforms.Compose([\n", " transforms.Scale(256),\n", " transforms.CenterCrop(256),\n", " transforms.ToTensor(),\n", " transforms.Normalize(CUB_MEAN, CUB_STD),\n", " ]))\n", "\n", "def get_cub_batch(batch_size):\n", " dataloader = torch.utils.data.DataLoader(\n", " cub_dataset, batch_size=batch_size, shuffle=True)\n", " inputs, targets = next(iter(dataloader))\n", " return inputs, targets\n", "\n", "cub_data_loader = torch.utils.data.DataLoader(\n", " cub_dataset, batch_size=CUB_BATCH_SIZE, shuffle=True)\n", "cub_class_names = cub_dataset.classes\n", "cub_batch = get_cub_batch(10)" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "## Swiss Roll" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "from sklearn.datasets.samples_generator import make_swiss_roll\n", "\n", "def get_swiss_roll(n_samples=100):\n", " noise = 0.2\n", " X, _ = make_swiss_roll(n_samples, noise)\n", " X = X.astype('float32')[:, [0, 2]]\n", " return X, _\n", "\n", "def plot_roll(data):\n", " # data.shape = (N, 2)\n", " if type(data) != np.ndarray:\n", " data = data.numpy()\n", " x = data[:,0]\n", " y = data[:,1]\n", " plt.scatter(x,y)\n", "\n", "SWISS_ROLL_BATCH_SIZE = 100\n", "X, _ = get_swiss_roll(100)\n", "swiss_roll_dataset = torch.utils.data.TensorDataset(torch.FloatTensor(X), torch.FloatTensor(_))\n", "swiss_roll_loader = torch.utils.data.DataLoader(swiss_roll_dataset, batch_size=SWISS_ROLL_BATCH_SIZE, shuffle=True)\n", " \n", "# Test\n", "data = get_swiss_roll(1000)[0]\n", "plot_roll(data)\n", "inputs,targets = next(iter(swiss_roll_loader))\n", "print(inputs.size(),targets.size())\n", "plot_roll(inputs)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Image Handling" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "## Normalization" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "def norm_meanstd(arr, mean, std):\n", " return (arr - mean) / std\n", "\n", "def denorm_meanstd(arr, mean, std):\n", " return (arr * std) + mean\n", "\n", "def norm255_tensor(arr):\n", " \"\"\"Given a color image/where max pixel value in each channel is 255\n", " returns normalized tensor or array with all values between 0 and 1\"\"\"\n", " return arr / 255.\n", " \n", "def denorm255_tensor(arr):\n", " return arr * 255." ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "## Loading" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "def load_img_as_pil(img_path):\n", " return Image.open(img_path)\n", "\n", "def load_img_as_np_arr(img_path):\n", " return scipy.misc.imread(img_path) #scipy\n", "\n", "def load_img_as_tensor(img_path):\n", " pil_image = Image.open(img_path)\n", " return transforms.ToTensor()(pil_image)" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "## Saving" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "hidden": true }, "outputs": [], "source": [ "def save_tensor_img(tns, fpath):\n", " tv_utils.save_image(tns, fpath)\n", " \n", "def save_pil_img(pil_img, fpath):\n", " pil_img.save(fpath)\n", " \n", "def save_numpy_img(np_arr, fpath):\n", " scipy.misc.imsave(fpath, np_arr)" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "## Plotting" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "def plot_np_array(arr_img, fs=(3,3)):\n", " plt.figure(figsize=fs)\n", " plt.imshow(arr_img.astype('uint8'))\n", " plt.show()\n", "\n", "def plot_tensor(tns_img, fs=(3,3)):\n", " \"Takes a normalized tensor [0,1] and plots PIL image\"\n", " pil_from_tns = transforms.ToPILImage()(tns_img)\n", " plt.figure(figsize=fs)\n", " plt.imshow(pil_from_tns)\n", " plt.show()\n", "\n", "def plot_pil(pil_img, fs=(3,3)):\n", " plt.figure(figsize=fs)\n", " plt.imshow(pil_img)\n", " plt.show()\n", "\n", "def imshow(inp, mean_arr, std_arr, title=None):\n", " # Input is normalized Tensor or Numpy Arr\n", " if inp.size(0) == 1:\n", " inp = np.squeeze(inp.numpy())\n", " kwargs = {'cmap':'gray'}\n", " else:\n", " inp = inp.numpy().transpose((1, 2, 0))\n", " kwargs = {}\n", " inp = std_arr * inp + mean_arr\n", " plt.imshow(inp, **kwargs)\n", " if title is not None:\n", " plt.title(title)\n", " plt.pause(0.001)\n", "\n", "def visualize_preds(model, data_loader, class_names, mean_arr, std_arr, num_images=6):\n", " images_so_far = 0\n", " fig = plt.figure()\n", "\n", " for i, data in enumerate(data_loader):\n", " inputs, labels = data\n", " inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())\n", "\n", " outputs = model(inputs)\n", " _, preds = torch.max(outputs.data, 1)\n", " preds = preds.cpu().numpy()\n", " labels = labels.data.cpu().numpy()\n", " for j in range(inputs.size()[0]):\n", " images_so_far += 1\n", " ax = plt.subplot(num_images//2, 2, images_so_far)\n", " ax.axis('off')\n", " ax.set_title('P: {}, A:{}'.format(class_names[preds[j][0]], \n", " class_names[labels[j]]))\n", " imshow(inputs.cpu().data[j], mean_arr, std_arr)\n", "\n", " if images_so_far == num_images:\n", " return\n", " plt.tight_layout()\n", "\n", "def plot_bw_samples(arr, dim=(4,4), figsize=(6,6)):\n", " if type(arr) is not np.ndarray:\n", " arr = arr.numpy()\n", " bs = arr.shape[0]\n", " arr = arr.reshape(bs, 28, 28)\n", " plt.figure(figsize=figsize)\n", " for i,img in enumerate(arr):\n", " plt.subplot(*dim, i+1)\n", " plt.imshow(img, cmap='gray')\n", " plt.axis('off')\n", " plt.tight_layout()\n", "\n", "def plot_samples(arr, mean, std, dim=(4,4), figsize=(6,6)):\n", " if type(arr) is not np.ndarray:\n", " arr = arr.numpy().transpose((0, 2, 3, 1))\n", " arr = denorm_meanstd(arr, mean, std)\n", " plt.figure(figsize=figsize)\n", " for i,img in enumerate(arr):\n", " plt.subplot(*dim, i+1)\n", " plt.imshow(img)\n", " plt.axis('off')\n", " plt.tight_layout()\n", "\n", "# Test\n", "img_path = get_random_image_path(CUB_IMGS_PATH)\n", "plot_pil(load_img_as_pil(img_path))\n", "plot_np_array(load_img_as_np_arr(img_path))\n", "plot_tensor(load_img_as_tensor(img_path))\n", "inps,targs = get_mnist_batch(12)\n", "plot_bw_samples(inps)\n", "inps,targs = get_cifar_batch(12)\n", "plot_samples(inps, CIFAR_MEAN, CIFAR_STD)\n", "inps,targs = get_cub_batch(12)\n", "plot_samples(inps, CUB_MEAN, CUB_STD)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Models" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Logging" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "#https://docs.python.org/3/howto/logging-cookbook.html\n", "\n", "def get_logger(ch_log_level=logging.ERROR, \n", " fh_log_level=logging.INFO):\n", " logging.shutdown()\n", " imp.reload(logging)\n", " logger = logging.getLogger(\"cheatsheet\")\n", " logger.setLevel(logging.DEBUG)\n", " \n", " # Console Handler\n", " if ch_log_level:\n", " ch = logging.StreamHandler()\n", " ch.setLevel(ch_log_level)\n", " ch.setFormatter(logging.Formatter('%(message)s'))\n", " logger.addHandler(ch)\n", " \n", " # File Handler\n", " if fh_log_level:\n", " fh = logging.FileHandler('cheatsheet.log')\n", " fh.setLevel(fh_log_level)\n", " formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')\n", " fh.setFormatter(formatter)\n", " logger.addHandler(fh)\n", "\n", " return logger\n", "\n", "#Test\n", "logger = get_logger() #Singleton\n", "logger.info(\"LOG TO FILE\")\n", "logger.error(\"LOG TO FILE and CONSOLE\")\n", "\n", "logger = get_logger(ch_log_level=logging.DEBUG, \n", " fh_log_level=None)\n", "logger.debug(\"Init Console Logger Only\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Linear" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "class LinearNetMNIST(nn.Module):\n", " def __init__(self):\n", " super(LinearNetMNIST, self).__init__()\n", " # Input size (bs, 784)\n", " self.layer1 = nn.Linear(784, 500)\n", " self.layer2 = nn.Linear(500, 250)\n", " self.layer3 = nn.Linear(250, 10)\n", " self.softmax = nn.Softmax()\n", " \n", " def forward(self, x):\n", " x = x.view(x.size(0), -1)\n", " x = self.layer1(x)\n", " x = self.layer2(x)\n", " x = self.layer3(x)\n", " x = self.softmax(x)\n", " return x\n", " \n", "class LinearNetCIFAR(nn.Module):\n", " def __init__(self):\n", " super(LinearNetCIFAR, self).__init__()\n", " # Input (bs, 3, 32, 32)\n", " # Flatten to fit linear layer (bs,32*32*3)\n", " self.layer1 = nn.Linear(3072, 512)\n", " self.relu1 = nn.ReLU()\n", " self.layer2 = nn.Linear(512, 256)\n", " self.relu2 = nn.ReLU()\n", " self.layer3 = nn.Linear(256, 10)\n", " self.softmax = nn.Softmax()\n", " \n", " def forward(self, x):\n", " x = x.view(x.size(0), -1) # Flatten\n", " x = self.layer1(x)\n", " x = self.relu1(x)\n", " x = self.layer2(x)\n", " x = self.relu2(x)\n", " x = self.layer3(x)\n", " x = self.softmax(x)\n", " return x" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "inputs, targets = get_mnist_batch(2)\n", "net = LinearNetMNIST()\n", "print(net(Variable(inputs)))\n", "\n", "inputs, targets = get_cifar_batch(2)\n", "net2 = LinearNetCIFAR()\n", "print(net2(Variable(inputs)))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## CNN" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Tips and Guidelines\n", "http://cs231n.github.io/convolutional-networks/\n", "\n", "* The input layer (that contains the image) should be divisible by 2 many times\n", "* The conv layers should be using small filters (e.g. 3x3 or at most 5x5), using a stride of S=1, and crucially, padding the input volume with zeros in such way that the conv layer does not alter the spatial dimensions of the input. That is, when FS=3, then using P=1 will retain the original size of the input. When FS=5, P=2. For a general FS, it can be seen that P = (FS − 1)/2 preserves the input size. \n", "* The pool layers are in charge of downsampling the spatial dimensions of the input. Introduces zero parameters since it simply computes a fixed function of the input. The most common setting is to use max-pooling with 2x2 receptive fields (i.e. FS=2), and with a stride of 2 (i.e. S=2). Output dimensions equal W2 = (W1 − FS)/S + 1." ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "### Basic CNN" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "hidden": true }, "outputs": [], "source": [ "class BasicCNN(nn.Module):\n", " def __init__(self, logger=None):\n", " super(BasicCNN, self).__init__()\n", " self.logger = logger\n", " self.conv1 = nn.Conv2d(in_channels=3, out_channels=50, \n", " kernel_size=2, stride=1, padding=0)\n", " self.conv2 = nn.Conv2d(in_channels=50, out_channels=100, \n", " kernel_size=2, stride=1, padding=0)\n", " self.linear = nn.Linear(in_features=100*30*30, out_features=10)\n", " self.softmax = nn.Softmax()\n", " \n", " def log(self, msg):\n", " if self.logger:\n", " self.logger.debug(msg)\n", "\n", " def forward(self, x):\n", " self.log(x.size()) # (bs,3,32,32)\n", " x = self.conv1(x)\n", " x = self.conv2(x)\n", " x = x.view(x.size(0), -1) # (bs,100*30*30)\n", " x = self.linear(x)\n", " x = self.softmax(x)\n", " return x\n", " \n", "# Test\n", "logger = get_logger(logging.DEBUG, logging.DEBUG)\n", "net = BasicCNN(logger)\n", "inputs,targets = next(iter(cifar_train_loader))\n", "inputs = Variable(inputs[:2])\n", "net(inputs)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Deeper CNN" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "class DeeperCNN(nn.Module):\n", " def __init__(self, logger=None):\n", " super(DeeperCNN, self).__init__()\n", " self.logger = logger \n", " # Conv Dims - W2 = (W1-FS+2P)/S + 1)\n", " self.conv1 = nn.Conv2d(in_channels=3, out_channels=64, \n", " kernel_size=3, stride=1, padding=1)\n", " self.bn1 = nn.BatchNorm2d(num_features=64, momentum=0.9)\n", " self.relu1 = nn.ReLU()\n", " self.conv2 = nn.Conv2d(in_channels=64, out_channels=128, \n", " kernel_size=3, stride=1, padding=1)\n", " self.bn2 = nn.BatchNorm2d(num_features=128, momentum=0.9)\n", " self.relu2 = nn.ReLU()\n", " # Pool Dims - W2 = (W1 − FS)/S + 1\n", " self.pool = nn.MaxPool2d(kernel_size=2, stride=2) #shrinks by half\n", " self.linear1 = nn.Linear(in_features=128*16*16, out_features=512)\n", " self.dropout = nn.Dropout(p=0.5)\n", " self.linear2 = nn.Linear(in_features=512, out_features=10)\n", " self.softmax = nn.Softmax()\n", " \n", " def log(self, msg):\n", " if self.logger:\n", " self.logger.debug(msg)\n", "\n", " def forward(self, x):\n", " self.log(x.size())\n", " x = self.relu1(self.bn1(self.conv1(x)))\n", " x = self.relu2(self.bn2(self.conv2(x)))\n", " x = self.pool(x)\n", " x = x.view(x.size(0), -1)\n", " x = self.linear1(x)\n", " x = self.dropout(x)\n", " x = self.linear2(x) \n", " x = self.softmax(x)\n", " return x\n", " \n", "# Test\n", "net = DeeperCNN(logger)\n", "inputs,targets = next(iter(cifar_train_loader))\n", "inputs = Variable(inputs[:2])\n", "net(inputs)" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "## RNN" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "hidden": true }, "outputs": [], "source": [ "# TODO" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "## GAN" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "hidden": true }, "outputs": [], "source": [ "# TODO" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "## Finetuning" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true, "hidden": true }, "source": [ "### Pretrained Model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "# Get DenseNet trained on Imagenet\n", "densenet = models.densenet121(pretrained=True)\n", " \n", "# Replace final layer with new output layer\n", "last_layer_n_features = densenet.classifier.in_features\n", "densenet.classifier = nn.Linear(last_layer_n_features, len(CIFAR_CLASSES))\n", "print(densenet.classifier)" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true, "hidden": true }, "source": [ "### Freeze Layers" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "hidden": true }, "outputs": [], "source": [ "# Freeze layers to avoid retraining\n", "# To create a \"Feature Extractor\"\n", "for param in densenet.parameters():\n", " param.requires_grad = False" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true, "hidden": true }, "source": [ "### Reuse Specific Layers" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "vgg = models.vgg16(pretrained=True)\n", "\n", "class MyNewModel(nn.Module):\n", " def __init__(self):\n", " super(MyNewModel, self).__init__()\n", " self.features = nn.Sequential(\n", " # stop at 10th layer\n", " *list(vgg.features.children())[:10])\n", "\n", " def forward(self, x):\n", " x = self.features(x)\n", " return x\n", "\n", "print(MyNewModel())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Training" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Methods" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def train(net, dataloader, criterion, optimizer, epoch=1):\n", " net.train()\n", " n_batches = len(dataloader)\n", " total_loss = 0\n", " total_acc = 0\n", " for inputs,targets in dataloader:\n", " inputs = Variable(inputs.cuda())\n", " targets = Variable(targets.cuda())\n", " \n", " ## Forward Pass\n", " output = net(inputs)\n", " \n", " ## Clear Gradients\n", " net.zero_grad()\n", " \n", " loss = criterion(output, targets)\n", " \n", " ## Backprop\n", " loss.backward()\n", " optimizer.step()\n", " \n", " preds = get_predictions(output)\n", " accuracy = get_accuracy(preds, targets.data.cpu().numpy())\n", " \n", " total_loss += loss.data[0]\n", " total_acc += accuracy\n", " \n", " mean_loss = total_loss / n_batches\n", " mean_acc = total_acc / n_batches\n", " return mean_loss, mean_acc\n", "\n", "def get_predictions(model_output):\n", " # Flatten and Get ArgMax to compute accuracy\n", " val,idx = torch.max(model_output, dim=1)\n", " return idx.data.cpu().view(-1).numpy()\n", "\n", "def get_accuracy(preds, targets):\n", " correct = np.sum(preds==targets)\n", " return correct / len(targets)\n", "\n", "def test(net, test_loader, criterion, epoch=1):\n", " net.eval()\n", " test_loss = 0\n", " test_acc = 0\n", " for data, target in test_loader:\n", " data = Variable(data.cuda(), volatile=True)\n", " target = Variable(target.cuda())\n", " output = net(data)\n", " test_loss += criterion(output, target).data[0]\n", " pred = get_predictions(output)\n", " test_acc += get_accuracy(pred, target.data.cpu().numpy())\n", " test_loss /= len(test_loader) #n_batches\n", " test_acc /= len(test_loader)\n", " return test_loss, test_acc\n", "\n", "def adjust_learning_rate(lr, decay, optimizer, cur_epoch, n_epochs):\n", " \"\"\"Sets the learning rate to the initially \n", " configured `lr` decayed by `decay` every `n_epochs`\"\"\"\n", " new_lr = lr * (decay ** (cur_epoch // n_epochs))\n", " for param_group in optimizer.param_groups:\n", " param_group['lr'] = new_lr\n", "\n", "def weights_init(m):\n", " if isinstance(m, nn.Conv2d):\n", " init.kaiming_uniform(m.weight) \n", " m.bias.data.zero_()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## MNIST" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "# Linear Model\n", "mnistnet = LinearNetMNIST().cuda()\n", "\n", "N_EPOCHS = 5\n", "criterion = nn.CrossEntropyLoss().cuda()\n", "optimizer = optim.Adam(mnistnet.parameters(), lr=1e-4)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "train_loss_history = []\n", "train_acc_history = []\n", "test_loss_history = []\n", "test_acc_history = []\n", "for epoch in range(1, N_EPOCHS+1):\n", " trn_loss, trn_acc = train(mnistnet, mnist_train_loader, criterion, optimizer, epoch)\n", " test_loss, test_acc = test(mnistnet, mnist_test_loader, criterion, epoch)\n", " print('Epoch %d, TrainLoss: %.3f, TrainAcc: %.3f, TestLoss: %.3f, TestAcc: %.3f' % (\n", " epoch, trn_loss, trn_acc, test_loss, test_acc))\n", " train_loss_history.append(trn_loss)\n", " test_loss_history.append(test_loss)\n", " train_acc_history.append(trn_acc)\n", " test_acc_history.append(test_acc)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "plt.plot(np.stack([train_loss_history, test_loss_history],1))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "plt.plot(np.stack([train_acc_history, test_acc_history], 1))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "visualize_preds(mnistnet, mnist_test_loader, MNIST_CLASSES, MNIST_MEAN, MNIST_STD, 6)" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "## CIFAR" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "# Linear Model\n", "cifarnet = LinearNetCIFAR().cuda()\n", "\n", "# Basic CNN\n", "cifarnet = BasicCNN().cuda()\n", "\n", "# Deeper CNN\n", "cifarnet = DeeperCNN().cuda()\n", "\n", "N_EPOCHS = 5\n", "criterion = nn.CrossEntropyLoss().cuda()\n", "optimizer = optim.Adam(cifarnet.parameters(), lr=1e-4)\n", "\n", "print(' + Number of params: {}'.format(\n", " sum([p.data.nelement() for p in cifarnet.parameters()])))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "train_loss_history = []\n", "train_acc_history = []\n", "test_loss_history = []\n", "test_acc_history = []\n", "for epoch in range(1, N_EPOCHS+1):\n", " trn_loss, trn_acc = train(cifarnet, cifar_train_loader, criterion, optimizer, epoch)\n", " test_loss, test_acc = test(cifarnet, cifar_test_loader, criterion, epoch)\n", " print('Epoch %d, TrainLoss: %.3f, TrainAcc: %.3f, TestLoss: %.3f, TestAcc: %.3f' % (\n", " epoch, trn_loss, trn_acc, test_loss, test_acc))\n", " train_loss_history.append(trn_loss)\n", " test_loss_history.append(test_loss)\n", " train_acc_history.append(trn_acc)\n", " test_acc_history.append(test_acc)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "plt.plot(np.stack([train_loss_history, test_loss_history],1))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "plt.plot(np.stack([train_acc_history, test_acc_history], 1))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "visualize_preds(cifarnet, cifar_test_loader, CIFAR_CLASS_NAMES, CIFAR_MEAN, CIFAR_STD, 6)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Experiments" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "## Load/Save Weights" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "def save_weights(model, weights_dir, epoch):\n", " weights_fname = 'weights-%d.pth' % (epoch)\n", " weights_fpath = os.path.join(weights_dir, weights_fname)\n", " torch.save({'state_dict': model.state_dict()}, weights_fpath)\n", "\n", "def load_weights(model, fpath):\n", " state = torch.load(fpath)\n", " model.load_state_dict(state['state_dict'])\n", "\n", "# Test\n", "model = LinearNetCIFAR()\n", "save_weights(model, MNIST_WEIGHTS_PATH, 1)\n", "load_weights(model, MNIST_WEIGHTS_PATH+'weights-1.pth')" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "## Visdom Web Server" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "hidden": true }, "outputs": [], "source": [ "# Real-time visualizations in your browser\n", "# https://github.com/facebookresearch/visdom\n", "\n", "import visdom\n", "viz = visdom.Visdom()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "hidden": true }, "outputs": [], "source": [ "def viz_plot_tst_trn(window, epoch, tst_val, trn_val, name='loss', env='main'):\n", " if window is None:\n", " return viz.line(\n", " X=np.array([epoch]),\n", " Y=np.array([[tst_val, trn_val]]),\n", " opts=dict(\n", " xlabel='epoch',\n", " ylabel=name,\n", " title=env+' '+name,\n", " legend=['Validation', 'Train']\n", " ),\n", " env=env\n", " )\n", " return viz.line(\n", " X=np.ones((1, 2)) * epoch,\n", " Y=np.expand_dims([tst_val, trn_val],0),\n", " win=window,\n", " update='append',\n", " env=env\n", " )\n", "\n", "def viz_plot_img(window, arr, mean, std, env='main', title='Image'):\n", " '''\n", " This function draws an img on your Visdom web app. \n", " It takes as input an `CxHxW` tensor `img`\n", " The array values can be float in [0,1] or uint8 in [0, 255]'''\n", " if type(arr) is not np.ndarray:\n", " arr = arr.numpy().transpose((1, 2, 0))\n", " arr = denorm_meanstd(arr, mean, std)\n", " arr = arr.transpose((2, 0, 1))\n", " viz.image(\n", " arr,\n", " opts=dict(title=title, caption='Silly image'),\n", " win=window,\n", " env=env\n", " )\n", " \n", "def viz_plot_text(window, text, env='main'):\n", " if window is None:\n", " return viz.text(\n", " text,\n", " env=env\n", " )\n", " return viz.text(\n", " text,\n", " win=window,\n", " env=env\n", " )\n", "\n", "def viz_plot_summary(window, epoch, tst_loss, trn_loss,\n", " tst_err, trn_err, env='main'):\n", " txt = (\"\"\"Epoch: %d\n", " Train - Loss: %.3f Err: %.3f\n", " Test - Loss: %.3f Err: %.3f\"\"\" % (epoch, \n", " trn_loss, trn_err, tst_loss, tst_err))\n", " return viz_plot_text(window, txt, env)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "hidden": true }, "outputs": [], "source": [ "#Visit http://localhost:8097 to view plots\n", "\n", "#Should plot one chart and update it\n", "txt_chart = viz_plot_summary(None, 1, 2, 3, 4, 5)\n", "txt_chart = viz_plot_summary(txt_chart, 5, 2, 3, 4, 5)\n", "txt_chart = viz_plot_summary(txt_chart, 5, 3, 8, 7, 6)\n", "\n", "#Should plot one chart and update it\n", "sum_chart = viz_plot_text(None, 'Hello, world3!')\n", "sum_chart = viz_plot_text(sum_chart, 'Hello, world4!')\n", "\n", "#Should plot one chart and update it\n", "#window, epoch, tst_val, trn_val, name='loss', env='main'\n", "loss_chart = viz_plot_tst_trn(None, 9, 14, 27, 'loss')\n", "loss_chart = viz_plot_tst_trn(loss_chart, 10, 18, 30, 'loss')\n", "loss_chart = viz_plot_tst_trn(loss_chart, 11, 19, 32, 'loss')\n", "\n", "#Should plot one chart and update it\n", "#window, epoch, tst_val, trn_val, name='loss', env='main'\n", "err_chart = viz_plot_tst_trn(None, 9, 14, 27, 'error')\n", "err_chart = viz_plot_tst_trn(err_chart, 10, 18, 30, 'error')\n", "err_chart = viz_plot_tst_trn(err_chart, 11, 19, 32, 'error')\n", "\n", "# Plot Image\n", "inputs, targets = next(iter(cifar_train_loader))\n", "img_chart = viz.image(\n", " np.random.rand(3,100,100),\n", " opts=dict(title=\"Image\", caption='Silly random'),\n", ")\n", "viz_plot_img(img_chart, inputs[0], CIFAR_MEAN, CIFAR_STD)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Experiment Class" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "import numpy as np\n", "import os\n", "import torch\n", "import visdom\n", "import shutil\n", "import sys\n", "from pathlib import Path\n", "\n", "\n", "class Experiment():\n", " def __init__(self, name, root, logger=None):\n", " self.name = name\n", " self.root = os.path.join(root,name)\n", " self.logger = logger\n", " self.epoch = 1\n", " self.best_val_loss = sys.maxsize\n", " self.best_val_loss_epoch = 1\n", " self.weights_dir = os.path.join(self.root, 'weights')\n", " self.history_dir = os.path.join(self.root, 'history')\n", " self.results_dir = os.path.join(self.root, 'results')\n", " self.latest_weights = os.path.join(self.weights_dir, 'latest_weights.pth')\n", " self.latest_optimizer = os.path.join(self.weights_dir, 'latest_optim.pth')\n", " self.best_weights_path = self.latest_weights\n", " self.best_optimizer_path = self.latest_optimizer\n", " self.train_history_fpath = os.path.join(self.history_dir, 'train.csv')\n", " self.val_history_fpath = os.path.join(self.history_dir, 'val.csv')\n", " self.test_history_fpath = os.path.join(self.history_dir, 'test.csv')\n", " self.loss_history = {\n", " 'train': np.array([]),\n", " 'val': np.array([]),\n", " 'test': np.array([])\n", " }\n", " self.acc_history = {\n", " 'train': np.array([]),\n", " 'val': np.array([]),\n", " 'test': np.array([])\n", " }\n", " self.viz = visdom.Visdom()\n", " self.visdom_plots = self.init_visdom_plots()\n", "\n", " def log(self, msg):\n", " if self.logger:\n", " logger.info(msg)\n", " \n", " def init(self):\n", " self.log(\"Creating new experiment\")\n", " self.init_dirs()\n", " self.init_history_files()\n", "\n", " def resume(self, model, optim, weights_fpath=None, optim_path=None):\n", " self.log(\"Resuming existing experiment\")\n", " if weights_fpath is None:\n", " weights_fpath = self.latest_weights\n", " if optim_path is None:\n", " optim_path = self.latest_optimizer\n", "\n", " model, state = self.load_weights(model, weights_fpath)\n", " optim = self.load_optimizer(optim, optim_path)\n", "\n", " self.best_val_loss = state['best_val_loss']\n", " self.best_val_loss_epoch = state['best_val_loss_epoch']\n", " self.epoch = state['last_epoch']+1\n", " self.load_history_from_file('train')\n", " self.load_history_from_file('val')\n", "\n", " return model, optim\n", "\n", " def init_dirs(self):\n", " os.makedirs(self.weights_dir)\n", " os.makedirs(self.history_dir)\n", " os.makedirs(self.results_dir)\n", "\n", " def init_history_files(self):\n", " Path(self.train_history_fpath).touch()\n", " Path(self.val_history_fpath).touch()\n", " Path(self.test_history_fpath).touch()\n", "\n", " def init_visdom_plots(self):\n", " loss = self.init_viz_train_plot('loss')\n", " accuracy = self.init_viz_train_plot('accuracy')\n", " summary = self.init_viz_txt_plot('summary')\n", " return {\n", " 'loss':loss,\n", " 'accuracy':accuracy,\n", " 'summary':summary\n", " }\n", "\n", " def init_viz_train_plot(self, title):\n", " return self.viz.line(\n", " X=np.array([1]),\n", " Y=np.array([[1, 1]]),\n", " opts=dict(\n", " xlabel='epoch',\n", " ylabel=title,\n", " title=self.name+' '+title,\n", " legend=['Train', 'Validation']\n", " ),\n", " env=self.name\n", " )\n", "\n", " def init_viz_txt_plot(self, title):\n", " return self.viz.text(\n", " \"Initializing.. \" + title,\n", " env=self.name\n", " )\n", "\n", " def viz_epochs(self):\n", " epochs = np.arange(1,self.epoch+1)\n", " return np.stack([epochs, epochs],1)\n", "\n", " def update_viz_loss_plot(self):\n", " loss = np.stack([self.loss_history['train'],\n", " self.loss_history['val']],1)\n", " window = self.visdom_plots['loss']\n", " return self.viz.line(\n", " X=self.viz_epochs(),\n", " Y=loss,\n", " win=window,\n", " env=self.name,\n", " opts=dict(\n", " xlabel='epoch',\n", " ylabel='loss',\n", " title=self.name+' '+'loss',\n", " legend=['Train', 'Validation']\n", " ),\n", " )\n", "\n", " def update_viz_acc_plot(self):\n", " acc = np.stack([self.acc_history['train'],\n", " self.acc_history['val']], 1)\n", " window = self.visdom_plots['accuracy']\n", " return self.viz.line(\n", " X=self.viz_epochs(),\n", " Y=acc,\n", " win=window,\n", " env=self.name,\n", " opts=dict(\n", " xlabel='epoch',\n", " ylabel='accuracy',\n", " title=self.name+' '+'accuracy',\n", " legend=['Train', 'Validation']\n", " )\n", " )\n", "\n", " def update_viz_summary_plot(self):\n", " trn_loss = self.loss_history['train'][-1]\n", " val_loss = self.loss_history['val'][-1]\n", " trn_acc = self.acc_history['train'][-1]\n", " val_acc = self.acc_history['val'][-1]\n", " txt = (\"\"\"Epoch: %d\n", " Train - Loss: %.3f Acc: %.3f\n", " Test - Loss: %.3f Acc: %.3f\"\"\" % (self.epoch,\n", " trn_loss, trn_acc, val_loss, val_acc))\n", " window = self.visdom_plots['summary']\n", " return self.viz.text(\n", " txt,\n", " win=window,\n", " env=self.name\n", " )\n", "\n", " def load_history_from_file(self, dset_type):\n", " fpath = os.path.join(self.history_dir, dset_type+'.csv')\n", " data = np.loadtxt(fpath, delimiter=',').reshape(-1, 3)\n", " self.loss_history[dset_type] = data[:,1]\n", " self.acc_history[dset_type] = data[:,2]\n", "\n", " def append_history_to_file(self, dset_type, loss, acc):\n", " fpath = os.path.join(self.history_dir, dset_type+'.csv')\n", " with open(fpath, 'a') as f:\n", " f.write('{},{},{}\\n'.format(self.epoch, loss, acc))\n", "\n", " def save_history(self, dset_type, loss, acc):\n", " self.loss_history[dset_type] = np.append(\n", " self.loss_history[dset_type], loss)\n", " self.acc_history[dset_type] = np.append(\n", " self.acc_history[dset_type], acc)\n", " self.append_history_to_file(dset_type, loss, acc)\n", "\n", " if dset_type == 'val' and self.is_best_loss(loss):\n", " self.best_val_loss = loss\n", " self.best_val_loss_epoch = self.epoch\n", "\n", " def is_best_loss(self, loss):\n", " return loss < self.best_val_loss\n", "\n", " def save_weights(self, model, trn_loss, val_loss, trn_acc, val_acc):\n", " weights_fname = self.name+'-weights-%d-%.3f-%.3f-%.3f-%.3f.pth' % (\n", " self.epoch, trn_loss, trn_acc, val_loss, val_acc)\n", " weights_fpath = os.path.join(self.weights_dir, weights_fname)\n", " torch.save({\n", " 'last_epoch': self.epoch,\n", " 'trn_loss': trn_loss,\n", " 'val_loss': val_loss,\n", " 'trn_acc': trn_acc,\n", " 'val_acc': val_acc,\n", " 'best_val_loss': self.best_val_loss,\n", " 'best_val_loss_epoch': self.best_val_loss_epoch,\n", " 'experiment': self.name,\n", " 'state_dict': model.state_dict()\n", " }, weights_fpath )\n", " shutil.copyfile(weights_fpath, self.latest_weights)\n", " if self.is_best_loss(val_loss):\n", " self.best_weights_path = weights_fpath\n", "\n", " def load_weights(self, model, fpath):\n", " self.log(\"loading weights '{}'\".format(fpath))\n", " state = torch.load(fpath)\n", " model.load_state_dict(state['state_dict'])\n", " self.log(\"loaded weights from experiment %s (last_epoch %d, trn_loss %s, trn_acc %s, val_loss %s, val_acc %s)\" % (\n", " self.name, state['last_epoch'], state['trn_loss'],\n", " state['trn_acc'], state['val_loss'], state['val_acc']))\n", " return model, state\n", "\n", " def save_optimizer(self, optimizer, val_loss):\n", " optim_fname = self.name+'-optim-%d.pth' % (self.epoch)\n", " optim_fpath = os.path.join(self.weights_dir, optim_fname)\n", " torch.save({\n", " 'last_epoch': self.epoch,\n", " 'experiment': self.name,\n", " 'state_dict': optimizer.state_dict()\n", " }, optim_fpath )\n", " shutil.copyfile(optim_fpath, self.latest_optimizer)\n", " if self.is_best_loss(val_loss):\n", " self.best_optimizer_path = optim_fpath\n", "\n", " def load_optimizer(self, optimizer, fpath):\n", " self.log(\"loading optimizer '{}'\".format(fpath))\n", " optim = torch.load(fpath)\n", " optimizer.load_state_dict(optim['state_dict'])\n", " self.log(\"loaded optimizer from session {}, last_epoch {}\"\n", " .format(optim['experiment'], optim['last_epoch']))\n", " return optimizer\n", " \n", " def plot_and_save_history(self):\n", " trn_data = np.loadtxt(self.train_history_fpath, delimiter=',').reshape(-1, 3)\n", " val_data = np.loadtxt(self.val_history_fpath, delimiter=',').reshape(-1, 3)\n", " \n", " trn_epoch, trn_loss, trn_acc = np.split(trn_data, [1,2], axis=1)\n", " val_epoch, val_loss, val_acc = np.split(val_data, [1,2], axis=1)\n", "\n", " # Loss\n", " fig, ax = plt.subplots(1, 1, figsize=(6, 5))\n", " plt.plot(trn_epoch, trn_loss, label='Train')\n", " plt.plot(val_epoch, val_loss, label='Validation')\n", " plt.xlabel('Epoch')\n", " plt.ylabel('Loss')\n", " plt.legend()\n", " ax.set_yscale('log')\n", " loss_fname = os.path.join(self.history_dir, 'loss.png')\n", " plt.savefig(loss_fname)\n", "\n", " # Accuracy\n", " fig, ax = plt.subplots(1, 1, figsize=(6, 5))\n", " plt.plot(trn_epoch, trn_acc, label='Train')\n", " plt.plot(val_epoch, val_acc, label='Validation')\n", " plt.xlabel('Epoch')\n", " plt.ylabel('Accuracy')\n", " ax.set_yscale('log')\n", " plt.legend()\n", " acc_fname = os.path.join(self.history_dir, 'accuracy.png')\n", " plt.savefig(acc_fname)\n", "\n", " # Combined View - loss-accuracy.png\n", " loss_acc_fname = os.path.join(self.history_dir, 'loss-acc.png')\n", " os.system('convert +append {} {} {}'.format(loss_fname, acc_fname, loss_acc_fname))\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## New Experiment" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "N_EPOCHS = 5\n", "MAX_PATIENCE = 50\n", "LEARNING_RATE = 1e-4\n", "LR_DECAY = 0.995\n", "DECAY_LR_EVERY_N_EPOCHS = 1\n", "EXPERIMENT_NAME = 'cifarexp1'\n", "\n", "logger = get_logger(ch_log_level=logging.INFO, fh_log_level=logging.INFO)\n", "model = DeeperCNN(logger).cuda()\n", "criterion = nn.CrossEntropyLoss().cuda()\n", "optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)\n", "\n", "print(' + Number of params: {}'.format(\n", " sum([p.data.nelement() for p in model.parameters()])))\n", "exp = Experiment(EXPERIMENT_NAME, CIFAR10_PATH, logger)\n", "\n", "# Create New Experiment\n", "exp.init()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "for epoch in range(exp.epoch, exp.epoch+N_EPOCHS):\n", " since = time.time()\n", "\n", " ### Train ###\n", " trn_loss, trn_acc = train(model, cifar_train_loader, criterion, optimizer, epoch)\n", " logger.info('Epoch {:d}: Train - Loss: {:.4f}\\tAcc: {:.4f}'.format(epoch, trn_loss, trn_acc)) \n", " time_elapsed = time.time() - since \n", " logger.info('Train Time {:.0f}m {:.0f}s'.format(\n", " time_elapsed // 60, time_elapsed % 60))\n", " \n", " ### Test ###\n", " val_loss, val_acc = test(model, cifar_test_loader, criterion, epoch) \n", " logger.info('Val - Loss: {:.4f}, Acc: {:.4f}'.format(val_loss, val_acc))\n", " time_elapsed = time.time() - since \n", " logger.info('Total Time {:.0f}m {:.0f}s\\n'.format(\n", " time_elapsed // 60, time_elapsed % 60))\n", "\n", " ### Save Metrics ###\n", " exp.save_history('train', trn_loss, trn_acc)\n", " exp.save_history('val', val_loss, val_acc)\n", " \n", " ### Checkpoint ### \n", " exp.save_weights(model, trn_loss, val_loss, trn_acc, val_acc)\n", " exp.save_optimizer(optimizer, val_loss)\n", " \n", " ### Plot Online ###\n", " exp.update_viz_loss_plot()\n", " exp.update_viz_acc_plot()\n", " exp.update_viz_summary_plot()\n", " \n", " ## Early Stopping ##\n", " if (epoch - exp.best_val_loss_epoch) > MAX_PATIENCE:\n", " logger.info((\"Early stopping at epoch %d since no \" \n", " + \"better loss found since epoch %.3\") \n", " % (epoch, exp.best_val_loss))\n", " break\n", "\n", " ### Adjust Lr ###\n", " adjust_learning_rate(LEARNING_RATE, LR_DECAY, optimizer, \n", " epoch, DECAY_LR_EVERY_N_EPOCHS)\n", " \n", " exp.epoch += 1" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Resume Experiment" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "logger = get_logger(ch_log_level=logging.INFO, fh_log_level=logging.INFO)\n", "model = DeeperCNN(logger).cuda()\n", "criterion = nn.CrossEntropyLoss().cuda()\n", "optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)\n", "exp = Experiment(EXPERIMENT_NAME, CIFAR10_PATH, logger)\n", "\n", "#Load saved weights and optimizer\n", "model, optimizer = exp.resume(model, optimizer)\n", "\n", "loss,acc = test(model, cifar_test_loader, criterion, exp.epoch)\n", "print(\"LOSS\",str(loss),\"ACC\",str(acc))\n", "exp.plot_and_save_history()" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "# Resources" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true, "hidden": true }, "source": [ "## Tutorials" ] }, { "cell_type": "markdown", "metadata": { "hidden": true }, "source": [ "* [Official Pytorch Tutorials](http://pytorch.org/tutorials/)\n", "* [Pytorch Documentation](http://pytorch.org/docs)\n", "* [Pytorch Transfer Learning](http://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html)" ] }, { "cell_type": "markdown", "metadata": { "hidden": true }, "source": [ "## Examples" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true, "hidden": true }, "source": [ "### CNNs" ] }, { "cell_type": "markdown", "metadata": { "hidden": true }, "source": [ " * [VGG](https://github.com/pytorch/vision/blob/master/torchvision/models/vgg.py)\n", " * [ResNet](https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py)\n", " * [InceptionNet](https://github.com/pytorch/vision/blob/master/torchvision/models/inception.py)\n", " * [SqeezeNet](https://github.com/pytorch/vision/blob/master/torchvision/models/squeezenet.py)\n", " * [DenseNet](https://github.com/pytorch/vision/blob/master/torchvision/models/densenet.py)\n", " * [FCDenseNet (Tiramisu)](https://github.com/bfortuner/pytorch_tiramisu/blob/master/tiramisu-pytorch.ipynb)\n", " * [Sub-pixel CNN (superresolution)](https://github.com/pytorch/examples/tree/master/super_resolution)" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true, "hidden": true }, "source": [ "### GANs" ] }, { "cell_type": "markdown", "metadata": { "hidden": true }, "source": [ "* [DCGAN](https://github.com/pytorch/examples/tree/master/dcgan)\n", "* [Wasserstein GAN](https://github.com/martinarjovsky/WassersteinGAN)" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true, "hidden": true }, "source": [ "### Other" ] }, { "cell_type": "markdown", "metadata": { "hidden": true }, "source": [ "* [Pytorch Examples](https://github.com/pytorch/examples)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" }, "latex_envs": { "LaTeX_envs_menu_present": true, "autocomplete": true, "bibliofile": "biblio.bib", "cite_by": "apalike", "current_citInitial": 1, "eqLabelWithNumbers": true, "eqNumInitial": 1, "hotkeys": { "equation": "Ctrl-E", "itemize": "Ctrl-I" }, "labels_anchors": false, "latex_user_defs": false, "report_style_numbering": false, "user_envs_cfg": false }, "toc": { "colors": { "hover_highlight": "#DAA520", "running_highlight": "#FF0000", "selected_highlight": "#FFD700" }, "moveMenuLeft": true, "nav_menu": { "height": "192px", "width": "254px" }, "navigate_menu": true, "number_sections": true, "sideBar": true, "threshold": 4, "toc_cell": false, "toc_section_display": "block", "toc_window_display": false, "widenNotebook": false } }, "nbformat": 4, "nbformat_minor": 2 }