{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "Logging Curves", "provenance": [], "collapsed_sections": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "code", "metadata": { "id": "Tx5C-o8hy0y6" }, "source": [ "!pip install comet_ml" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "vIofsUO4zQdh", "outputId": "0753b72e-f64a-41fb-dcba-1055e8aef55a" }, "source": [ "import comet_ml\n", "import getpass, os\n", "os.environ[\"COMET_API_KEY\"] = getpass.getpass(\"Paste your COMET API KEY: \")" ], "execution_count": null, "outputs": [ { "output_type": "stream", "text": [ "Paste your COMET API KEY: ··········\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "X9_BISZsbdVp" }, "source": [ "experiment = comet_ml.Experiment(project_name=\"logging-curves\")" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "YQZUPneCzR8x" }, "source": [ "import numpy as np\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "import torch.optim as optim\n", "from torchvision import datasets, transforms\n", "from torch.optim.lr_scheduler import StepLR\n", "\n", "\n", "class Net(nn.Module):\n", " def __init__(self):\n", " super(Net, self).__init__()\n", " self.conv1 = nn.Conv2d(3, 6, 5)\n", " self.pool = nn.MaxPool2d(2, 2)\n", " self.conv2 = nn.Conv2d(6, 16, 5)\n", " self.fc1 = nn.Linear(16 * 5 * 5, 120)\n", " self.fc2 = nn.Linear(120, 84)\n", " self.fc3 = nn.Linear(84, 10)\n", "\n", " def forward(self, x):\n", " x = self.pool(F.relu(self.conv1(x)))\n", " x = self.pool(F.relu(self.conv2(x)))\n", " x = x.view(-1, 16 * 5 * 5)\n", " x = F.relu(self.fc1(x))\n", " x = F.relu(self.fc2(x))\n", " x = self.fc3(x)\n", " x = F.log_softmax(x, 1)\n", " \n", " return x" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "U2BnGvKyE1gN" }, "source": [ "from sklearn.metrics import precision_recall_curve, roc_curve\n", "\n", "N_CLASSES = 10" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "ZDN37X4c9xn4" }, "source": [ "def train(model, device, train_loader, optimizer, epoch):\n", " model.train()\n", " for batch_idx, (data, target) in enumerate(train_loader):\n", " data, target = data.to(device), target.to(device)\n", " optimizer.zero_grad()\n", " output = model(data)\n", " loss = F.nll_loss(output, target)\n", " loss.backward()\n", " optimizer.step()\n", "\n", "\n", "def test(model, device, test_loader, epoch):\n", " model.eval()\n", " test_loss = 0\n", " correct = 0\n", " \n", " class_probabilities = []\n", " targets = []\n", "\n", " with torch.no_grad():\n", " for data, target in test_loader:\n", " data, target = data.to(device), target.to(device)\n", " output = model(data)\n", "\n", " output_proba = output.exp().squeeze(0).cpu().detach().numpy().tolist()\n", " \n", " class_probabilities.extend(output_proba)\n", " targets.extend(F.one_hot(torch.tensor(target), N_CLASSES).cpu().numpy().tolist())\n", "\n", " test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss\n", " pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability\n", " correct += pred.eq(target.view_as(pred)).sum().item()\n", "\n", " test_loss /= len(test_loader.dataset)\n", "\n", " print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n", " test_loss, correct, len(test_loader.dataset),\n", " 100. * correct / len(test_loader.dataset)))\n", " \n", " return np.array(targets), np.array(class_probabilities)\n", "\n", "\n", "def log_curves(y_true, y_pred, step): \n", " for i in range(N_CLASSES):\n", " precision, recall, _ = precision_recall_curve(y_true[:, i], y_pred[:, i])\n", " fpr, tpr, _ = roc_curve(y_true[:, i], y_pred[:, i])\n", " \n", " experiment.log_curve(f\"pr-curve-class-{i}\", recall, precision, step=step)\n", " experiment.log_curve(f\"roc-curve-class-{i}\", fpr, tpr, step=step)" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "wyZcSZd-BGAU", "outputId": "bc75d19e-4e01-4bc9-9ee9-f2e3d476cdc6" }, "source": [ "RANDOM_STATE = 42\n", "torch.manual_seed(RANDOM_STATE)\n", "\n", "device = torch.device(\"cuda\")\n", "\n", "train_kwargs = {'batch_size': 128}\n", "test_kwargs = {'batch_size': 128}\n", "\n", "cuda_kwargs = {'num_workers': 1, 'pin_memory': True, 'shuffle': True}\n", "train_kwargs.update(cuda_kwargs)\n", "test_kwargs.update(cuda_kwargs)\n", "\n", "transform=transforms.Compose([\n", " transforms.ToTensor(),\n", " transforms.Normalize((0.1307,), (0.3081,))\n", "])\n", "train_data = datasets.CIFAR10('../data', train=True, download=True,\n", " transform=transform)\n", "test_data = datasets.CIFAR10('../data', train=False,\n", " transform=transform)\n", "\n", "train_loader = torch.utils.data.DataLoader(train_data,**train_kwargs)\n", "test_loader = torch.utils.data.DataLoader(test_data, **test_kwargs)" ], "execution_count": null, "outputs": [ { "output_type": "stream", "text": [ "Files already downloaded and verified\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "GJfCfbJtBWJP" }, "source": [ "EPOCHS = 10\n", "\n", "model = Net().to(device)\n", "optimizer = optim.Adadelta(model.parameters(), lr=1.0)\n", "\n", "scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n", "for epoch in range(1, EPOCHS + 1):\n", " train(model, device, train_loader, optimizer, epoch)\n", " targets, class_probabilities = test(model, device, test_loader, epoch)\n", " \n", " log_curves(targets, class_probabilities, epoch)\n", " \n", " scheduler.step()" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "dts-GrIX6qOb" }, "source": [ "" ], "execution_count": null, "outputs": [] } ] }