{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "Logging Curves", "provenance": [], "collapsed_sections": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "code", "metadata": { "id": "Tx5C-o8hy0y6" }, "source": [ "%pip install comet_ml" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "vIofsUO4zQdh", "outputId": "0753b72e-f64a-41fb-dcba-1055e8aef55a" }, "source": [ "import comet_ml\n", "comet_ml.login()" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "X9_BISZsbdVp" }, "source": [ "experiment = comet_ml.start(project_name=\"logging-curves\")" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "YQZUPneCzR8x" }, "source": [ "import numpy as np\n", "\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "import torch.optim as optim\n", "from torchvision import datasets, transforms\n", "from torch.optim.lr_scheduler import StepLR\n", "\n", "\n", "class Net(nn.Module):\n", " def __init__(self):\n", " super(Net, self).__init__()\n", " self.conv1 = nn.Conv2d(3, 6, 5)\n", " self.pool = nn.MaxPool2d(2, 2)\n", " self.conv2 = nn.Conv2d(6, 16, 5)\n", " self.fc1 = nn.Linear(16 * 5 * 5, 120)\n", " self.fc2 = nn.Linear(120, 84)\n", " self.fc3 = nn.Linear(84, 10)\n", "\n", " def forward(self, x):\n", " x = self.pool(F.relu(self.conv1(x)))\n", " x = self.pool(F.relu(self.conv2(x)))\n", " x = x.view(-1, 16 * 5 * 5)\n", " x = F.relu(self.fc1(x))\n", " x = F.relu(self.fc2(x))\n", " x = self.fc3(x)\n", " x = F.log_softmax(x, 1)\n", "\n", " return x" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "U2BnGvKyE1gN" }, "source": [ "from sklearn.metrics import precision_recall_curve, roc_curve\n", "\n", "N_CLASSES = 10" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "ZDN37X4c9xn4" }, "source": [ "def train(model, device, train_loader, optimizer, epoch):\n", " model.train()\n", " for batch_idx, (data, target) in enumerate(train_loader):\n", " data, target = data.to(device), target.to(device)\n", " optimizer.zero_grad()\n", " output = model(data)\n", " loss = F.nll_loss(output, target)\n", " loss.backward()\n", " optimizer.step()\n", "\n", "\n", "def test(model, device, test_loader, epoch):\n", " model.eval()\n", " test_loss = 0\n", " correct = 0\n", "\n", " class_probabilities = []\n", " targets = []\n", "\n", " with torch.no_grad():\n", " for data, target in test_loader:\n", " data, target = data.to(device), target.to(device)\n", " output = model(data)\n", "\n", " output_proba = output.exp().squeeze(0).cpu().detach().numpy().tolist()\n", "\n", " class_probabilities.extend(output_proba)\n", " targets.extend(\n", " F.one_hot(torch.tensor(target), N_CLASSES).cpu().numpy().tolist()\n", " )\n", "\n", " test_loss += F.nll_loss(\n", " output, target, reduction=\"sum\"\n", " ).item() # sum up batch loss\n", " pred = output.argmax(\n", " dim=1, keepdim=True\n", " ) # get the index of the max log-probability\n", " correct += pred.eq(target.view_as(pred)).sum().item()\n", "\n", " test_loss /= len(test_loader.dataset)\n", "\n", " print(\n", " \"\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n\".format(\n", " test_loss,\n", " correct,\n", " len(test_loader.dataset),\n", " 100.0 * correct / len(test_loader.dataset),\n", " )\n", " )\n", "\n", " return np.array(targets), np.array(class_probabilities)\n", "\n", "\n", "def log_curves(y_true, y_pred, step):\n", " for i in range(N_CLASSES):\n", " precision, recall, _ = precision_recall_curve(y_true[:, i], y_pred[:, i])\n", " fpr, tpr, _ = roc_curve(y_true[:, i], y_pred[:, i])\n", "\n", " experiment.log_curve(f\"pr-curve-class-{i}\", recall, precision, step=step)\n", " experiment.log_curve(f\"roc-curve-class-{i}\", fpr, tpr, step=step)" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "wyZcSZd-BGAU", "outputId": "bc75d19e-4e01-4bc9-9ee9-f2e3d476cdc6" }, "source": [ "RANDOM_STATE = 42\n", "torch.manual_seed(RANDOM_STATE)\n", "\n", "device = torch.device(\"cuda\")\n", "\n", "train_kwargs = {\"batch_size\": 128}\n", "test_kwargs = {\"batch_size\": 128}\n", "\n", "cuda_kwargs = {\"num_workers\": 1, \"pin_memory\": True, \"shuffle\": True}\n", "train_kwargs.update(cuda_kwargs)\n", "test_kwargs.update(cuda_kwargs)\n", "\n", "transform = transforms.Compose(\n", " [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]\n", ")\n", "train_data = datasets.CIFAR10(\"../data\", train=True, download=True, transform=transform)\n", "test_data = datasets.CIFAR10(\"../data\", train=False, transform=transform)\n", "\n", "train_loader = torch.utils.data.DataLoader(train_data, **train_kwargs)\n", "test_loader = torch.utils.data.DataLoader(test_data, **test_kwargs)" ], "execution_count": null, "outputs": [ { "output_type": "stream", "text": [ "Files already downloaded and verified\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "GJfCfbJtBWJP" }, "source": [ "EPOCHS = 10\n", "\n", "model = Net().to(device)\n", "optimizer = optim.Adadelta(model.parameters(), lr=1.0)\n", "\n", "scheduler = StepLR(optimizer, step_size=1, gamma=0.1)\n", "for epoch in range(1, EPOCHS + 1):\n", " train(model, device, train_loader, optimizer, epoch)\n", " targets, class_probabilities = test(model, device, test_loader, epoch)\n", "\n", " log_curves(targets, class_probabilities, epoch)\n", "\n", " scheduler.step()" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "dts-GrIX6qOb" }, "source": [ "" ], "execution_count": null, "outputs": [] } ] }