{ "cells": [ { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "import torch\n", "import numpy as np" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Tensors" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "tensor_from_list = torch.tensor([[1., 0.], [0., 1.]])" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor([[1., 0.],\n", " [0., 1.]])\n" ] } ], "source": [ "print(tensor_from_list)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "torch.Size([2, 2])\n" ] } ], "source": [ "print(tensor_from_list.size())" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "tensor_from_numpy = torch.tensor(np.array([[1., 0.], [0., 1.]]))" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor([[1., 0.],\n", " [0., 1.]], dtype=torch.float64)\n" ] } ], "source": [ "print(tensor_from_numpy)" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor([[ 0.0000e+00, -1.5846e+29],\n", " [-7.5247e+03, 2.0005e+00],\n", " [ 9.8091e-45, 0.0000e+00]])\n" ] } ], "source": [ "tensor_empty = torch.empty(3, 2)\n", "print(tensor_empty)" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor([[0.8953, 0.7131],\n", " [0.7226, 0.4733],\n", " [0.7516, 0.9558]])\n" ] } ], "source": [ "tensor_rand = torch.rand(3, 2)\n", "print(tensor_rand)" ] }, { "cell_type": "code", "execution_count": 46, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor([[0.7644, 0.3751],\n", " [0.0751, 0.5308],\n", " [0.9660, 0.2770]])\n" ] } ], "source": [ "# Set the seed for random number generator to get consistent, reproducible results\n", "torch.manual_seed(24)\n", "tensor_rand = torch.rand(3, 2)\n", "print(tensor_rand)" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor([[0., 0.],\n", " [0., 0.],\n", " [0., 0.]])\n" ] } ], "source": [ "tensor_zeros = torch.zeros(3, 2)\n", "print(tensor_zeros)" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor([[1., 1.],\n", " [1., 1.],\n", " [1., 1.]])\n" ] } ], "source": [ "tensor_ones = torch.ones(3, 2)\n", "print(tensor_ones)" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.float32" ] }, "execution_count": 48, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tensor_from_list.dtype" ] }, { "cell_type": "code", "execution_count": 50, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor([[1., 0.],\n", " [0., 1.]], dtype=torch.float64)\n" ] } ], "source": [ "tensor_from_list_float64 = torch.tensor([[1., 0.], [0., 1.]], \n", " dtype=torch.float64)\n", "print(tensor_from_list_float64)" ] }, { "cell_type": "code", "execution_count": 52, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "device(type='cpu')" ] }, "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tensor_rand.device" ] }, { "cell_type": "code", "execution_count": 53, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "False" ] }, "execution_count": 53, "metadata": {}, "output_type": "execute_result" } ], "source": [ "torch.cuda.is_available()" ] }, { "cell_type": "code", "execution_count": 60, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[0.2989, 0.3510],\n", " [0.0529, 0.1988],\n", " [0.8022, 0.1249]])" ] }, "execution_count": 60, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x = torch.rand(3, 2)\n", "x" ] }, { "cell_type": "code", "execution_count": 61, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[0.6708, 0.9704],\n", " [0.4365, 0.7187],\n", " [0.7336, 0.1431]])" ] }, "execution_count": 61, "metadata": {}, "output_type": "execute_result" } ], "source": [ "y = torch.rand(3, 2)\n", "y" ] }, { "cell_type": "code", "execution_count": 62, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[0.9697, 1.3214],\n", " [0.4894, 0.9176],\n", " [1.5357, 0.2680]])" ] }, "execution_count": 62, "metadata": {}, "output_type": "execute_result" } ], "source": [ "torch.add(x, y)" ] }, { "cell_type": "code", "execution_count": 63, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[0.9697, 1.3214],\n", " [0.4894, 0.9176],\n", " [1.5357, 0.2680]])" ] }, "execution_count": 63, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x + y" ] }, { "cell_type": "code", "execution_count": 64, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[0.29888242, 0.35096592],\n", " [0.05293709, 0.19883835],\n", " [0.8021769 , 0.12490124]], dtype=float32)" ] }, "execution_count": 64, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x_numpy = x.numpy()\n", "x_numpy" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Dataset" ] }, { "cell_type": "code", "execution_count": 66, "metadata": {}, "outputs": [], "source": [ "from sklearn.datasets import make_classification\n", "from torch.utils.data import Dataset" ] }, { "cell_type": "code", "execution_count": 70, "metadata": {}, "outputs": [], "source": [ "X, y = make_classification(n_samples=100, \n", " n_features=5, \n", " random_state=42)" ] }, { "cell_type": "code", "execution_count": 94, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([-0.43066755, 0.67287309, -0.72427983, -0.53963044, -0.65160035])" ] }, "execution_count": 94, "metadata": {}, "output_type": "execute_result" } ], "source": [ "X[0]" ] }, { "cell_type": "code", "execution_count": 150, "metadata": {}, "outputs": [], "source": [ "class CustomDataset(Dataset):\n", " def __init__(self,\n", " X, y,\n", " transform=None):\n", " self.X = X\n", " self.y = y\n", " self.transform = transform\n", " \n", " def __len__(self):\n", " return len(self.X)\n", " \n", " def __getitem__(self, idx):\n", " x = X[idx]\n", " label = y[idx]\n", " \n", " if self.transform:\n", " x = self.transform(x)\n", " label = self.transform(label)\n", " \n", " return x, label" ] }, { "cell_type": "code", "execution_count": 151, "metadata": {}, "outputs": [], "source": [ "custom_dataset = CustomDataset(X, y)" ] }, { "cell_type": "code", "execution_count": 152, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "100" ] }, "execution_count": 152, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(custom_dataset)" ] }, { "cell_type": "code", "execution_count": 153, "metadata": {}, "outputs": [], "source": [ "x_0, y_0 = custom_dataset[0]" ] }, { "cell_type": "code", "execution_count": 154, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([-0.43066755, 0.67287309, -0.72427983, -0.53963044, -0.65160035])" ] }, "execution_count": 154, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x_0" ] }, { "cell_type": "code", "execution_count": 155, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0" ] }, "execution_count": 155, "metadata": {}, "output_type": "execute_result" } ], "source": [ "y_0" ] }, { "cell_type": "code", "execution_count": 195, "metadata": {}, "outputs": [], "source": [ "from functools import partial\n", "torch_tensor_float32 = partial(torch.tensor, dtype=torch.float32)\n", "transformed_dataset = CustomDataset(X, y,\n", " transform=torch_tensor_float32)" ] }, { "cell_type": "code", "execution_count": 196, "metadata": {}, "outputs": [], "source": [ "x_0, y_0 = transformed_dataset[0]" ] }, { "cell_type": "code", "execution_count": 197, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([-0.4307, 0.6729, -0.7243, -0.5396, -0.6516])" ] }, "execution_count": 197, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x_0" ] }, { "cell_type": "code", "execution_count": 198, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor(0.)" ] }, "execution_count": 198, "metadata": {}, "output_type": "execute_result" } ], "source": [ "y_0" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# DataLoader" ] }, { "cell_type": "code", "execution_count": 160, "metadata": {}, "outputs": [], "source": [ "from torch.utils.data import DataLoader" ] }, { "cell_type": "code", "execution_count": 199, "metadata": {}, "outputs": [], "source": [ "dataloader = DataLoader(transformed_dataset,\n", " batch_size=4,\n", " shuffle=True)" ] }, { "cell_type": "code", "execution_count": 200, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[Batch 0] Number of rows in batch: 4\n", "[Batch 1] Number of rows in batch: 4\n", "[Batch 2] Number of rows in batch: 4\n", "[Batch 3] Number of rows in batch: 4\n", "[Batch 4] Number of rows in batch: 4\n", "[Batch 5] Number of rows in batch: 4\n", "[Batch 6] Number of rows in batch: 4\n", "[Batch 7] Number of rows in batch: 4\n", "[Batch 8] Number of rows in batch: 4\n", "[Batch 9] Number of rows in batch: 4\n", "[Batch 10] Number of rows in batch: 4\n", "[Batch 11] Number of rows in batch: 4\n", "[Batch 12] Number of rows in batch: 4\n", "[Batch 13] Number of rows in batch: 4\n", "[Batch 14] Number of rows in batch: 4\n", "[Batch 15] Number of rows in batch: 4\n", "[Batch 16] Number of rows in batch: 4\n", "[Batch 17] Number of rows in batch: 4\n", "[Batch 18] Number of rows in batch: 4\n", "[Batch 19] Number of rows in batch: 4\n", "[Batch 20] Number of rows in batch: 4\n", "[Batch 21] Number of rows in batch: 4\n", "[Batch 22] Number of rows in batch: 4\n", "[Batch 23] Number of rows in batch: 4\n", "[Batch 24] Number of rows in batch: 4\n", "CPU times: user 12.7 ms, sys: 3.9 ms, total: 16.6 ms\n", "Wall time: 15.5 ms\n" ] } ], "source": [ "%%time\n", "for i_batch, sample_batched in enumerate(dataloader):\n", " print(f\"[Batch {i_batch}] Number of rows in batch: {len(sample_batched[0])}\")" ] }, { "cell_type": "code", "execution_count": 201, "metadata": {}, "outputs": [], "source": [ "dataloader = DataLoader(transformed_dataset,\n", " batch_size=4,\n", " shuffle=True,\n", " num_workers=4)" ] }, { "cell_type": "code", "execution_count": 202, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[Batch 0] Number of rows in batch: 4\n", "[Batch 1] Number of rows in batch: 4\n", "[Batch 2] Number of rows in batch: 4\n", "[Batch 3] Number of rows in batch: 4\n", "[Batch 4] Number of rows in batch: 4\n", "[Batch 5] Number of rows in batch: 4\n", "[Batch 6] Number of rows in batch: 4\n", "[Batch 7] Number of rows in batch: 4\n", "[Batch 8] Number of rows in batch: 4\n", "[Batch 9] Number of rows in batch: 4\n", "[Batch 10] Number of rows in batch: 4\n", "[Batch 11] Number of rows in batch: 4\n", "[Batch 12] Number of rows in batch: 4\n", "[Batch 13] Number of rows in batch: 4\n", "[Batch 14] Number of rows in batch: 4\n", "[Batch 15] Number of rows in batch: 4\n", "[Batch 16] Number of rows in batch: 4\n", "[Batch 17] Number of rows in batch: 4\n", "[Batch 18] Number of rows in batch: 4\n", "[Batch 19] Number of rows in batch: 4\n", "[Batch 20] Number of rows in batch: 4\n", "[Batch 21] Number of rows in batch: 4\n", "[Batch 22] Number of rows in batch: 4\n", "[Batch 23] Number of rows in batch: 4\n", "[Batch 24] Number of rows in batch: 4\n", "CPU times: user 30.6 ms, sys: 36.6 ms, total: 67.2 ms\n", "Wall time: 101 ms\n" ] } ], "source": [ "%%time\n", "for i_batch, sample_batched in enumerate(dataloader):\n", " print(f\"[Batch {i_batch}] Number of rows in batch: {len(sample_batched[0])}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Model Definition" ] }, { "cell_type": "code", "execution_count": 203, "metadata": {}, "outputs": [], "source": [ "model = torch.nn.Sequential(\n", " torch.nn.Linear(5, 5),\n", " torch.nn.ReLU(),\n", " torch.nn.Linear(5, 3),\n", " torch.nn.ReLU(),\n", " torch.nn.Linear(3, 1),\n", " torch.nn.Sigmoid()\n", ")" ] }, { "cell_type": "code", "execution_count": 204, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Sequential(\n", " (0): Linear(in_features=5, out_features=5, bias=True)\n", " (1): ReLU()\n", " (2): Linear(in_features=5, out_features=3, bias=True)\n", " (3): ReLU()\n", " (4): Linear(in_features=3, out_features=1, bias=True)\n", " (5): Sigmoid()\n", ")\n" ] } ], "source": [ "print(model)" ] }, { "cell_type": "code", "execution_count": 205, "metadata": {}, "outputs": [], "source": [ "num_features = 5\n", "num_outputs = 1\n", "layer_dims = [num_features, 5, 3, num_outputs]" ] }, { "cell_type": "code", "execution_count": 206, "metadata": {}, "outputs": [], "source": [ "class BinaryClassifier(torch.nn.Sequential):\n", " def __init__(self, layer_dims):\n", " super(BinaryClassifier, self).__init__()\n", " \n", " for idx, dim in enumerate(layer_dims):\n", " if (idx < len(layer_dims) - 1):\n", " module = torch.nn.Linear(dim, layer_dims[idx + 1])\n", " self.add_module(f\"linear{idx}\", module)\n", " \n", " if idx < len(layer_dims) - 2:\n", " activation = torch.nn.ReLU()\n", " self.add_module(f\"relu{idx}\", activation)\n", " elif idx == len(layer_dims) - 2:\n", " activation = torch.nn.Sigmoid()\n", " self.add_module(f\"sigmoid{idx}\", activation)" ] }, { "cell_type": "code", "execution_count": 207, "metadata": {}, "outputs": [], "source": [ "bc_model = BinaryClassifier(layer_dims)" ] }, { "cell_type": "code", "execution_count": 208, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "BinaryClassifier(\n", " (linear0): Linear(in_features=5, out_features=5, bias=True)\n", " (relu0): ReLU()\n", " (linear1): Linear(in_features=5, out_features=3, bias=True)\n", " (relu1): ReLU()\n", " (linear2): Linear(in_features=3, out_features=1, bias=True)\n", " (sigmoid2): Sigmoid()\n", ")\n" ] } ], "source": [ "print(bc_model)" ] }, { "cell_type": "code", "execution_count": 209, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Linear(in_features=5, out_features=5, bias=True)" ] }, "execution_count": 209, "metadata": {}, "output_type": "execute_result" } ], "source": [ "bc_model.linear0" ] }, { "cell_type": "code", "execution_count": 210, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Linear(in_features=5, out_features=3, bias=True)" ] }, "execution_count": 210, "metadata": {}, "output_type": "execute_result" } ], "source": [ "bc_model.linear1" ] }, { "cell_type": "code", "execution_count": 211, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "6" ] }, "execution_count": 211, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(bc_model)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Automatic Differentiation" ] }, { "cell_type": "code", "execution_count": 315, "metadata": {}, "outputs": [], "source": [ "x = torch.ones(2, 3, \n", " requires_grad=True)" ] }, { "cell_type": "code", "execution_count": 316, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[1., 1., 1.],\n", " [1., 1., 1.]], requires_grad=True)" ] }, "execution_count": 316, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x" ] }, { "cell_type": "code", "execution_count": 317, "metadata": {}, "outputs": [], "source": [ "w = 2 * x" ] }, { "cell_type": "code", "execution_count": 318, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[2., 2., 2.],\n", " [2., 2., 2.]], grad_fn=)" ] }, "execution_count": 318, "metadata": {}, "output_type": "execute_result" } ], "source": [ "w" ] }, { "cell_type": "code", "execution_count": 319, "metadata": {}, "outputs": [], "source": [ "y = w * w * w + 3 * w * w + 2 * w + 1" ] }, { "cell_type": "code", "execution_count": 320, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[25., 25., 25.],\n", " [25., 25., 25.]], grad_fn=)" ] }, "execution_count": 320, "metadata": {}, "output_type": "execute_result" } ], "source": [ "y" ] }, { "cell_type": "code", "execution_count": 321, "metadata": {}, "outputs": [], "source": [ "z = torch.sum(y)" ] }, { "cell_type": "code", "execution_count": 322, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor(150., grad_fn=)" ] }, "execution_count": 322, "metadata": {}, "output_type": "execute_result" } ], "source": [ "z" ] }, { "cell_type": "code", "execution_count": 323, "metadata": {}, "outputs": [], "source": [ "z.backward()" ] }, { "cell_type": "code", "execution_count": 324, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[52., 52., 52.],\n", " [52., 52., 52.]])" ] }, "execution_count": 324, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x.grad" ] }, { "cell_type": "code", "execution_count": 312, "metadata": {}, "outputs": [], "source": [ "t = torch.sum(w)" ] }, { "cell_type": "code", "execution_count": 327, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([[52., 52., 52.],\n", " [52., 52., 52.]], grad_fn=)" ] }, "execution_count": 327, "metadata": {}, "output_type": "execute_result" } ], "source": [ "(3 * w * w + 6 * w + 2) * 2" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Model Training" ] }, { "cell_type": "code", "execution_count": 330, "metadata": {}, "outputs": [], "source": [ "criterion = torch.nn.BCELoss()" ] }, { "cell_type": "code", "execution_count": 331, "metadata": {}, "outputs": [], "source": [ "optimizer = torch.optim.Adam(bc_model.parameters())" ] }, { "cell_type": "code", "execution_count": 332, "metadata": {}, "outputs": [], "source": [ "num_epochs = 10" ] }, { "cell_type": "code", "execution_count": 221, "metadata": {}, "outputs": [], "source": [ "for epoch in range(num_epochs):\n", " for idx, (X_batch, labels) in enumerate(dataloader):\n", " optimizer.zero_grad()\n", " outputs = bc_model(X_batch)\n", " loss = criterion(outputs, labels)\n", " loss.backward()\n", " optimizer.step()" ] }, { "cell_type": "code", "execution_count": 222, "metadata": {}, "outputs": [], "source": [ "pred_var = bc_model(transformed_dataset[0][0])" ] }, { "cell_type": "code", "execution_count": 223, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "tensor([0.5884], grad_fn=)" ] }, "execution_count": 223, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pred_var" ] }, { "cell_type": "code", "execution_count": 224, "metadata": {}, "outputs": [], "source": [ "pred_var.grad" ] }, { "cell_type": "code", "execution_count": 225, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.58835465" ] }, "execution_count": 225, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pred_var.detach().numpy()[0]" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.5" } }, "nbformat": 4, "nbformat_minor": 2 }