{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Net from Scratch in Numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss 9726.30003915 at run 0\n",
      "loss 814.959949267 at run 100\n",
      "loss 320.872497012 at run 200\n",
      "loss 259.944891952 at run 300\n",
      "loss 249.216935453 at run 400\n",
      "loss 245.718992076 at run 500\n",
      "loss 243.451981739 at run 600\n",
      "loss 241.492220103 at run 700\n",
      "loss 239.674597278 at run 800\n",
      "loss 237.9424254 at run 900\n",
      "CPU times: user 284 ms, sys: 576 ms, total: 860 ms\n",
      "Wall time: 1.06 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "n_in, n_h, n_out = 5, 4, 2\n",
    "W1 = np.random.rand(4, 5)\n",
    "W2 = np.random.rand(2, 4)\n",
    "\n",
    "M = 1000 # no of training examples\n",
    "\n",
    "x_in = np.random.rand(5, M) \n",
    "y = np.random.rand(2, M)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "\n",
    "for t in range(1000):\n",
    "    # forward pass\n",
    "    h = W1.dot(x_in)\n",
    "    relu_h = np.maximum(h, 0)\n",
    "    out = W2.dot(relu_h)\n",
    "    \n",
    "    # calculate loss\n",
    "    \n",
    "    loss = np.square(out - y).sum()\n",
    "    \n",
    "    # backprop\n",
    "    out_grad = 2*(out - y)\n",
    "    W2_grad = out_grad.dot(relu_h.T)\n",
    "    grad_relu_h = W2.T.dot(out_grad)\n",
    "    grad_h = grad_relu_h.copy()\n",
    "    grad_h[h < 0] = 0\n",
    "    W1_grad = grad_h.dot(x_in.T) \n",
    "    \n",
    "    # update params\n",
    "    W1 -= learning_rate * W1_grad\n",
    "    W2 -= learning_rate * W2_grad\n",
    "    \n",
    "    if t%100==0:\n",
    "        print(\"loss {0} at run {1}\".format(loss, t))\n",
    "    # repeat\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Net from Scratch using PyTorch Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss 14459.4521484 at run 0\n",
      "loss 745.285827637 at run 100\n",
      "loss 260.097869873 at run 200\n",
      "loss 222.204483032 at run 300\n",
      "loss 218.247543335 at run 400\n",
      "loss 217.245193481 at run 500\n",
      "loss 216.542648315 at run 600\n",
      "loss 215.900558472 at run 700\n",
      "loss 215.294845581 at run 800\n",
      "loss 214.721298218 at run 900\n",
      "CPU times: user 320 ms, sys: 48 ms, total: 368 ms\n",
      "Wall time: 500 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "W1 = Variable(torch.rand(4, 5), requires_grad = True)\n",
    "W2 = Variable(torch.rand(2, 4), requires_grad = True)\n",
    "\n",
    "M = 1000 # no of training examples\n",
    "\n",
    "x_in = Variable(torch.rand(5, M), requires_grad = False) \n",
    "y = Variable(torch.rand(2, M), requires_grad = False)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "\n",
    "for t in range(1000):\n",
    "    # forward pass\n",
    "    out = W2.mm(W1.mm(x_in).clamp(min = 0))\n",
    "    \n",
    "    # calculate loss\n",
    "    loss = (out - y).pow(2).sum()\n",
    "    \n",
    "    if t%100==0:\n",
    "        print(\"loss {0} at run {1}\".format(loss.data[0], t))\n",
    "    \n",
    "    # backprop\n",
    "    loss.backward()\n",
    "    \n",
    "    # param update\n",
    "    W1.data -= learning_rate * W1.grad.data\n",
    "    W2.data -= learning_rate * W2.grad.data\n",
    "    \n",
    "    # setting in graph grads zero\n",
    "    W1.grad.data.zero_()\n",
    "    W2.grad.data.zero_()\n",
    "    #repeat"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deeplearning",
   "language": "python",
   "name": "deeplearning"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}