{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Knet CNN Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# After installing and starting Julia run the following to install the required packages:\n",
    "# Pkg.init(); Pkg.update()\n",
    "# for p in (\"CUDAdrv\",\"IJulia\",\"Knet\"); Pkg.add(p); end\n",
    "# Pkg.checkout(\"Knet\",\"ilkarman\") # make sure we have the right Knet version\n",
    "# Pkg.build(\"Knet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Knet\n",
    "True=true # so we can read the python params\n",
    "include(\"common/params.py\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OS: Linux\n",
      "Julia: 0.6.1\n",
      "Knet: 0.8.5+\n",
      "GPU: Tesla K80\n",
      "\n"
     ]
    }
   ],
   "source": [
    "println(\"OS: \", Sys.KERNEL)\n",
    "println(\"Julia: \", VERSION)\n",
    "println(\"Knet: \", Pkg.installed(\"Knet\"))\n",
    "println(\"GPU: \", readstring(`nvidia-smi --query-gpu=name --format=csv,noheader`))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define model\n",
    "function initmodel(; atype=KnetArray, dtype=Float32, winit=xavier, binit=zeros)\n",
    "    w(dims...)=atype(winit(dtype,dims...))\n",
    "    b(dims...)=atype(binit(dtype,dims...))\n",
    "    return Any[\n",
    "        w(3,3,3,50), b(1,1,50,1),\n",
    "        w(3,3,50,50), b(1,1,50,1),\n",
    "        w(3,3,50,100), b(1,1,100,1),\n",
    "        w(3,3,100,100), b(1,1,100,1),\n",
    "        w(512,6400), b(512,1),\n",
    "        w(10,512), b(10,1)\n",
    "    ]\n",
    "end;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define loss and its gradient\n",
    "function predict(w,x)\n",
    "    convbias(x,w,b) = conv4(w,x;padding=1) .+ b\n",
    "    fc(x,w,b) = w * mat(x) .+ b;\n",
    "    x = relu.(convbias(x,w[1],w[2]))\n",
    "    x = relu.(pool(convbias(x,w[3],w[4])))\n",
    "    x = dropout(x,0.25)\n",
    "    x = relu.(convbias(x,w[5],w[6]))\n",
    "    x = relu.(pool(convbias(x,w[7],w[8])))\n",
    "    x = dropout(x,0.25)\n",
    "    x = relu.(fc(x,w[9],w[10]))\n",
    "    x = dropout(x,0.5)\n",
    "    return fc(x,w[11],w[12])\n",
    "end\n",
    "\n",
    "loss(w,x,y)=nll(predict(w,x),y) # nll: negative log likelihood\n",
    "lossgradient = grad(loss);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[36mINFO: \u001b[39m\u001b[22m\u001b[36mReading cifar-10-binary.tar.gz...\n",
      "\u001b[39m"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  3.525842 seconds (1.27 M allocations: 1.783 GiB, 18.79% gc time)\n",
      "32×32×3×50000 Array{Float32,4}\n",
      "50000-element Array{UInt8,1}\n",
      "32×32×3×10000 Array{Float32,4}\n",
      "10000-element Array{UInt8,1}\n"
     ]
    }
   ],
   "source": [
    "# load data\n",
    "include(Knet.dir(\"data\",\"cifar.jl\"))\n",
    "@time (xtrn,ytrn,xtst,ytst,lbls)=cifar10()\n",
    "for d in (xtrn,ytrn,xtst,ytst); println(summary(d)); end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare for training\n",
    "model = optim = nothing; knetgc() # Clear memory from last run\n",
    "model = initmodel()\n",
    "optim = optimizers(model, Momentum; lr=LR, gamma=MOMENTUM);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 25.437272 seconds (4.03 M allocations: 784.659 MiB, 11.86% gc time)\n"
     ]
    }
   ],
   "source": [
    "# cold start\n",
    "@time for (x,y) in minibatch(xtrn,ytrn,BATCHSIZE;shuffle=true,xtype=KnetArray)\n",
    "    grads = lossgradient(model, x, y)\n",
    "    update!(model, grads, optim)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare for training\n",
    "model = optim = nothing; knetgc() # Clear memory from last run\n",
    "model = initmodel()\n",
    "optim = optimizers(model, Momentum; lr=LR, gamma=MOMENTUM);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[36mINFO: \u001b[39m\u001b[22m\u001b[36mTraining...\n",
      "\u001b[39m"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 15.909965 seconds (1.88 M allocations: 670.408 MiB, 0.41% gc time)\n",
      " 15.772903 seconds (1.88 M allocations: 670.285 MiB, 0.42% gc time)\n",
      " 15.829510 seconds (1.88 M allocations: 670.285 MiB, 0.44% gc time)\n",
      " 15.961176 seconds (1.88 M allocations: 670.285 MiB, 0.43% gc time)\n",
      " 15.869710 seconds (1.88 M allocations: 670.285 MiB, 0.46% gc time)\n",
      " 15.872871 seconds (1.88 M allocations: 670.353 MiB, 0.46% gc time)\n",
      " 15.839494 seconds (1.88 M allocations: 670.285 MiB, 0.42% gc time)\n",
      " 16.007868 seconds (1.88 M allocations: 670.285 MiB, 0.47% gc time)\n",
      " 15.859198 seconds (1.88 M allocations: 670.285 MiB, 0.47% gc time)\n",
      " 15.870192 seconds (1.88 M allocations: 670.285 MiB, 0.43% gc time)\n",
      "158.797837 seconds (18.77 M allocations: 6.547 GiB, 0.44% gc time)\n"
     ]
    }
   ],
   "source": [
    "# 159s\n",
    "info(\"Training...\")\n",
    "@time for epoch in 1:EPOCHS\n",
    "    @time for (x,y) in minibatch(xtrn,ytrn,BATCHSIZE;shuffle=true,xtype=KnetArray)\n",
    "        grads = lossgradient(model, x, y)\n",
    "        update!(model, grads, optim)\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  2.123045 seconds (559.28 k allocations: 145.928 MiB, 1.10% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7754407051282052"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test accuracy 77.54\n",
    "testdata = minibatch(xtst,ytst,BATCHSIZE;xtype=KnetArray)\n",
    "@time accuracy(model,testdata,predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.6.1",
   "language": "julia",
   "name": "julia-0.6"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}