{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: GeForce GTX TITAN (CNMeM is disabled)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "import lasagne\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import gzip\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Seed for reproducibility\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2015-11-08 15:37:00--  http://deeplearning.net/data/mnist/mnist.pkl.gz\r\n",
      "Resolving deeplearning.net (deeplearning.net)... 132.204.26.28\r\n",
      "Connecting to deeplearning.net (deeplearning.net)|132.204.26.28|:80... connected.\r\n",
      "HTTP request sent, awaiting response... 200 OK\r\n",
      "Length: 16168813 (15M) [application/x-gzip]\r\n",
      "Server file no newer than local file ‘mnist.pkl.gz’ -- not retrieving.\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "# Download the MNIST digits dataset\n",
    "!wget -N http://deeplearning.net/data/mnist/mnist.pkl.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# Load training and test splits as numpy arrays\n",
    "train, val, test = pickle.load(gzip.open('mnist.pkl.gz'))\n",
    "\n",
    "X_train, y_train = train\n",
    "X_val, y_val = val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 784)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The original 28x28 pixel images are flattened into 784 dimensional feature vectors\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAArsAAABZCAYAAAA+cOwiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFO9JREFUeJzt3WeQFNXbxuEffxUTBkyIoqJEE0ZQgUIFIyoGFERFzBEU\nc0KMJGMJKCgGFCyBEgTBiKCCoJS5ygCCCmLCgFnBxPvB996zPbvDLrszPdO99/VlYcPsmbMzPT13\nP+c5tVasWIGZmZmZWRr9r9ADMDMzMzPLF5/smpmZmVlq+WTXzMzMzFLLJ7tmZmZmllo+2TUzMzOz\n1PLJrpmZmZmllk92zczMzCy1fLJrZmZmZqnlk10zMzMzSy2f7JqZmZlZaq2erxuuVatWjdiHeMWK\nFbUq+701YU48H1GejyjPR1mVnRPPR5TnI8rzEeX5iKrp8+Fk18zMzMxSyye7ZmZmZpZaPtk1MzMz\ns9Tyya6ZmZmZpZZPds3MzMwstXyya2ZmZmaplbfWY1Zc9thjDwB69uwJwMknnwzAI488AsCQIUMA\neOuttwowOjOz4nPXXXcBcMEFFwDw3nvvAXD44YcDsGjRosIMzKyGmDZtGgC1av3XUax9+/ZVuh0n\nu2ZmZmaWWolOdldbbTUANthgg3K/rhRznXXWAaBZs2YAnH/++QDcdtttAHTr1q3kZ5YtWwbAwIED\nAbjhhhtyPezY7LrrriX/njp1KgDrr78+ACtW/Ndfunv37gB06tQJgI033jjOIRa9Dh06APDoo48C\nsO+++wIwb968go0pTn369AHC8+B///vv/fF+++0HwMsvv1yQcVm81ltvPQDq1KkDwGGHHQbApptu\nCsAdd9wBwPLlywswutxr2LAhACeddBIA//77LwDbb789AM2bNwdqTrLbtGlTANZYYw0A2rVrB8A9\n99wDhPmpyKRJkwA4/vjjSz73559/5myccdN8tG7dGoD+/fsD0KZNm4KNKQ3uvPPOkn9rbnUVuqqc\n7JqZmZlZahV1srv11lsDULt2bSCc4bdt2xaADTfcEIDOnTtX6vY+//xzAAYPHgzA0UcfDcAvv/xS\n8j3vvvsukOzEqlWrVgCMHz++5HNKv5Xo6j7rXbUS3b333hsItbuFfNet9EBje+KJJ2IfQ8uWLQF4\n/fXXY//dhXTKKacAcMUVVwBlkxs9jiydlGzq77/PPvsAsNNOO5X7/fXr1wdCbWvSffvttwDMmDED\nCFe+aoodd9wRCMeB4447DghXdrbYYgsgHBcqezzQPA4fPrzkc7179wbg559/ruao46fX1RdffBGA\nr7/+GoDNN9888n+rHF1RP+ecc0o+99dffwGhdreqnOyamZmZWWr5ZNfMzMzMUqvoyhhKL6qaPn06\nkH0BWmXpUosW2/z6669AWHT01VdflXzvDz/8ACRrAZIW4O2+++4AjB49GgiXFsszf/58AG655RYA\nxowZA8CsWbOAMFcDBgzIw4grR4ugmjRpAsRbxqDLddtuuy0A22yzDRDan6Sd7u9aa61V4JHk1157\n7QWEhUhagKjLuHLppZcC8OWXXwKhlErPtTlz5uR/sHmkBVe6pHziiScCsPbaawPhcb948WIglEFp\nwVaXLl2AsGBp7ty5cQw7b3777Teg5ixAy6TjfseOHfNy+2p9CfDAAw8A4bUnyVS+4DKGqlEZpRb+\nAbzyyisAjBs3rlq37WTXzMzMzFKr6JLdzz77rOTf33//PVD5ZFfpyo8//gjA/vvvD4RFVqNGjcrZ\nOIvJvffeC0RbqFVEKbBaCWlBntLUFi1a5HCEVaN3/6+++mrsv1up+JlnngmEBC/piVVFDjjgAAB6\n9eoV+bzut5rpL1myJN6B5VjXrl2BsGnAJptsAoQE86WXXgJCa61bb7018vP6Pn29dCulJNAxddCg\nQUCYD7UYy6QrQQcffDAQkhc9LjR/+ph0Wvy8yy67FHgkhaFWlZnJ7jfffAOENFZXwDIXsGoxua6U\n1BQ15cpfNlpUfs011wDhnGTp0qUr/Tl9nxbAfvzxxyVf01W16nKya2ZmZmapVXTJbul3AJdddhkQ\n0qS3334bCK3D5J133gHgwAMPBEK9leruLrzwwjyOuHC0BbAavGe+qyzdPm3y5MlA2EhDtYeaU9Uq\nayu+YniHqtSgEO6///7I/5VspZVqUB966CGg7NUUJZtJrWFcffX/DnV77rknACNGjABCvbtaTN10\n001AqBNbc801gVAvdtBBB0Vu94033sjnsPNGbRfPOOOMlX6fEhYdW1Wz27hx4zyOrvD0uFD7y0xq\nSahkO6nPi2yGDRsGwMSJEyOfVxuoimpRtXmRtldWqzIpfbtJfQ6VRy3Y0r7WIZv77rsPCOtsdthh\nByAcT7O5+uqrgdBmVFdUIbSDrS4nu2ZmZmaWWkWX7Jamd3/qyqAVwKqjOv3004GQVirRlffffx+A\ns846K/+DjZE6VmTbAviZZ54BojW8qp1SlwUll2qerndPqr1SWqzaXm0yEQfVC9erVy+235kpM9nU\nXKdVjx49gLIJjGpXq7tVY6Gp20JmYq+/q2pWMxvb6/OZia42qHn44YdzP9gYaJOATAsXLgTCJira\nVEKJrqgLQ1rpytfIkSMBuP766yNf1/+1PmTo0KFxDS0Wf//9N1D2715Zqu2uW7duuV/X8wfSs8V0\nabqC9NprrxV4JPH6/fffgcon3DqXUfcfnX/kIxl3smtmZmZmqVXUya5kpi0//fRT5P+q7xg7dixQ\ndmVoWjRt2hQItcxKH7/77jsg9AtW2qR+wgBPPfVU5GNF1F/zkksuAULfzThoBbDGECelyeqvK198\n8UXsY8m30ivnTzvtNCA8d5RY3XzzzfEPLIdUg6uaMCUO6gerKx3ZtirVquJM2hZXV0aSRsdMXfV6\n/vnnAViwYAEQVt1nU8irLnHS4ycz2bXyqSuJHl/ZjuF9+/aNbUz5pARc5yR6TW7UqFHBxlQIep7s\nvPPOAHz44YdA9nrbddddFwhXjlQjryT88ccfz/kYneyamZmZWWolItnNpHfZ6kagelT1CFVKkRZa\nEa7aZCWfqmFWP1qtas1lIpptNXI+NWvWLPJ/1V7HQXOs5Oqjjz4CwlynQcOGDQEYP3581u8ZMmQI\nAC+++GIcQ8qp0qmREl312n7uueeAkCj88ccfkZ9VrZhqdPX4V3cSJd2TJk3Ky9jjoprUqiaW++yz\nTw5HU/yy9ZOt6XTF78orrwRCl47SO2CVps5J6uqQdLoCNnPmTCB0jqopttpqKyAk+Uq6e/bsCWS/\n8nXHHXcAYe2Ajkdt2rTJ21id7JqZmZlZaiUy2VXXBb2bUKcA9c5UGqWk8+677wZCvV7S7LbbbkDZ\n3WyOPPJIINpPN420MjyX1MHikEMOAcJq/cxV96pF0jv4NNB9Lm+XvGnTpgFhZ7Ek0a5X5513Xsnn\n9JxXonvUUUeV+7NKpB599FEgXDUS1ZDdcsstORxx8VJNsmrrMqk2T2bPng0UZrfDOCjRTepryKrS\n1Z/u3bsD4appJvXnzjYvqoVX8vv0008DZa+oWLJop7MnnngCCOs/dEUw2zmJdkM75ZRTIp/v169f\nPoYZ4WTXzMzMzFIrkcmuaHcfvUvQ7k96N6qPSifUK1RdC5JC9S2qG9S7pnwkusVYm7bRRhut9Ovq\nu6z5UQrRoEEDAGrXrg1EO0rofiphmDNnDhB6PmrHrTfffLP6d6BIKNUcOHBgma9phxv1283seJIE\n+juX7jIhSio322wzAE499VQAOnXqBISkok6dOkBIqvRx9OjRQNle3kmnVdDa6ei6664Dyl5FynZc\nUK2d5vOff/7J32At7/Q8ePLJJ4Hqr9lQLat21qoptBNYGui1EMIV0AceeAAoe1xQLf9VV10FhHMX\nvYarRlev1Tonu/fee/N3B/6fk10zMzMzS61EJ7uiupH58+cD4d1Ehw4dAOjfvz8QdulQfUix907V\nyk7tMqKUSe+68yGzNk2rZ+OktFVjGD58OBBW1mdS7aneLWpFqHZz+eCDDwB48MEHS35G9dxKx5cs\nWQKEnX3U0WLu3LnVvj+FVpnuC5988gkQ5iGJ1HGh9ArgTTfdFIBPP/0UyF5bqIRSNYb169cHQg/r\nyZMn52HE8dMqea0D0GNC91fPPc2HanBV560kWJT6HHPMMUCo9dbfwpJJx1J9zKaiK4F6DTv00EOB\nsLtn2umKURqodzKEHSh1HNXfXf25tXOcPmpd0ZZbbgmE44yO0ervHgcnu2ZmZmaWWqlIduW9994D\noEuXLgAcccQRQKjlPfvsswFo0qQJAAceeGDcQ1wlShdVi6hdjbRTXC6oh29mv83p06cDofYmTlpN\nv2jRIgBat2690u//7LPPAJg4cSIQdm9ZlX3JtZOUkkAlnWmgnrIrq8Mur443adQxo3THhSlTpgCh\nZkx1/uqTO3LkSACWLl0KwJgxY4CQQOj/SabjB4SEdsKECZHvueGGG4DwvJ81axYQ5k2fV02n6Pky\nYMAAoOxzEUIdfJJlSzDbtWsHwNChQ2MfUz7oNXS//fYDQo2mupksW7ZspT9/+umnA9CrV688jbA4\nqQNUmvrsdu3aFQjnTxD6I+tYe8IJJwDwww8/AHD77bcDYe8DJby6QqBEWOsqFi9eDITHm47P+eBk\n18zMzMxSK1XJruhdx6hRo4BQZ6L6Mr0b17uJl156Kd4BVpESklx0k1Ci26dPHwAuu+wyINSs6h3a\nr7/+Wu3fVVWDBg2K7XepvltWVt+aFKr1zuwdLKV3AZs3b14sY4qDOmtASB4romOCEgkleElO+FWf\nq9QWwvNcVEOp/pg6dmre1BdVfXVVi6t+w0p6VZunPsUvvPBCye/Q81jpjxRiPUBVZeuzq1pldbPQ\n+oCk01W1Ve1/qiuENS3Z1RUN0XNP64Q0n0miK+Gl75t2kCyd9pamv7u6K2TbaVFJrxLxfCa64mTX\nzMzMzFIrVcmuVuUfe+yxALRs2RKI9omD8O57xowZMY6u+nLRhUFpnxIe1eUo5evcuXO1f0caqMNH\nkj3//PMA1K1bN/J51TJn7mJTk6k+PjPBS2LN7mqrrQaE3f+0axGEPsHa0Ur3T4muauxUg6quDep0\nc+655wIhkdFOhKqrVy/r0qvRp06dGhmf6vS23XbbKt/HuKkjjNKuTKr57927d2xjKkYHH3xwoYdQ\nEOoAJEoudQU1iXROULq+X8/dbFSLm1nb361bNyDUhIuuJMfBya6ZmZmZpVaik91mzZoB0LNnTyDU\nT22++eblfr9291HNazHtElaezF6HWmV+4YUXrvJtXXTRRQBce+21AGywwQZAqLE7+eSTqzdYKzra\nxSfzcX7PPfcAha3HLjZabZ4GShmV6KrfNIRkUqn/3nvvDYQd0NQPVUn3jTfeCIQavcxkR32Jn332\n2chHJTkQVmyLjkVJkoZ+25lUV1q6pl9dN9RvubL0+FGf5ZpGKageJ82bNwdC0q8OQ0myKn9LnU9o\nhzRd8VEt7rhx43I8ulXnZNfMzMzMUitRya4SW6UGSnS1Q1Q22i1LK0vzuQNZLqluUB91/wcPHgyE\nHcG+//57IKQ03bt3B2CXXXYpua0GDRoAYWWlkiylfPYfpehNmzYFVq1Xb7FQCqfeoJlmz54d53AS\nIU21hn379o38XzW8EGr1tWq+cePG5d6Gvq7+uboqVlmPPfZYuf9OKnWr0GrzRo0aRb6uq236vjhW\nl1dV27ZtAbjmmmuAaL951VFXVJup/ssdO3YEwq6lmTvsKSGuqD9vWuiKiXYMu/jiiws5nNgouVZN\nv/YEaN++fcHGlMnJrpmZmZmllk92zczMzCy1irqMoV69ekBo2K12OCr+zkZN5W+99VYgFI8X+4K0\niuhypC4ZqE2YFoloG+Ty6NK1WgZlXuq0/6hkJFsJQDFTW7kDDjgACI93bQRw9913A7BkyZICjK64\nbbfddoUeQs58/fXXQNgYonT7o9KlTRA2jVAbRm3zu3DhQmDVyxfS7v333wfKPl6S9Nqi19HM9lAA\nl19+OQC//PLLSm9DpQ+77747UHazDW3UNGzYMCC87tQUmg8de9NKm2acccYZQLjf9913HxBva7GK\nJO8V3czMzMyskoom2VXBu7aZg5BUVZS6KLXUFrdafLWq7VOKzauvvgrA66+/DoRNMkQL1pSAixas\nlW6IX5V2ZTWZtjkcOXJkYQeyCjbccEOgbOu9L774AohuLmBRM2fOBEKin6SkLpO2PlarQqVvEBaO\naHGrtvBNewKVK0qsjjjiiAKPJD+0wGhV6XE1efJkILze1JSFaZnUekvbaKdhk6LyaMMYJbyjR48G\n4LrrrivYmLJxsmtmZmZmqVWwZHevvfYCQiucVq1aAaFlx8qoSbpacPXv3x8IW2GmhepdtFmGGsL3\n6dOn3O9XE2jVSS1YsCDfQ0wdtR6zmkXbWGpbXF1NUoupb7/9tjADqwLVW44aNSry0apPW81/+OGH\nAGy//faFHE6VaJtwtVHr0aNHpX9WLdX0GqwrIkq8M7eDrWm6dOkCwPLly4HwOEkrtbnU1uRaH1WM\nnOyamZmZWWrVylxFmbMbrlVrpTc8cOBAICS75dG76ClTpgDw999/A6E298cff6z+QKtpxYoVlY4C\nK5qTNEjqfCjtUC3jiBEjgJCmV1Wc86Fa3bFjxwKhefynn34KZN9AIE7F/vjQ4+D+++8H4OWXXwZC\nCqZjUi5Vdk6K6fmST56PqHzNh7p06DEPcPPNNwNQt25dIHTnUG2mkjt1/CiEYn58aJ2MEv9OnToB\nsGjRorz9zmKej0LINh9Ods3MzMwstQqW7KZFsSdVcfN8RHk+oop9PrSKety4cUDoWTxhwgQATj31\nVCC36wOczER5PqI8H1GejyjPR5STXTMzMzOrcZzsVlOxJ1Vx83xEeT6ikjIfSnj79esHhP6jLVq0\nAHJbu+tkJsrzEeX5iPJ8RHk+opzsmpmZmVmN42S3mpKSVMXF8xHl+YjyfJTlZCbK8xHl+YjyfER5\nPqKc7JqZmZlZjZO3ZNfMzMzMrNCc7JqZmZlZavlk18zMzMxSyye7ZmZmZpZaPtk1MzMzs9Tyya6Z\nmZmZpZZPds3MzMwstXyya2ZmZmap5ZNdMzMzM0stn+yamZmZWWr5ZNfMzMzMUssnu2ZmZmaWWj7Z\nNTMzM7PU8smumZmZmaWWT3bNzMzMLLV8smtmZmZmqeWTXTMzMzNLLZ/smpmZmVlq+WTXzMzMzFLL\nJ7tmZmZmllo+2TUzMzOz1Po/Fv0oWLHGMx0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb397b0e3d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the first few examples \n",
    "plt.figure(figsize=(12,3))\n",
    "for i in range(10):\n",
    "    plt.subplot(1, 10, i+1)\n",
    "    plt.imshow(X_train[i].reshape((28, 28)), cmap='gray', interpolation='nearest')\n",
    "    plt.axis('off')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# For training, we want to sample examples at random in small batches\n",
    "def batch_gen(X, y, N):\n",
    "    while True:\n",
    "        idx = np.random.choice(len(y), N)\n",
    "        yield X[idx].astype('float32'), y[idx].astype('int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# A very simple network, a single layer with one neuron per target class.\n",
    "# Using the softmax activation function gives us a probability distribution at the output.\n",
    "l_in = lasagne.layers.InputLayer((None, 784))\n",
    "l_out = lasagne.layers.DenseLayer(\n",
    "    l_in,\n",
    "    num_units=10,\n",
    "    nonlinearity=lasagne.nonlinearities.softmax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# Symbolic variables for our input features and targets\n",
    "X_sym = T.matrix()\n",
    "y_sym = T.ivector()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "# Theano expressions for the output distribution and predicted class\n",
    "output = lasagne.layers.get_output(l_out, X_sym)\n",
    "pred = output.argmax(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "# The loss function is cross-entropy averaged over a minibatch, we also compute accuracy as an evaluation metric\n",
    "loss = T.mean(lasagne.objectives.categorical_crossentropy(output, y_sym))\n",
    "acc = T.mean(T.eq(pred, y_sym))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[W, b]\n"
     ]
    }
   ],
   "source": [
    "# We retrieve all the trainable parameters in our network - a single weight matrix and bias vector\n",
    "params = lasagne.layers.get_all_params(l_out)\n",
    "print(params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([(W, Elemwise{sub,no_inplace}.0), (b, Elemwise{sub,no_inplace}.0)])\n"
     ]
    }
   ],
   "source": [
    "# Compute the gradient of the loss function with respect to the parameters.\n",
    "# The stochastic gradient descent algorithm produces updates for each param\n",
    "grad = T.grad(loss, params)\n",
    "updates = lasagne.updates.sgd(grad, params, learning_rate=0.05)\n",
    "print(updates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# We define a training function that will compute the loss and accuracy, and take a single optimization step\n",
    "f_train = theano.function([X_sym, y_sym], [loss, acc], updates=updates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "# The validation function is similar, but does not update the parameters\n",
    "f_val = theano.function([X_sym, y_sym], [loss, acc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "# The prediction function doesn't require targets, and outputs only the predicted class values\n",
    "f_predict = theano.function([X_sym], pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We'll choose a batch size, and calculate the number of batches in an \"epoch\"\n",
    "# (approximately one pass through the data).\n",
    "BATCH_SIZE = 64\n",
    "N_BATCHES = len(X_train) // BATCH_SIZE\n",
    "N_VAL_BATCHES = len(X_val) // BATCH_SIZE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Minibatch generators for the training and validation sets\n",
    "train_batches = batch_gen(X_train, y_train, BATCH_SIZE)\n",
    "val_batches = batch_gen(X_val, y_val, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPwAAAD8CAYAAABTq8lnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADP9JREFUeJzt3WuMFfUZx/HfA0IUJVpjXUG2Qrw0TUMC0ZAa22AaihDi\nLUbJJtWVeCVKpTFR7IvKG+MlaqRvjI0ISFEoXihtUi+Yxnq/NAhaVxQVgwYXGgTZaILWpy92SNcF\n/rN7ZuacWZ7vJyGcM885Mw8DP+bM/ufM39xdAGIY1uoGADQPgQcCIfBAIAQeCITAA4EQeCCQhgNv\nZjPM7D0z+8DMbi6zKQDVsEbG4c1suKRNkqZJ+kzSG5I63L2rz2sY4AdayN2t/7JGj/BTJG129y3u\n/o2klZLOL9IcgOo1GvgTJW3t8/zTbBmAGms08HxcB4agRgP/maT2Ps/b1XuUB1BjjQb+TUmnmtl4\nMxspabakteW1BaAKhzXyJnf/1syul/S0pOGSFvf9CT2AempoWG5AK2ZYDmipMoflAAxBBB4IhMAD\ngRB4IBACDwRC4IFACDwQCIEHAiHwQCAEHgiEwAOBEHggEAIPBELggUAIPBAIgQcCIfBAIAQeCITA\nA4EQeCAQAg8EQuCBQAg8EAiBBwIh8EAgBB4IhMADgRB4IBACDwRC4IFACDwQyGFF3mxmWyR9Kem/\nkr5x9yllNIXWu/7663Nf4+5N6OTgxo0bl6zPnTu30u0//fTTyfrs2bMr3X4jCgVekks62913ltEM\ngGqV8ZHeSlgHgCYoGniXtM7M3jSzq8poCEB1in6kP8vdt5nZDyU9a2bvufsLZTQGoHyFjvDuvi37\nfYekJyXxQzugxhoOvJmNMrPR2eMjJU2X9HZZjQEoX5GP9G2SnjSzfetZ4e7PlNIVgEo0HHh3/1jS\npBJ7QYlGjBiRrHd0dCTrixYtyt1Gq8fhW23ChAmtbmHQuNIOCITAA4EQeCAQAg8EQuCBQAg8EAiB\nBwIpei09amrYsPT/5Z2dnYW38dFHHyXrGzduLLT+zZs3J+tLliwptP6LL744WZ80KX2ZyYoVKwpt\nvxU4wgOBEHggEAIPBELggUAIPBAIgQcCIfBAIFbVd5rNLPaXpSt29NFHJ+tLly5N1s8999xk/ZNP\nPsnt4ZxzzknW88bRUS133++O0hzhgUAIPBAIgQcCIfBAIAQeCITAA4EQeCAQvg8/ROXdd37y5MmF\n1j99+vTc13z44YeFtoHm4wgPBELggUAIPBAIgQcCIfBAIAQeCITAA4HkjsOb2UOSZkna7u4Ts2XH\nSlol6SRJWyRd4u67KuwznHHjxiXrc+fOTdbHjh2brHd1dSXrjLEfmgZyhF8iaUa/ZQskPevup0l6\nLnsOoOZyA+/uL0j6ot/i8yQtyx4vk3RByX0BqECj5/Bt7t6dPe6W1FZSPwAqVPiHdt57UzzuXwcM\nAY0GvtvMTpAkMxsjaXt5LQGoSqOBXytp3/SjnZLWlNMOgCrlBt7MHpX0sqQfm9lWM5sj6Q5JvzKz\n9yX9MnsOoOa4L31NdXR0JOvLly9P1vfu3Zusz5w5M1l//vnnk3XUH/elB4Ij8EAgBB4IhMADgRB4\nIBACDwRC4IFAuC99TbW3txd6/+rVq5P1MsbZR48enayb7TcMXKqvvvoqWf/2228r3f5QxBEeCITA\nA4EQeCAQAg8EQuCBQAg8EAiBBwJhHL5FrrjiimR9zpw5hdZ//PHHJ+u33XZbsj6QMfT58+cn6yNH\njsxdRxF33313sr5gAXdP748jPBAIgQcCIfBAIAQeCITAA4EQeCAQAg8Ewn3pW+Smm25K1m+//fYm\ndXJgw4blHwu+++67JnTSuB07diTr06ZNS9bfeeedMttpOu5LDwRH4IFACDwQCIEHAiHwQCAEHgiE\nwAOB5I7Dm9lDkmZJ2u7uE7NlCyVdKWnfQOct7v5Uv/cxDp9wxBFHJOsrV65M1mfNmlVo+6+88kqy\nvn79+tx15P3bufPOO5P1PXv2JOtTp05N1tesWZOs53nkkUeS9csuu6zQ+lut0XH4JZJm9F+XpHvd\nfXL266kDvA9AzeQG3t1fkPTFAUrVTisCoHRFzuHnmdkGM1tsZseU1hGAyjQa+PslTZA0SdI2SfeU\n1hGAyjQUeHff7hlJD0qaUm5bAKrQUODNbEyfpxdKerucdgBUKfc21Wb2qKSpko4zs62SbpV0tplN\nUu9P6z+WdE2lXQIoRW7g3b3jAIsfqqCXUL7++utkPe++9Icffnih7X/55ZfJek9PT6H1lyFvH+XZ\nvXt3sv7AAw8UWv9QxJV2QCAEHgiEwAOBEHggEAIPBELggUAIPBAI88PX1M6dO1vdQuXGjh2brC9c\nuLDQ+tetW5esv/TSS4XWPxRxhAcCIfBAIAQeCITAA4EQeCAQAg8EQuCBQA7ZcfjLL788Wb/22muT\n9fnz5yfrr7766mBbCmfMmDHJ+qpVq5L1M888M1nP+07/okWLkvWIOMIDgRB4IBACDwRC4IFACDwQ\nCIEHAiHwQCCH7Dj86aefnqyfccYZyfrq1auT9a6urmR93rx5yfqmTZuS9aGgra0tWb/mmvT8JEXH\n2Z944olk/eWXX07WI+IIDwRC4IFACDwQCIEHAiHwQCAEHgiEwAOBJMfhzaxd0sOSjpfkkv7o7n8w\ns2MlrZJ0kqQtki5x910V9zoop5xySqH3532XO68+ceLEZN3MkvW9e/cm61u3bk3WTz755GS9DEuX\nLk3W86512LNnT7KeN85+5ZVXJuvYX94R/htJv3X3n0r6maTrzOwnkhZIetbdT5P0XPYcQM0lA+/u\nn7v7W9njHkldkk6UdJ6kZdnLlkm6oMomAZRjwOfwZjZe0mRJr0lqc/furNQtKX2NJYBaGFDgzewo\nSY9LusHdv3fi5e6u3vN7ADWXG3gzG6HesC939zXZ4m4zOyGrj5G0vboWAZQlGXjr/VHyYknvuvt9\nfUprJXVmjzslren/XgD1k/f12LMk/VrSRjNbny27RdIdkv5sZlcoG5arrEMApbHeU/AKVmzW0vP6\nSy+9NFmfMmVKsn711Vcn68OHDx90T4Oxe/fuZD1v7vOLLrqo0PbzrhOQpJ6enmR91670pRk33nhj\nsv7YY4/l9oCDc/f9/hK50g4IhMADgRB4IBACDwRC4IFACDwQCIEHAjlkx+GL6uzsTNZHjRrVpE5a\nY9iw/GPBhg0bkvUXX3yxrHbQAMbhgeAIPBAIgQcCIfBAIAQeCITAA4EQeCAQxuGBQxTj8EBwBB4I\nhMADgRB4IBACDwRC4IFACDwQCIEHAiHwQCAEHgiEwAOBEHggEAIPBELggUCSgTezdjP7h5n928ze\nMbPfZMsXmtmnZrY++zWjOe0CKCL5fXgzO0HSCe7+lpkdJelfki6QdImkPe5+b+K9fB8eaKEDfR/+\nsJw3fC7p8+xxj5l1SToxK++3MgD1NuBzeDMbL2mypFezRfPMbIOZLTazYyroDUDJBhT47OP8Y5Ju\ncPceSfdLmiBpkqRtku6prEMApcm9p52ZjZD0N0l/d/f7DlAfL+mv7j6x33LO4YEWGvQ97czMJC2W\n9G7fsJvZmD4vu1DS22U1CaA6eT+l/7mkf0raKGnfC38nqUO9H+dd0seSrnH37n7v5QgPtNCBjvDc\npho4RHGbaiA4Ag8EQuCBQAg8EAiBBwIh8EAgBB4IhMADgRB4IBACDwRC4IFACDwQCIEHAiHwQCAE\nHgiEwAOBEHggkMrueAOgfjjCA4EQeCCQpgTezGaY2Xtm9oGZ3dyMbQ6GmW0xs43ZxJiv16Cfh8ys\n28ze7rPsWDN71szeN7NnWjnbz0H6q8UEo4kJUGux/1o9QWvl5/BmNlzSJknTJH0m6Q1JHe7eVemG\nB8HMPpZ0urvvbHUvkmRmv5DUI+nhfRN8mNldkv7j7ndl/2n+wN0X1Ki/W5UzwWiTejvYBKhzVIP9\nV2SC1jI04wg/RdJmd9/i7t9IWinp/CZsd7BqMzmmu78g6Yt+i8+TtCx7vEy9/0ha4iD9STXYh+7+\nubu/lT3ukbRvAtRa7L9Ef1IT9l8zAn+ipK19nn+q//8B68IlrTOzN83sqlY3cxBtfSb76JbU1spm\nDqJWE4z2mQD1NdVw/7VigtZmBH4ojPud5e6TJc2UdF32kbW2vPc8rG77tVYTjGYflx9X7wSoe/rW\n6rD/WjVBazMC/5mk9j7P29V7lK8Nd9+W/b5D0pPqPQ2pm+7s/G/f3H7bW9zP97j7ds9IelAt3IfZ\nBKiPS1ru7muyxbXZf336+9O+/pq1/5oR+DclnWpm481spKTZktY2YbsDYmajzGx09vhISdNVz8kx\n10rqzB53SlqTeG3T1WWC0YNNgKqa7L9WT9DalCvtzGympPskDZe02N1vr3yjA2RmE9R7VJekwySt\naHV/ZvaopKmSjlPv+ebvJf1F0p8l/UjSFkmXuPuumvR3q6SzlTPBaJN6O9AEqLdIel012H9FJmgt\nZftcWgvEwZV2QCAEHgiEwAOBEHggEAIPBELggUAIPBAIgQcC+R8s56mYXXoOYAAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb3930124d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Try sampling from the batch generator.\n",
    "# Plot an image and corresponding label to verify they match.\n",
    "X, y = next(train_batches)\n",
    "plt.imshow(X[0].reshape((28, 28)), cmap='gray', interpolation='nearest')\n",
    "print(y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, Train (val) loss 0.621 (0.379) ratio 0.610\n",
      "Train (val) accuracy 0.844 (0.900)\n",
      "Epoch 1, Train (val) loss 0.389 (0.346) ratio 0.890\n",
      "Train (val) accuracy 0.895 (0.907)\n",
      "Epoch 2, Train (val) loss 0.356 (0.320) ratio 0.901\n",
      "Train (val) accuracy 0.900 (0.913)\n",
      "Epoch 3, Train (val) loss 0.340 (0.303) ratio 0.893\n",
      "Train (val) accuracy 0.904 (0.913)\n",
      "Epoch 4, Train (val) loss 0.329 (0.299) ratio 0.909\n",
      "Train (val) accuracy 0.909 (0.916)\n",
      "Epoch 5, Train (val) loss 0.320 (0.299) ratio 0.935\n",
      "Train (val) accuracy 0.911 (0.919)\n",
      "Epoch 6, Train (val) loss 0.308 (0.286) ratio 0.929\n",
      "Train (val) accuracy 0.914 (0.919)\n",
      "Epoch 7, Train (val) loss 0.307 (0.302) ratio 0.985\n",
      "Train (val) accuracy 0.914 (0.918)\n",
      "Epoch 8, Train (val) loss 0.304 (0.286) ratio 0.941\n",
      "Train (val) accuracy 0.915 (0.921)\n",
      "Epoch 9, Train (val) loss 0.299 (0.281) ratio 0.940\n",
      "Train (val) accuracy 0.917 (0.922)\n"
     ]
    }
   ],
   "source": [
    "# For each epoch, we call the training function N_BATCHES times,\n",
    "# accumulating an estimate of the training loss and accuracy.\n",
    "# Then we do the same thing for the validation set.\n",
    "# Plotting the ratio of val to train loss can help recognize overfitting.\n",
    "for epoch in range(10):\n",
    "    train_loss = 0\n",
    "    train_acc = 0\n",
    "    for _ in range(N_BATCHES):\n",
    "        X, y = next(train_batches)\n",
    "        loss, acc = f_train(X, y)\n",
    "        train_loss += loss\n",
    "        train_acc += acc\n",
    "    train_loss /= N_BATCHES\n",
    "    train_acc /= N_BATCHES\n",
    "\n",
    "    val_loss = 0\n",
    "    val_acc = 0\n",
    "    for _ in range(N_VAL_BATCHES):\n",
    "        X, y = next(val_batches)\n",
    "        loss, acc = f_val(X, y)\n",
    "        val_loss += loss\n",
    "        val_acc += acc\n",
    "    val_loss /= N_VAL_BATCHES\n",
    "    val_acc /= N_VAL_BATCHES\n",
    "    \n",
    "    print('Epoch {}, Train (val) loss {:.03f} ({:.03f}) ratio {:.03f}'.format(\n",
    "            epoch, train_loss, val_loss, val_loss/train_loss))\n",
    "    print('Train (val) accuracy {:.03f} ({:.03f})'.format(train_acc, val_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(784, 10)\n"
     ]
    }
   ],
   "source": [
    "# We can retrieve the value of the trained weight matrix from the output layer.\n",
    "# It can be interpreted as a collection of images, one per class\n",
    "weights = l_out.W.get_value()\n",
    "print(weights.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAArsAAABZCAYAAAA+cOwiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3UezvktVNvALxYA5YA6YURDxeBDxBAExlUVhQKTkAzBi\noE6YGD6EDnDiwInlQMuyKMCAHEE4Bw5BVDBnRTEB5nzegfX7d+9r781+7n+9u956n1rXZO8n9d29\neq113+vq1asf99hjj2UwGAwGg8FgMDhHfMj/6w4MBoPBYDAYDAa3hXnYHQwGg8FgMBicLeZhdzAY\nDAaDwWBwtpiH3cFgMBgMBoPB2WIedgeDwWAwGAwGZ4t52B0MBoPBYDAYnC3mYXcwGAwGg8FgcLaY\nh93BYDAYDAaDwdliHnYHg8FgMBgMBmeLedgdDAaDwWAwGJwtHn9bDf/Ij/zIY0nyvve9L0ny2Z/9\n2Rc+f+9735skefKTn5wk+Yu/+IskySd+4ife+c6HfdiHJUn+8A//MEnyX//1X0mSj/mYj0mSfPqn\nf3qS5O/+7u+SJJ/5mZ+ZJPnv//7vJMlf/uVfJkk+/uM/Pkny/ve/P0nyz//8z0mSz/iMz7jQ7n7N\nD//wD0+SfMRHfESS5E//9E8vtPWEJzwhSfI93/M9j7tZGv+L7/u+73ssSR73uP/9yR//8R9f+Pxj\nP/ZjkyQf/dEffWFcxvMJn/AJSZLHP/7xF973vc/5nM9JkrznPe/Jfp0d+v9Xf/VXF9r63M/93CTJ\nn/zJnyRJPu3TPi1J8p//+Z9Jkv/5n/9JsubH/JmD//iP/0iS/OiP/ujJ8vjJn/zJx5Ilczpgflzz\naU972oW+/NZv/VaSJY9P+ZRPSZL89V//dZLkSU96UpI1Z//yL/+SJPmkT/qkO9eml93/Jz7xiUkS\nx2j//u///oXv6SP5kIe5+IM/+IMkyRd90RclSb7zO7/zZHm84hWveCxZsqVjX/iFX3hBHvpGf+j9\nb/7mbyZZc6lP+vjud787ydIB7SVL5z7kQ/43/v37v//7JMs+P/CBDyRZMv/t3/7tC31kJ//+7/+e\nZNkUnSavF73oRSfL4/u///svnGXebdFV4/mHf/iHJFfPs3ll1//0T/90YZx05s///M8vvDZOvkJ7\nH/qhH5pk+SI6nCR/+7d/m2TJ/9/+7d8uXPvzPu/zkix/9EM/9EMnyeQlL3nJY8mao0/91E/N/poO\nf9RHfdQFeZibZM2jvvzrv/5rkiVT4zN/bEcbvse3kjV5/uM//mOSpRc7PvmTPzlJ8pEf+ZEX+qJt\nfXnFK15xkjxe/vKXP5asOeAf9rlIlj8hJ3ObLBswTm0YFx9Klvyl+ScvNmYs5McHsa9k2Sf7NRd8\nDFl6/6UvfelJ8vjxH//xx/a+QvfRHLnen/3Zn935Lp2is2yOT/qsz/qsJEvXfI/O+Z45Jo+P+7iP\nS7LmYp8j/aUPX/IlX3Lhu8CeXvayl50kjx/8wR98bG/HHLTu932Yr0iSL/iCL0iSPPLII0mWPJ76\n1KcmSR5++OEkyy7YPp0k6/vuuy/Jujf7vt9/2Zd92Z1r8rH8g/74Lf0wVz/wAz9wSB76SC5tw+TT\nPjBZOkSn2I++mCPjI1tjoDd/8zd/k2TpAf/KXj7/8z//zjXZpt/ol3stX0Off/iHf/hKedzaw64L\nmzATRbAG7+ZCwLuCE4ybO0fjZtE3XIZHYH7nWoTGEfl8f9hlnNpiEMbjcwpzBB4wjZHzd8PtGwSH\nwVHokwcuDzkcDQgsPHhc1Sal5LT9paQ+bxn1Qy5HddXN7SZ4SHdNjoSDoNS/8Ru/kWTJ3vybzz/6\noz9KsuTBeRurm4uHvWQFBtpgtOabIXmQ1leypsduoHTTX3p/BPSWs+NY3HB/53d+50Kf6fXv/u7v\nJlnzTk4cLjv68i//8iTrpuLGnizddE3Ol57TVeMiL07KuOkLZ6gvHNURkDHZ93jMIbl46CXHZD2s\nt4yMo/tLn+kcp8tPffEXf3GSZbPk4EaRLBmyU4EPm2L/5vlUsMF+aKQPbJg99A0oWfrKN9BvvyVz\n8nAt/tk8mwvj5g+0t893+y3ycS0+RR9Ohb6Zd32h1/rML1z1cOOhhSw9xPNnxrHbyt7npzzlKUmW\nvvBh5gT48OTy/QzMr3vlUfCb5NAP5v7yG+zEnCXrgZA/MA5+j180HjrGDvhyOk4e7Mr7dCJZ/pk9\nsDVt0hN9OhV8lz4aG7KEHvYDPHtNkne84x1JknvvvTfJmiO65CGWTzW35ICA8VCrD6754IMPXtt/\n/f7VX/3VJJfv90ftxfj0XTtea8+c85G7ftB7v2X3fAyd4h/6mQz4If7VXAku9mBIv+mD+6E2yHrX\nqaswaQyDwWAwGAwGg7PFrTG7ooWO/ERAns49tWOARArJiuR6yVBkhtnxPQyQyFX0IQ0CA6APfrcv\n5/qNiEw0JfIXPfRS0SkQFWKXtCkCbzalGSCRu0gX+yAKwkqQ084ukmsz7GQm4uxoTh+wJD7HdJDh\nvnR8Knp5wjjIRRQp8idzc4DBwC5CR3jktkeoInBtNPtvPD1efaMf5NKpB3fD/LuG+TY3WER903d2\nwcaM01xa+aBPdPsqtsp3/DWvroXRIWvXwNY1S6lPWDR9OwLjbSbfHOgTtvUqm+QrMFFYYDaC3fE9\nbIHvm1fvY4jJBaOBkUiWTHxm/lrH2PupIFNsGQasV1WwJXR0ZxDNL6ZFX62i0HfjNc/GixHWB36c\nzRnbzmxqWxv0zrWMi76fCvKVzsKP86/Gbw5dB6t2Vb/5CPcIfae/5GLlw7Wf/vSnX+ib+xn5+F6y\nZMivuS/QB3a69/MUkKvxsz19dk/u+9DOwtMV8+s3ZKtvdE5b/KA5YV9tk502llxeBXLP1S+y16dT\n8aVf+qVJlo6TB7/pfsMmsLj7iqiVIfcJ83jPPfdcGB9f2ytZ5OkabJPeeTbZ72GddtL3bit0neZx\nE3yf/zQH5pA+0hd2Y273cRiXPtFVv2XbWFfvmwv6wzfSq179SpYvxrj/3u/93oU22KpxXYdhdgeD\nwWAwGAwGZ4tbY3Y7qsTOiCJET83O7oyARHVsSrNuNg91JChi23M0k/Xkj3UTMYlWksusmci8NyDZ\nNHQEIjXX0A+ywDy7tvcxtqKp3tDVeaOi6p3R0f+OfkTuWBBMjr5hOlzL/IlMzdG+yeFUNOPQm2BE\n0c1Imi+RKMYGs4GN6cT1d77znXeuTbZf/dVfnWRFh+Tg2qJaUaVo2Ofa9pfse9PDKfjar/3aJEvG\nvVEPsADGR37YRXKSj0iexkjPyD9ZNkTfbXb7iq/4iiTJ6173uiRrjugFdhGz8TVf8zVJls6y+6P5\ndsnSMbplvHTReI3f93d7JkPzav4wD/JbyYw+0/+WS48bw7ezq5iIzmejM/p9lMmks/ruOs3GGjO2\nBXuSLP/FD/hu59SxpdZF7HRvKvtgeYT0Vz/J0m8xd3te6ymgz9pnq+RgrjBl7ht0YR+nNtgvn2L8\nvTGZnzT/mD9sIv0wZn4mWWye8ZOHfrqG+96pYOd0mRx6lYpeYO72zeN0hQ3RXfda8mGD5sz4tWVs\nvq8drNy+Eb03kLUuGldvPLwJWEf3Qez7W97ylguf89VWp9xXkqVj7Aaryh/2JlF+76u+6quSJL/2\na7+WZD3nYI/5l/vvvz/JxVXYvsd85Vd+5YVr0dmjq4edj0136SO96dWbnWUlS/OuTd/pDZrGQq/M\noT60vXjG2+eAHNzXtUHm5sjf6zDM7mAwGAwGg8HgbHFrzK6oAdsiIhAhezoX1XeFhf03osOO8LwP\nIhOshWhK1C6aEH2IRneGS4QiihB1Y2ObVT0CUZAoR/QjYhPdyOvBluiv/ouqescvGRrvXkIFRNzY\nMr8ha33zvpwkfcSKdPmSPQfrVBgnBmMvkZRcrkKhT8agryJyDAYdw2CIrnfmyHjpTO8mNV5s0aOP\nPnqhTXqgHXPVZeGOwHyKtDFT9Bmja04wtyJ9f+0cNgY29fznPz/Jyk/d5Y25MJ9WLugcxvPNb35z\nkmVb4Fr0gg669tHKA3ubbNB8Y1vZi++Ry15dhY/o6i9s3ued34+VxX4Zb5fqolO7D/FdbM11TETb\n701on4l56dUacmCT+wpP55gbB5n13gTsc5fW6pxWetLySi7nNZNxr+rt5bmOgG7SOTbMX2CbyYX+\nJMtm6IU+8HNWB8manrgWG8WE9gpZ5zQny4eYF6sLdPGqlZdT0OUk+UN9NlbXM4bdV3XFIv7RvLvv\n6WuvtpFXr7KZA/Lay2L2/bjtp1c2TwWZa9893OqTCglWrzC8e2UV77ET9t8+1qohH9vMpfHTCzIn\n730VWtvui+49fKg84q4QchPYLN/WlSXaV/IVe941G7b3oUvzaaP344DXXXLWX3O/r2roh7bNI1ny\nizf5j2F2B4PBYDAYDAZni1tjdkXvotM+rKEjJWzFnssDXagcwyVKEA0873nPu/A97KU29934ycpD\n2lkIUSSGC1Mlcml28QjknohMuj5kVwi4LgoUkYkWm20g4z2qMvYueN8yJKuuXYclEqkbi3ndC7Wf\nCgxj57mKlvUFeyZa/rqv+7oL19Y3eiKaFn127eBkMVhYHjL1+u1vf3uSlUdn3ruGZde2lNt7Nznd\nPc90EXPlc3lj5ERv2Jy5MXdeY29F41i4JHnWs56VZDF5ZOd1R+hYkT7IhU56be7uhtntii3mQN6f\nlQFMlZy8PV+6q2hgLdhB54+SAxaZ7MlB5QdoJi9Z9idvT51M4yCTfZfzKSBT49b3nldjpMP0JFl6\nbhx8Rq/QdGUQOqktNkuH6GDXI0+WHcsFtOrntT7s+ngK2FpXyOgan/rehwvtn2FusV/uHdqU30lu\nXZOUffDF5sIY9yL573rXuy58pw9icK2jq2XmCkvmmj3GZlL3FQafud+5f/vb/o6M2Rw9oidy/9kX\ne9pz3PlS12QffsPuj9bqNl591HfMvz0SmFP6tDO7bI6/ICt67r7J/vVZW10T27X7uWe/fz700ENJ\nVp43+bBdq2tH6+zKp24f3at0na+/s/AY6H424eu6/njXAncv+vVf//Uka66tQvDtu89iQ67Vtaz5\nu5uY7mF2B4PBYDAYDAZni1tjdj3Rd01BUbgIoBmXvfakSE4k0jV6HSMr+haNi75F8j7H+InSMCv7\nLj4RGWarj8l1jaM7Q5MVoRq7SKWj6D4xBHPVJ56I+ESuGAHf2xkMLKFoSETlGlhuERp2QN/Mj0gd\ni2B+7+bUH9Fs506L3LBp2AbzjUUwTpGpufn5n//5C33viiDJkh25aMs4XFNkbr472r2OTbrpNJer\n4Br+9o5w1+xavuag5WEsbVv6urMJnYuKsWUbXdNVLi6Wij7QabLu3dhH0LmcXpNHs8f0fWfV9a+Z\nCKsK/I+2u2YjmfXOZPJhTztz3Tl12Kxm7I6eOsif8Qf0wXj7FC/f33P3ya5rjHaeuL9sx+/IWN9b\nR7vCQLJkaEWCvdNbdt71sm9Cn9rnHoM9Y8PmDGvExyXrHqD/fEjn4ptn+tTskjn46Z/+6SRrzjH/\n+2oDXeF7zUVXY7hqz8UHQ9eyNpb2YXweHd/nCsPmfmec7KNXW7qWtHxk9ta53fRj9wfmqavuGD99\nP1pnt/2+uVDdoI/OJj96so8H6Br/rrKDvnlW6d8ZY9cG71NDk7VC1Tmq5qvPMDgV/JFx0wf3EXNm\npYG89hNaXbvPCKBT/Ab9J1vX+PZv//YLr7vesDnbK6aYD75Hm50FsFfhuQrD7A4Gg8FgMBgMzhbz\nsDsYDAaDwWAwOFvcWhpDb6ayTC5J2jJ5l8PYk9DR0pYALJ2g7yU5W75Ca0tBsBygTfS3ZTNLMpZu\nk7Xk5dqWECyJ91G/R4Ca72Us17QEYrnTNYxfvy0D6r+lEH21bLOXrrEM7btdqNk1LGH0cbH6aqmt\nN/sdLaOUrOW6PrLY8qaUEssoXejc39e+9rVJll50InsfVbiP02/e+ta3Jrm8NNJlnszVXt4qWUss\n5GMTzhHoH31smUvjsBxlSYlcfvEXfzHJWlK0nEuO7KSP50zWktnb3va2JGuZ3jKT8bADy3C9ocly\nraViy1OWtY7AuOh7lwU0TnPoe3Q9uVy+zlK/eWZr5pPes1X23kcu01ly2g9V6f6wU/ZtKfPocbBs\nzMYjsqV7vUGJ/uy2ad7YNR1Qxs642zf1Bi22S5foAXlpP1mytmxqg6lNjuRy9EhpcuQ3tCNdzdJq\nl8Hay1v6rJfQvXZ/okdKL+mrubWc7R7zmte8JsmS15ve9KY71+Sf6STfQnb0er8vnQJjYcu9TE1X\n2SJ92Q8d6dKK7EKKCV2iJ70hy/iNse9Zffzwfg3v+dtpWDcdGtAwLvqnD1IpXYeu6vu+CdV8m2d9\n6tRK9yxpfvyg9BaHUEihYKNSjfbjprUtpYSd98bzPd3iFGinfXcf+kKPXI8d7f3tNB/+w72WHtkE\n6FmG7WnTa2PmR/eysuxC/7osKjva+3kVhtkdDAaDwWAwGJwtbo3ZFUWLcESVvbmijzLcN351gW/M\nFLahC/h3MW0RighZ1NAboPaNXCIR7DBGWpSobczoEYhEsESiZFGPCKuLYPeRfNgUMtw3XiWLhdiZ\nIxGZyBQ7JCryufFqQ5/9bUZM5Or3RyDSFrn3fJsXr/vQAH97Mw2WQZRN3vuGLDI2r31oAH3VB3PU\nG526nNRVCfanQsRNL7pAPzapN5xhuOiLa9N/Y+ijYvcNUvpt3OSiT70BRZvYZgd30GGsg4Ltd7OB\nkZ32Jg5zR3cxuRiAfXMgppau0QHMCn3GXP3SL/1SkiWbPmyhmQzt77LUNl+G/dEvLEYfA30T6Jz2\n6D194d/62OSd2TUOumU8mCky1oY+9rGpfVyoMe2+FHymrBt95kOtJhzdgNQrfc1UWS3E5vP3+3Xo\nO1aIfpOT8XSJrT4YhN+wgc3c+N2+gVH5Jv6bv9I//dbWqTDfffwy30yX+zCe/T6hD5i16zaDug92\nabJmUfkPDB6b2Mt70UW/dS3z2ocenArt8UX8Iea/NxPzq7s8jB873odfdWkxr/k7bZGr9tw33aN2\n/9HHjPeRun2fOBWej7rsIn1oeZirfeNoH4/MDqxcdJlBf8mev2T72rMKSVd3G6XPbFUbnuv4/V5t\nbQyzOxgMBoPBYDA4W9was+tJXmTTuVFd0gRrt7Nvnui14bcijz4Gsdk0kY+IQASAAehDHJLFjoga\nRGj+YjZE40fQxZxFgyIrkWWP01/99Lk+dMkyr/cSTP7HBmIyRENKMYEo0jVFm818utbRoxyTNedd\nWkV0KOI3/+QkSvZ9uoQxksstSuwc7WTpGZkZl75gcIy7i8x3niSmQ2Tvmkcgl0xOlvk1PvLAvrMP\n0TUd7aOcf+zHfixJcs8991zoq6LlyWJNMDdeY0XkrrE1KyDKvGHT6AX2lQ0ezbfb26Sj7LtZWZ97\nfz+cgH5iJ/ibXiXqfmIcMNdy7dgDJgKzuZfnMfdy3rDe9JiMjuao0lGsIfuhH5gYn3vfWJPLh3+w\nGfNNt8hanl+vImiTHWnXnD3jGc+4c036Ri+VIKMj5LT74VPQJavYrOv1kehXHZPNZnoVhFzYID0x\nt8bbeZN93LbX+/3CNfSTX8J20Y+jhwaYf9c0TjaNPaSzfSR0snSsmX+/YWN8Lz1gg3Qco0d/lOMz\nxn3PS+e20z3j6fKkp4I/Zbtsnw47Ntjck9N+LzNu80lW3/zN35xk2TaGll+Qo2ss9913X5JlXy94\nwQuSXF5BS5Yt9j4K4/Hdo+VP7bvoEmvuWfSNDrBtdpssf+q7fWw6+ZCp8QJ5GZvvt98lx2TprawA\nukTHOt/3OgyzOxgMBoPBYDA4W9was9sFrEUJHSF7whfl7gyJCEQkLKr09C/qEomJVFzb7zs3riPH\nPdLHhugH5s7ro3lDO0TBWCTjErGKmrr4vPGKzLooNBnb0ek6+w54DOUrX/nKJCtaw+Aal6hOpK5v\noqeuCOD3RwtcJ0snzBO5iMCNFxugL3RI9Gdsb3zjG5MsZku7GB99339jvvWl86HpabMifkcnOw/q\nKGuXXD6ukhzogfnEumAo6L2oGVOGbcBsOWbye7/3e5NcZFf8huzI7IEHHkiybMZvHnnkkQt97woY\n+nw3VTqgd7x3IXN/zRF7wEInS4+xF9h+tkbvX/WqVyVZzP1TnvKUJIuRAfaByejjRJPLFR56FUQb\nR48L7goZZN1H+9JRPnVnCDExmEvMEvmwuc5Zdi0+S86ia5CbOXBEcrIYePPSTKPfHj1kgzzYizHx\n1eSLEcLu7/sLzJ825Eeya7LtHFw2SC5dnYf+Yw/3o39dw7h79YHd3nvvvaeI4Q7YrHbZnjkhX7Z8\n1bHy+sm26KpxN8uq7+yFPLCQ5NYHwuzy6pU7/TUOvtk8ngr3QX6RvtDxZvOt/O4VefoQDTJ0D7Yn\ngc65f7g2Hyb/WiUSjHcf7Z0k999//4X3+Df3Wve/m47HbegLX+W5SDs+50f0bc+fdZ9/5zvfeeE7\nfkNOv/zLv5xkrfCwL3PLBr1uee26L8edTmqLzvnNvpJ9FYbZHQwGg8FgMBicLW6N2RU19PGimAWR\nsQjP0/rOnHb02DkaIj85miLbzt3Qtsi5qxfsENWJSEVw/opg9lqep6KPlu3dkT7v3YXG1Wyh7+ur\nyBhzsLOtfTyw8XR+mz5pA0skYhOhq+bQu7mPQD6PNvq4UEwtebz61a9OsiJxfRHp6gO9oRcYjne8\n4x132sbyqgcrn0+kabe9uaKv5EYH5R/qE13d8yRPBcYF82B82AIRLtYJo9vjNFf0HIv93Oc+98Lr\nq3S4mQx96OOQzR3mh4316oW+dl3mU9BH+QImgq5iqDrHL1kMArvlfzB6naMNZISZwFTRMfLA0Ow1\nUc0j+9U/7CmZHWW96TnwB2y0awHr486YYonk+5IPH4Rlazlhq+kH30InyZ5e7Oyp/9mclShMm9+S\n5algc83w+kv+fBUd2Jm7rv/KzrGJ7Nz7rolldG3yYru9V2Xfi8JHdLWMnt+jOcxdH7VXNOmu8RvL\n/j3fwZLRC7bG/2GDselkS469csLXkBd7SpZO8hXa8tfK3r7qewr2Fdvkcu4nf2ju+K69ckbbqDmy\nqua3L37xi5Osew9/h9G1Cumva2And//BHr7hG74hyaqfbo9F28+paEbf773vnsfG3Y/35w59kwdt\nnNrQN88VXveKUR/hzIfx3fsZAV2H3xzQNf27Kcd9mN3BYDAYDAaDwdni1phdT+aewjGLnsK9L8ro\nqCK5XLOxc9a0ITrAcPUpJ3JORKedIyQ6TS6zJB0tu3azkKcAO+Aa2jYuEaz+i/7svhQVYqOwkfok\nyhJ9YS2T5KGHHkpymQ0hQ/Ml4nINMiMrjBeZ+/xozdBkRbPYA7lqWAV9+Zmf+Zkki+HEQpMflkkf\nRJP6Jq9qr58owpR7qg26guHAAoj66VjXRta2Pu61Xk9FsyiYHXODTTEXncskz4mtYQKwtHIm7Wbd\nmQ+2ol4iFsXfRx99NMmqP80eyBH7jvlhz3TxbupSu7a2uzIEX+Jz+r+vaNAhMjIvmAk69/Vf//VJ\n1jyrCdunLGEpm9naWSc20jlxZIMtPZpzpz0yxdCZO+/zofq6M4TmS//1tXNS+0Q5+i6PkO60b8VU\n7acV8lvGSxesiujTzqidAn3sutTGRA5sko1itZNlK/TZPPNBfAm9IA86+LznPS/JWhnovpDHzq5r\n23zxNWTLJ7tPnIpmPrGMnausr+R31Ultxs3XGI/VMfqCwTPu5zznOUmWL/M+HaQDO2OqDX7Kd9u3\nHM3ZdR8lZ78nB3sYjMX19r5ZkabfGOmuw0z23/iN35hk1Xa2z8I9qysJ8e385d5Pemsvh/t9+8NT\nYZz8CFvvSktWQq6qx97nJrBl9o7BtzJMB/kqutbsMXi9r/Lwb3xOn4jKrvfc86swzO5gMBgMBoPB\n4Gxxa8yuqMqTumhVNNE1REX1opn9MxEHJgCjg5XDmHQdVuystkUEfUrYzkqKIkS2fVKQiHXv56kQ\nQemfaFEk59r66/M+EQczITrSJ/IS6eynVolAfUeEJlIVqXV1ht5l3LspRerYqiMQQYtURdF0xLj0\nBfNBPlg784dF6XyxZkb3/vaJMGRH5qJhfTFe18QIAjliQo9AfzFhfeIWBogtqRQgCtZncmAfxqQd\nDPC+Q9zuWTLFrpATZgezZQ4w3RidriTRtnsEza6zn14VMi5933My2an54hNalvQaq8GXeC3nrOtw\n8xN7vph+sTn5muwVQ3G0Tia/4VoYF2Nhk+zKHO359Fg0et+sc7fdqykYLJ/TWdfy+/00I/cA18KK\n6gM93/XxFJAj28Syy6ek93Tvqlx1tuZ+ZBzmtdlQc/ut3/qtSZYc+Ak+hh/gb/caseaAfzdusiaX\nzuG9CfrWVRa6Yog5ch/ZWXjvdV1d/oyPpsvyXvmgXvExhq4Jvdey1b/OOXcNvudoJSTzrx33Ovr3\n1re+NcnyGz/7sz+b5OI9z5yw4be85S1JVlUF+mJF4Cd+4ieSrCocfDA5WCEwx+7RWOb9WnRO//tU\nQ/p8Kjw/0Ddz0itFXSVltxdtdMUo9u41W2QHPicvekBvXJt8d3tx/V5NtkLS1TSuwzC7g8FgMBgM\nBoOzxa0xuxiRhkhRBOh7orqrWFZP7l0DtneIY2NAhNKRS5/6tV9TpNFVBkQXor7rxvfBgJnBvIh+\nRDld49VfLIP+i6q8L5rqCgl7zrN+i27tEjW+rreJhekTgzB2cpGMpRngUyDKE/Wab9Gj/FFs4TOf\n+cwkq54wZse1sTDkIbKXP7TnfGGJ+6x2uiAfts9RJxd5r13LEsjxCPSBXor+sW4YMuMlH3Misu36\nw9oxZ30q4H5N9ROxC2wNG4XRoFvaFnWLyMnDXNzNCWpdVYAu9rzrqzHsubDY7j5lTK4Z9qvzwswf\nH2L82tMOu9hPb5RjL2/Pb7SBsbkqV/KDodkRTF1Xl+Fb9WnPUeVD2QadMj9ssWt6dn4p+RiTdjFX\ney4edgfMLWn5AAAbeElEQVTj2rnHZHi3TKbf6wOWlo/rurNX1R3GzPrOs5/97CSrXjA95mPoHLm1\njzZXzbonl0+llAfrtXvkUZvpEzmNqX2898lhv//1Xhu62qcZQp+g5lrmxD2p6zbvOZnuc/SB7nVt\n86PVKbrOsvacSEZH7Qkxx7s8+CB+0cmR7kHsyHitqhkT+zBeNqkPXTM7uXx6J1/rvkhnj+qHnF/t\nuDadp9PXrQgka4WnKyIAmWPj/e0VJWMyBjqv/f3kNbKkO/yZ+0LnHF+HYXYHg8FgMBgMBmeLW2N2\nPYWL5kVVIkERpOihd70n6ylfjo1oQJt2Uj/88MNJLkfMog/Miid/0RoWZ48YRbsiEJGqqBlL0DX8\nToFIqhlNkaS/GCwReNe7E9nIIxQ1yU0zzj0HTmQpisPqmQeMRJ98JVLVVwxFV8q4m5PlzDk2sE8v\nevDBB5MsFtX3yFHEJ/L3V2Tau9H36JnMRLmqDJhX7JAcNXop+sVC0CU6ij3xuyMga6wy1kBfsEds\nCWtCV0Xu+ur75MfWjHmvZdgVHqwyqOhAt+hoR/bNyriGudiZz1NhTjAZdIzu0gd99b2dReMLenxs\nsGs2m18yZINsji513tue19YsaNfm1add/qegT5mkH2TbOak7owt96pz50cfOF2wGBkvbrBt5dm3b\n/T32TaZ8kt8e9SF8sbl1HX6Fn6APXTkgWTKzGqJKB1sxfuNV4YZsMVfGhPH2Ob+w7/HgY3yHbPkz\n1z5am7pPMSQftti6Tg77Soj3WqfI1PjdL/yW7fEH3/Ed33Fh3F3Td/fFPjN/7nfu/+zk6Ooh/8H/\nkTMd770zdNmKTLJ8qzbYnlPZzLc55ZN+4Rd+4cIYAItPZ117Z8z9bx8EX0t/6XOz7DeBfrB1c2Hc\n7M+ql+eNfa+LVSRzxk96zZZf//rXJ1nyUBGJvui7HF3XIt/9/qlfdLLnos9quA7D7A4Gg8FgMBgM\nzha3xuxiBjx19y4+T+edp7mzHaLKZgswu32KkyhaBPeCF7wgyYqI5NKJeOGqeob6L8oRiYls9nzH\nUyESaeZHhCU67FN7Ol9YNNW7l7XTDHayco4wW10nU6QuwhI9Yl9EsBgQYxCZ3k01BswFRqPrCPfJ\nR3QEW0AezTrRGzomMrb7Nlky7PE3GyYP1px0TUjX8v4b3vCGJMfzMZM133RMziE2hf5irMwhpoL8\nzKE5kdurdqwofGe4RMXY4G/5lm9JsmTYJ02RWzO85qRrPe87vk+FNtpeemUEW4B92lkjMjB/7FkV\nCTqDPVddw/z1rmk25lpd6SW5zAZ63ez20bx/9mz+zbM8ud6zwK/t+ZH0lm6xg65oY77ID0PjtWsa\nt2s1m5pcPvmSLLue9Ac72fIq6KM56uor+tKnd1pB2fvi3kF32Bb7Jmu+WJ/5KnNNvtjJzpNM1l4N\nbfAd7mvkdFPd0Ibx9t4Wufrk26svu+7Sg2bTyME4ydD3+CA+xb2ZntBde0b2exN/TofoGLu/29r2\nTuyj/+43bFffMN3k813f9V132jDv9ip4LtAX1Sh+7ud+7sLn5tBKwStf+cok6z7Kd3VFkWT5efrt\nNX0h26NMtzlkj10FCVttDunH7k+7oow5o2v0hU1qi55guvWBPtFJ9rPX9nVP5qu1YX5d+6Z9MsPs\nDgaDwWAwGAzOFrfG7IrsPel7wvcUjmERbXct0eTyqUUiG5Fe13hsVkpE2zUlRSeih/20K3mw+u2z\nzjFudvgUYEdaJtgVUa/xYAX6rHByEDV2TWDRkXyY5PLOTQxV79wWxYkCsQtYGH0Q0bnWXjfxVIju\nsSsYDuMTyTX7L6oUkTt9Rg6Wee1qG3uOk++IGjtS7fnH/HTVELmsvk/n5Isdgeievu+1BpOlc+SC\nETFuv5MXhik2VsyR7+273zEVL3rRi5IsWYuisVGdo9cna9FF36MvR2vK7n1oxrJ3J5tXumqcO8yv\n37JjbWMJsK2+x4boWp8ARf/3PnY+r1xVsqGvR0+VI2v2oA/Gwh9iqJqlTZZO+G3Lluz4iz6Vjr/o\n/QRtR7utkWXn7bFz/e6+3ARy5Sf5D3aELSJvfd9Xb7BmXWcY+gRJDKZrskn3DTrpPsaP7v7Ae736\nyYf0aVOnoivj6GvLmS3q087skk3nMps7Muab/DVev3cvIz/y1bddHl1f3LXNxd3m7NIzc9Q59OTN\n9vvUuCR58YtfnOSyvyNb+4bcw/ggNsdvshPXZD/2RLzuda+7c82uosP3kH1X0zgVnaOtj33iYLOs\n7CZZ9xzjsopgnsnanGGPPVfwC67d9f/ZLv3Zr4Vl71q99Ma99zoMszsYDAaDwWAwOFvcGrMrWvM0\njjHBFIk2PdljzHaGsPM7RLwiDdGVaEn05NqiDhGRiKXZpj3Sx1BhQEQmGL4+f/4ImpHq6gLgzGtR\nkL+u3WdBY+rIWns7m0sWokUMjChR21hAMjYvXUdVhPfBzle/CfsO9mRFkr3DVfRLH4zXHInIMVp9\nQhDm09iTJQ/j7EoXxul156LSKTKWA+m1z4+A/mILRNyu3XUzrQg0y4KZ8NocddSslmiSfNu3fVuS\nFR1juI2/q3GQaZ/qQ8byylTUYHNHwBd05Yte+TCX7H5nMrF6ZOY3bKiZJ7Lxmg6qt4qx6ComV+0u\n158+4cf8HJWJuXBtPtQ8985ubKOVgGTl7vdOd7bIp9JFttjzbNwYOn7D73d2jG+gf/rTp3EdrU7R\ndWO7ik1XnmDj+x4Ncmh2i75jj41P/mfLtquSdM30PQeRfmLY6DU50bmjp3Sak66o4HWfzEjHd3nw\nwfI3sWq+0/dz4yMXcmsWtutd76tK9NoKhX65RrOGpwIjyj7IV1/4BHrpPmPOkyVDfo/Pdd/wvIDZ\nNbeubbzyhMmlnyvsGUhWPjRdcn9j19q07+BU9JywH3K10kQeVi33/Un6yaf5rr55n7z4OG2zeXNA\nZ+lX54YnawXEe+bTPUY/b6o7PMzuYDAYDAaDweBscWvMbp9MJGJshkxkKPoUESWX80NFQU4v6YgP\nC6Ft0QWmVEQkuhDB7bkvIhVRt2jYOcyYoZ3JORW9c72jGOMR2RuXKBIj3PUUsS2iJRHPXgu36wfb\nZW+euh4kpgdDKQLTZ5G8PtxNNQbzqp8YGOPWN3MiEpdnd8899yRZumWMzWSS386gdz4SFq7bIvOW\nn+/TLdGmOVarUUWQU6AN48Z4YBPoILbyp37qp5Is9oXtOJcd860KBeaCHPY6nuyv89vML9ZNlM92\n6AmbEsH7nDzozxFgNLRFz42zWQFM575SQ0YYmq7oQVfIBAPTuWf0wvt+73e7rflNV+pgI33W/ano\nGqX0ha+iB131ZWd0HnjggSSXV4/oBpvpyjCuTX7mnezpkrnZT6Eybv3rnOq7Zbr5i2bQ92snlxnd\nvYanE7HoNeaqfSs9xjaSm8/JnJ4ZS+cn7/3s0/jMk7aP7oPAbLEL1+Q/sI69D2HPHTff+t86ipGk\nw/rIPvgN4yZr92aMsWsny99h6PTf/ZtO3u0Jexhd4+xVO3pJh3eG0Nzwpcb9mte8Jsli9umP5wVy\nIgfjZQvkq4/78wSZ+3vvvfdeGBf/crSaC12lf1ZCycFcNvu+26VqTmyPbyMXK2NqBGNl6ToZ86PG\naEXxqtMDjdc89eq0fu65xVdhmN3BYDAYDAaDwdni1pjdfnLvWnc+79Mv5NIlK6ITVckHE0WKqvzF\nBImyuiac6MLv9Wk/DU1bohmROiak6+Qdgf40a9A7YHsHcJ8UAlgWzAZWirz2XCzRrGi55S8CxVhh\nEbVFRqJjn/v+0XPL93FjU7AK2sSAk5M8QixkM2Siwa7PaA73POtmE7BNxkVeIlURrD7rk0jUNTG6\nux6fis5/Ihd6agXA3KnGYSwi3u/+7u9OstgHTPCv/MqvJFmMuBqR+3jYjt3X5hsLgE1QpYDMMXq9\nY5b+HN1JvY9T250H2dU8MAA7s4eB7FUULIDvGidbwjS0rnlN1l3VJFnzZ8zs2W+7pvWpMD7t94lH\nfBTZu/6eG2/+umpAV1HAQOljM/T8iGvwqX26YbJ0R9vNfHeVnVNhhadZRq/lvncd9d3fs/uucGKu\n9BEjTn/ovVU0/rNP+ev6o3sb2D9soP6T19FTOpvB1V6fAuoeZqy7HtKPnnffpTd8M9mSB10z7l5t\n4xf3+wV9IHO+Vj44O9/zOE+BdvkgNvrmN785ydJhc8a3f9M3fdOdNuTqdw1zttZ5v+TnfmAl2Pt8\nNjtyj99PjsWG0mO5uRhZK3c35ag2uiZ0M/3ur2zZHKnuk1zes9F1la0Y9ymXzey6Z5ET+cJ+v6AP\nnpu6dm+vvlyHYXYHg8FgMBgMBmeLedgdDAaDwWAwGJwtbi2NoTcloMYth/RyMBp/p7NR2WjsPgbS\n0pNlCktJ6PjesNIlm65aeu8i+uh3Sz9dcugIurBybyiAPv6OLC1rW7aUtoG+tzTQG/WStYRhzJYb\n9ME4LaFps48o7Q1M5tWSwhH0BjV90Tfj0we6Qw6WTiyNWWKzDKTP+njVMocUATph6dBvybIPrLAc\ns29MStayXS/LnAJLPpYCLX3qm7bJje10eoxr6xs7euYzn5lkLee98Y1vvHNtS6rGZQnYaxtNzEkf\nEkMf7r///iRr6c3mCkthL3nJS06WRx//aPyW0iwB8g+WbS1TJqtEVBdUv24Dou9ZliRb7ZAtnWSb\n+/Gn2mBrrmE+zPN+XPMp6CVx7bLJLndnCXUvuacNqQ5siN73xmH+jn1737jNu7FJNbP8nyydoXf8\nHVn2JuZTwa8bi74Zr/Qd8mq/kCzfwafSOZuayNT4eiMan9Ublr3fhfqTdW/xGX1gz506cyo6naNL\nkWm/N/7uZSN9lx7YHGru2A/ZS51zz/U78mo/2EfB7/3RBj8vTcXro6Xp6IE5UyaMTyMPvu6FL3xh\nkov6YW72TY1739iFY9HJz0YufkPfH3nkkSTLTvRl91m+S8f4kd706550FO0/2YD7hNQ9c76XRTN/\ndIsf6E1u/AK76hRUNut3/azmGSC5fL8z/j5kim5eh2F2B4PBYDAYDAZni1tjdnvjh0hfRCCK8Peq\nTVi9acBf0RaWTTRps5Eo2ueiTBHTfhRdcjHKxDJg/DCd2nSNu9mQhT3tUjn634cqiLj0W1K8yMbv\n+oCOTpbf/8dA9KESrqENbYrUyEX0LarsY4SPoFmCLoqvb5Ldjde16RIWhVx6gxC5Y5j28TSz5+jI\nZkUxPF2ySSRvzryvMPgRaFtUa67IViSOLemjF43BphuHRtDl3vCHddjHQQfpEJn7LRmL8G0iwVYB\nGzWmu9mw15tgjJddYxUwU9gXrEtymVXto0i7VJBx0ymy0gcMBXvy+33VoA/BwbDSJW2zpaPo0mLG\n0sfh0vv96F5sUcuQLWJ9yK3t3CpaH+Vs06Ox7UwY22B/9NrfPhzjVHTZN3aCwaXL9Iic9hWvV73q\nVUlWaSi289rXvjZJ8oxnPCPJ5UNUzC07eMMb3pBk6QH56sN+lGmvONExMu5DMk5Fl8nr8lD0AeOJ\nKdyZTDpkHM3KswP3Q7ZHtl2qzrjpiXb3smrusb2xmmx709+pUOasS7r1CrG+YWPpy35Nv9FvK1ie\nUXqzORlb4er7Cxbe5lvMcJK86U1vSrLsvDei382qYXL5yGbArNObLiO339v7vsCve57wmq71cfLG\n0MeOmxvy2VdK+Cb+gc+iL+brpvKnw+wOBoPBYDAYDM4Wt8bsevLHhIiQRAkiIdGVCGgvEyNiFz11\nzk7n7IEoqnOB3va2tyW5fPzqfryoKLPz4ZSkEi3vpWROhajWeDoa3hmYZEUumJGOcETXIjDv69su\nS9GPtsjfeJotFan18X8ifZGp9/vo31OAZRNxkovXomR96GOA+2hK0aIIz0qAPu/lXbo8nd/QBbrR\nh2zQBzmoxoAZva4Y+ymgrx2hKjWDeaCj7OG+++5Lkjz00EMX+tSHq/g+5vj1r3/9nWsoNdYF79lW\nl8NxPDJ2CCOGFWD35Nb5t6dAv5sJZAd01PtYA79LFhPRbKjf0Bm2hpmga5gc9oIBxEaS6Z5fyYbI\nhN718d9HDw1wLfrP/2GZ6WrbLL1KLh+tiYHssj78Qe8vgM5txuBflbvfJdd8l1/ma4+WUjJXxs0f\nGAud6/J5+6pcl2u06qfP/EEfD86+6aK5ZA98r/d3/0jmxu1aXRby6D3GXGifnhhDH3RhNWMv1acN\n/WcfGEh6bu7YYLPS7Iec+Oj2+cmaR/cx1zAnbPdu9oUk697nABHMqP055INR9f7eX/IwV/pE53zP\neJ/1rGclWSsejukmT74c8/3www/fuSbfSy7tL8j+6D2XPZCH1TjywNL3iut+9HcfwsLf80G9L0Zb\n7gfAF/RzhLHt7XQeML/a7PL+HHcVhtkdDAaDwWAwGJwtbo3ZlbMi6vaEj3XAXomURKM7Q/Loo48m\nWVGDiFzbogERYB8qIGIVqfXRsKL1nVHFCvSBBaIGUfe+W/BUYGb0p3dGk42ox2s5lyIsEU4fPtFH\n2u7FwkViWA75mlgfrIdcO211VYrOtb6qiPypME+dB3wdU9O7U8nH942JXEWdnQObXM6xwr7ZmQsd\neXauIzlhPl3LnB0BdoCtmBNVE7CK5mzPuU0WA6yP9Ft7dgJjgvfcLTLso6ox2FgEttj5Y+aS7pkr\nY7ByczfAvmHCtE0e9N/rvUJGH7DQBfbpt/kjh2aLydzRy8ZLXjvrRI+xGVixzsHejyk/BfrGP2gH\nG+c1W9W3vRg/VqdZEW1jrOi973dBfzbm+77HB++rSvph3vitzik+emgA3bvOR5GvFS/v06Nk5V7S\nMf02n2yHjvWx0b5Hz+kJebjn7GPrXF12Tw7uZ5jXU0H2zTazyV516aOzkyUrOmZVTF+Mi66Rg/f5\nZmPhg/o49b0CketjhXvvijaPrg7xc3wPeRi/uSEH98Z9xcuxtw6DMO7eq9AHdLT9dHUqzwK93yJZ\n95DONdcGX3WU6e7nHn6JHPSdHtGbfW+BeTWOXl3tvnmfnmjL5+zCX+3u+eqeXcw/G/PXXNxkL8Ps\nDgaDwWAwGAzOFrfG7IqURf6iCYyCKKvz1kRXyXr6t5tZFNE7xj35Yymwxb1LWTQlQsAA7YyfCMM1\n5dR0buqe13MqjL0rIIhQXLtzN/2V0woiGvUkO0q8Kgeu64OSjWu2rLzfR012/cjejX8EokGsoShQ\nFI1tIR/z632Mngi9mTTMwJ4L27llmNjOzSMPkTt2Civj/a78cHRn+f4bfekcbblKIljzTRdVgNBH\n9iNKfv7zn59k7Yzd65rSbzZlPGTY+dJsDftA9trElKsfebRGZrLmjR50XmAfp0qP9vfJkAzoVteo\n7iN9yZrO8TGYCewQubHF5PKqCbgWvT2ak9msI7ax62b2ysaePy5flr3SBbmm2uh6083cmG8+iS+9\nard953WyQ/6wqwScCnPjXtN+QXts2KrVXgOYP7Qb3jzao9HHiJNpH23bK15dSWHf0W6+yIqMewXS\nKsSpaEZXfdQ+2rXzsXeWtXO3faZt/t8Y+ME+XrqPOvc97ezMf9t5r5bpk9enAqNrTswtvbEyYG7d\nyx588ME7bXgeoTvutX1MOn9nDF21xxw3C0l39SFZvpJv1Qcy5f+O1vo3Pj7MXLEH1/PM4/t7vm3v\nfyEHc6aOOtle99zHfxgrfaIfe8UUKx1qgrfukTHffh2G2R0MBoPBYDAYnC1ujdnt3ZqiBPkoIhoR\nIVZvf6LHaNkl2MwItgW7gHXShijLNfr0D8zLvjsXU6efvtNMnxytI8AKYOREN/vpS8llRk7kIpIh\n0z4pDjvj9d5H1yYjURyGFjMj8ur6ssaN+ek6pM1inYKO+r0W7YnA9WVnA5IlJ5GfaLPzw4xt33Xc\ntWpdo/Mhm3UHESjdpDd3u9M+WdF/y7zZVOOy+uBabMrcYBfo/atf/eoLfd7rVory6RRZmW+6imVx\nTTpK57ACGIAHHnggyXFWJrnMTJI5ndU3eoHx2HcP05HO9zevXR+arPkM45T/R5f0jdz2HH7Xuu7a\ndE9/T0Wvhul71w3l/3qn+N5//lgfsWCdi8xPaKNPLdJOV7jZWevOz8MOYoPo5547egroAZujeyoq\n0EFy0bedQcbM+U6fkIZN0zeMJ9ukB/SF/9COud/lQY/9lkyxe+RxdLd950m6z7BZfrHzj/d80T45\nkK8wPnLqij+dy9w5zRhyLOG+TwaDrc3Ol+9Tt06FCjP0oU9c7ZUhJ0zu+y2sgNAZ+kIv2J555j+x\nsXw2v2KujRlLue8V8RtycB/na31+tA6z+2OfJrnnsCfLR9DHfYWhT5RjU9qke5hr/gSj3as6/tIH\n8tifycwTZpcfYYP0Yn92vArD7A4Gg8FgMBgMzha3xux6GhcRi6o8nYuUmnXbWU4RhUhUtCjaFLmJ\nMp7+9KcnWVGSiLVPPxExi0p21k60g7HRv44m/m/stvdXBCsSx8yJWLqmX++WJDt/RZf7WdGiYxFp\nn8LSUW/vUO1TaERod1v7MFnzaX4638v8GxemRl9Ecn7fO+07h21nBozDdzBY5EQ+omvjdToXFoIO\n9WkuIt0joFtsB9tGz5vRxABjk9S+xQT4Xus/Oe85Ts1I+S5Ggmxd03i9j9k0R12n+W5OCyN7LCy7\nNy7z3Pnne/64eSJDv+262eZV/Uu2hS3sfOmuW72zyZ3P3DVbe1ynAutBN10bu9Z1NMlnPzmQDzKP\nzdy3nZNP5/D7Pfs33+0f9mtcd+oitu9onV190LfON6ff5M1Wd9/XJ3iau2aJupIDvTam1hO/Z7P7\nvhAyaqbNSotVzqN5/+adXIzJePn4rnKw665xuTb/ry3Mps97Jcsc9CoVubLHPd+UTvZOfvrRfv9U\nkIf7Cfmydf6y2WsrYnsfzCf2V1WWPkHOcwJZ88VdOcqYPYvce++9d66pag5do0P2ZGjj6D3GuOgd\nuzHHxm+ufW7FcB9X/9Y9xb26T6Y0d+bfX76aX+m69cmSPdl6DiIfMjTP12GY3cFgMBgMBoPB2eJx\nN+1gGwwGg8FgMBgM/n/FMLuDwWAwGAwGg7PFPOwOBoPBYDAYDM4W87A7GAwGg8FgMDhbzMPuYDAY\nDAaDweBsMQ+7g8FgMBgMBoOzxTzsDgaDwWAwGAzOFvOwOxgMBoPBYDA4W8zD7mAwGAwGg8HgbDEP\nu4PBYDAYDAaDs8U87A4Gg8FgMBgMzhbzsDsYDAaDwWAwOFvMw+5gMBgMBoPB4GwxD7uDwWAwGAwG\ng7PFPOwOBoPBYDAYDM4W87A7GAwGg8FgMDhbzMPuYDAYDAaDweBsMQ+7g8FgMBgMBoOzxTzsDgaD\nwWAwGAzOFvOwOxgMBoPBYDA4W8zD7mAwGAwGg8HgbPF/ACxYFxmbzJy/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb3930123d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting the weight images, we can recognize similarities to the target images\n",
    "plt.figure(figsize=(12,3))\n",
    "for i in range(10):\n",
    "    plt.subplot(1, 10, i+1)\n",
    "    plt.imshow(weights[:,i].reshape((28, 28)), cmap='gray', interpolation='nearest')\n",
    "    plt.axis('off')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Exercises\n",
    "====="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "1. Logistic regression\n",
    "----------------------\n",
    "\n",
    "The simple network we created is similar to a logistic regression model. Verify that the accuracy is close to that of `sklearn.linear_model.LogisticRegression`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Uncomment and execute this cell for an example solution\n",
    "#%load spoilers/logreg.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. Hidden layer\n",
    "---------------\n",
    "\n",
    "Try adding one or more \"hidden\" `DenseLayers` between the input and output. Experiment with different numbers of hidden units."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Uncomment and execute this cell for an example solution\n",
    "#%load spoilers/hiddenlayer.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. Optimizer\n",
    "------------\n",
    "\n",
    "Try one of the other algorithms available in `lasagne.updates`. You may also want to adjust the learning rate.\n",
    "Visualize and compare the trained weights. Different optimization trajectories may lead to very different results, even if the performance is similar. This can be important when training more complicated networks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Uncomment and execute this cell for an example solution\n",
    "# %load spoilers/optimizer.py"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}