{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../../')\n",
    "from layer_for_theano import *\n",
    "#print __layer_version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load dataset in 2.7 secs\n",
      "(50000L, 196L)\n"
     ]
    }
   ],
   "source": [
    "import cPickle\n",
    "import numpy as np\n",
    "import gzip\n",
    "\n",
    "import time\n",
    "\n",
    "class dataset_mnist_small:\n",
    "    def __init__(self):\n",
    "        if 0: # no requirement for gzip, but big file\n",
    "            f = open('mnist_14x14.pkl', 'rb')\n",
    "            train_set, valid_set, test_set = cPickle.load(f)\n",
    "            f.close()\n",
    "        else: # requiring gzip           \n",
    "            f = gzip.open('mnist_14x14.pkl.gz','rb')\n",
    "            train_set, valid_set, test_set = cPickle.load(f)\n",
    "            f.close()\n",
    "\n",
    "        self.xs_tr, ys_tr = train_set\n",
    "        self.xs_val, ys_val = valid_set\n",
    "        self.xs_te, ys_te = test_set\n",
    "\n",
    "        self.ys_tr =  ys_tr.astype(np.int32)\n",
    "        self.ys_val = ys_val.astype(np.int32)\n",
    "        self.ys_te =  ys_te.astype(np.int32)\n",
    "\n",
    "t0 = time.time()\n",
    "        \n",
    "ds =  dataset_mnist_small()  \n",
    "print 'load dataset in %.1f secs' % (time.time()-t0)\n",
    "\n",
    "print ds.xs_tr.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_update_rmsprop(loss, all_parameters, learning_rate):\n",
    "    #all_grads = [theano.grad(loss, param) for param in all_parameters]\n",
    "    all_grads = T.grad(loss, all_parameters)\n",
    "    updates = []\n",
    "    for p, g in zip(all_parameters, all_grads):\n",
    "        MeanSquare = theano.shared(p.get_value() * 0.)\n",
    "        nextMeanSquare = 0.9 * MeanSquare + (1 - 0.9) * g ** 2\n",
    "        g = g / T.sqrt(nextMeanSquare + 0.000001)\n",
    "        updates.append((MeanSquare, nextMeanSquare))\n",
    "        updates.append((p, p - learning_rate * g))\n",
    "    return updates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nonlinear activation function =  Elemwise{tanh,no_inplace}\n",
      "before training, err= 0.91\n",
      "now training, cost =  2.36644118306\n",
      "after training, err= 0.77\n"
     ]
    }
   ],
   "source": [
    "from sgd_for_theano import *\n",
    "class mlp_rmsprop(tt_sgd_model):\n",
    "    def __init__(self):\n",
    "        #784\n",
    "        dim = 196\n",
    "        nc = 10\n",
    "\n",
    "        self.layers = []\n",
    "        self.layers += [InputLayer(dim)]\n",
    "\n",
    "        nonlinearfunc = T.tanh #T.nnet.sigmoid #rectify #rectify\n",
    "        print 'nonlinear activation function = ', nonlinearfunc\n",
    "        self.layers += [ HiddenLayer(self.layers[-1], n_out = 500,  activation = T.tanh)]\n",
    "        self.layers += [ HiddenLayer(self.layers[-1], n_out = nc,  activation = None)]\n",
    "        self.layers +=  [ SoftmaxLayer(self.layers[-1])]\n",
    "\n",
    "        self.target = T.ivector('target')\n",
    "        output_eval = self.layers[-1].output()\n",
    "        self.cost_eval = mcloss_negli(output_eval, self.target)\n",
    "        self.err_eval = mc_error(output_eval, self.target)\n",
    "        self.validate_model = theano.function([self.layers[0].input, self.target], self.err_eval)\n",
    "        self.cost_model = theano.function([self.layers[0].input,self. target], self.cost_eval)\n",
    "\n",
    "        output_tr = self.layers[-1].output(dropout_training=True)\n",
    "        self.cost_tr = mcloss_negli(output_eval, self.target)\n",
    "\n",
    "        learning_rate = 0.13\n",
    "        #all_para = all_parameters(self.layers[-1])\n",
    "        all_para = self.get_all_parameters()\n",
    "        #updates = gen_updates_sgd(self.cost_tr, all_para, learning_rate)\n",
    "        updates = get_update_rmsprop(self.cost_tr, all_para, learning_rate)\n",
    "        \n",
    "        \n",
    "        self.all_para = all_para        \n",
    "        self.train_model = theano.function([self.layers[0].input, self.target], self.cost_tr, updates=updates)\n",
    "\n",
    "model = mlp_rmsprop()\n",
    "\n",
    "if 1: #unit test\n",
    "    import sys\n",
    "    model._unit_test(ds.xs_tr[0:100], ds.ys_tr[0:100])\n",
    "    #sys.exit(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500L, 10L) (500L, 10L)\n",
      "(10L,) (10L,)\n",
      "(196L, 500L) (196L, 500L)\n",
      "(500L,) (500L,)\n"
     ]
    }
   ],
   "source": [
    "paras = model.all_para\n",
    "grad = T.grad(model.cost_tr, paras)\n",
    "\n",
    "for p, g in zip(paras,grad):\n",
    "    #p0 = p.eval({model.layers[0].input: ds.xs_tr[0:1]})\n",
    "    #g0 = g.eval({model.layers[0].input: ds.xs_tr[0:1]})\n",
    "    p0 = p.get_value()\n",
    "    g0 = g.eval({model.layers[0].input: ds.xs_tr[0:1], model.target: ds.ys_tr[0:1]})\n",
    "    print p0.shape, g0.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " epoch 1, minibatch 84/84, validation error 0.228922  training error 0.236409\n",
      "\n",
      " epoch 2, minibatch 84/84, validation error 0.324853  training error 0.320258\n",
      "\n",
      " epoch 3, minibatch 84/84, validation error 0.183137  training error 0.187123\n",
      "\n",
      " epoch 4, minibatch 84/84, validation error 0.170931  training error 0.172321\n",
      "\n",
      " epoch 5, minibatch 84/84, validation error 0.063775  training error 0.060397\n",
      "\n",
      " epoch 6, minibatch 84/84, validation error 0.091373  training error 0.092560\n",
      "\n",
      " epoch 7, minibatch 84/84, validation error 0.107549  training error 0.102004\n",
      "\n",
      " epoch 8, minibatch 84/84, validation error 0.060490  training error 0.054762\n",
      "\n",
      " epoch 9, minibatch 84/84, validation error 0.073137  training error 0.067381\n",
      "\n",
      " epoch 10, minibatch 84/84, validation error 0.062500  training error 0.054841\n"
     ]
    }
   ],
   "source": [
    "best_err, best_model,err_list_val, err_list_tr =  sgd_batch(model, ds.xs_tr, ds.ys_tr, ds.xs_te, ds.ys_te, batch_size=600, n_epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEKCAYAAAAYd05sAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8FHX+x/HXOwktdARRkCZFQETxFM8eFRWxYe9dyimg\nd2e9s+uJ/tQ7sXcFxS52FFFBsIEoWGgCgnREauiBfH5/zMQsa8oGksxm83k+HvPITv/MZPez3/3O\nd+YrM8M551zqSYs6AOecc2XDE7xzzqUoT/DOOZeiPME751yK8gTvnHMpyhO8c86lKE/wpUTSHElH\nRB1HoiRlSZoXM/6TpEMSWXYb9vWopBu2dX3n3LbxBF96LBxKjaRbJD1fmtssjJl1MrMx27sdSRdK\nGhu37b+Z2R3bu+1kJ+luSb+Hw13FLHuEpGmS1kr6VFLzuPl7SxojKVvSYkkDYua1lDQqXHdqYQUL\nSc9IypW0a8y0ppLelrRM0jxJfeLW2UvSt+G2J0jaM2ZeJ0kjJC2VlFvA/jqEx7JS0gxJPePmZ0p6\nJFx/paTPEjl/kpqH5yF2yJX095hlekuaKWmVpG8kHVhAfA3CfY+Nn5eqPMG7SkFSRgHT0ktx+32A\nE4HO4XB8fPKMWbYh8Abwb6A+MAF4JW7+B8CjQAOgNfBRzCZeAr4N5/0beD1cJ3YfBwG78udCxwvA\nLGBH4FjgTklZ4TpVgbeBIUA9YDDwtqQq4bqbgJeBSwo4poxw3XfCY+oNvCCpbcxiT4TbbR8uc2XM\n+oWePzOba2a18wZgDyA3PIdI2gu4DzjNzOoCTwNvSlJcmHcDUwo4J6nLzHwohQGYDVwHTAaWA88A\n1WLmHwdMAlYAXwB7xMy7FpgPrAamAYcD3YGNBB+qbGBiAfu8FngtbtogYFD4+iKCN/Rqgg9175jl\nsoB5MeNzgCPC1zWA58LjmAxcHbfsdcDMcLuTgZ7h9A7AemBzGPPycPpzwO0x6/cCZgDLCJLCzjHz\ncoE+wM/huXqoiHOumFh+J0iS9cN5LcNtXQz8CnwGXBCe+/+Gy99Wiv//L4FLY8YvAr4qZNnewOcx\n45nAOqBdOH4nMLiQddsBG4CaMdM+A/rEjGcA35GfCHcNp9cKxxvGLPs4MCR8fRQwP25/vwJHx01r\nA+TGTesEZMdNG5F3jgmS+iqgVimcv5uBT2LGzwbGxYzXDI+zccy0A8J9XAiMLet8kCyDl+BLjwje\naEcRlLjaATcASOpCUKroRVDqehx4R1IVSbsBlwP7mFmdcP05ZvYhwQf9ZQtKLl0K2OdLQA9JtcL9\npAOnAUPD+UuAY8PtXgT8L4ylILFVTDcDrQhKgEcTJMbYUs9M4KBwu7cSlNQam9lUoC/BB7O2mTWI\n37akw8PjOg3YmSCBvBwXy7HAPgQludMlHV1IzAOAE4BDwm2tAB6OW+YQguRyNMH/qCv5Jdg74zco\n6WxJKwoZlkvapZBYOgLfx4z/AOxeyLK7xy5rZusIzmne8vsBKyR9IWmJpHckNYtZ9xczWxuzve/j\n9vV34DMz+zH+8OL+QvArvlPMtn+IWyd+2yWRFrNuV4L/9W1hNckPkk6OWTah8xeWys8n+HWRZyzQ\nSlLX8DNwMUGBaEm4TjrwIMHnrFLxBF96jKC0ucDMVgD/Ac4K5/UGHjezbywwhKB0vj9BabcasLuk\nKhb8HP0lXE9s/WHceodmcwlKaieFkw4H1pnZ+HD+cDObHb4eQ/Az/+AEjuU04D9mttLM5hP8Kvgj\nDjN73cwWh69fJSiN7xcTc1HOAZ42s0lmtgm4Htg/rg76LjNbbWbzgFHAXoVsqw9wg5ktNLMcgi+b\nUyXFvq9vMbP1ZrYhHF9oZg+bWW7MtD+Y2YtmVr+QoUF4PgpSi6CEmmd1OK0gNcP5sVYDtcPXzQi+\nVAcAzQl+Hb5UyH62Wjf8IugN3FTAsWUT/IK5UVI1SXsDJxP8Yit228WYDvwm6eqw4HIUwZdrZjh/\nF4IvkpUEX8b9gMFhAaegfRd2/g4i+HJ+Pea45hEUpr4g+HVzI8F7I88A4Gszm5jAcaQUT/ClK7al\nyVygSfi6BfDP2NIgwRt+ZzObRVAXeQuwRNJLknYuwT5fJP+L5GzyS+9IOkbS1+EFtRVAD2CHBLbZ\npIBj+YOk8yVNjDmWTgluF/JL7QCEJdFlQNOYZRbHvF5H4YmyJUFda14cUwi+MBvHLBPf+mebWwMV\nYw1QJ2a8bjgtkWXzls8OX68DhpnZt2a2keCL6wBJtQtZtx75Xxj3E1SLZMfUQcd+6Z5D8OtsHsGv\nnRcIqgeLiiv+y+hPwi/YngS/vhYR/Ip4NWbb64Ec4A4z2xwWOEYR/GItaN+Fnb8LgNfDXz3BwUkn\nAP8EOphZFeA84D1JO0tqAvQn/DVd2XiCL13N414vCF/PJSgRx5YGa5nZKwBm9pKZHUzwRWAEF4Mg\nsYtBrwNZkpoSfMBeBJBUjeAi1P8BO5pZfWA4xZewIfiAxh8L4XZbEFwsuxxoEG73p5jtFhfzQoLE\nnLe9mgRfDgsKW6EIc4Hucec108wWxSwTH0+R8Uk6p4AWG3nD6iKqaCaz9S+NPQnOS2HLxrZOqUlQ\nrTc5nBRfTRK/7q551XIx+8pb93DgHkmLCM41wFeSzoQ/Llgeb2Y7mtn+QCNgfMy2O8ftr3PMtotk\nZj+aWZaZNTSzY8Jjytt23jEV9v4r9vxJqgGcytbVMxBUv71vZjPDOEYQvIcPAPYlKFRMCc/J/UBX\nSQsLuAibeqK+CJAqA8FFyh8ISqINgM8JSisAfyFIRl0J3uA1CUo6tQjq6g8nqKapSnBx9tlwvT4E\n9YsqZt/DgZHAtzHTahOUZg8J93kMsJb8i15ZbH3hdDZwePj6LmA0Qclwl/C45oXzOhKUxtoB6QR1\n+znAxeH87uG2qsRs+znCi6zAEcBvBB/gagTVP2Nilv3jomD8ugUc95UEpcDm4Xgj4ITwdctwW2kx\ny19IGV1gC/9XUwh+/TQlSFi9C1m2IUFVxclAdYIv4S9j5h9GcIF7T6AK8D+COvW8+V8B94Trnkxw\n7WGHmG3vGA6Nw3PQFagezm8fvjeqAucCS2PWrULwPh4Q/m8GhP/LjJh9Vw/fA7nhMrENCfYI52cC\nVxFc66gSzssgqMq7IXx9IMEvg3aJnj+CX6i/FHA+exNUEbUieK8fSfBebxce544xwwDga4JCT+R5\no6yHyANIlSH8IFwbvjFXAM/mfajC+UcTlGZWEJSsXiFI8HsA48I3+zKCZmY7hes0IEjwy4EJRez7\n3PAD98+46ZcRVHesIGj69iJbJ/i5cfHnJfgaBKWkFQSlqKvilr0jjHUpQfO0UeQn+CrAe+H838Jp\nzxLTYiX8MM+MOd4mMfO2sHWC32rduOMTQVXAtPD8zST/S7VluK3YBH8BMV8mZfAeuDs8pmUE1xFi\n5/0EnBUzfgQwlaA65lPCL6mY+X0JqjeWE7Q0ahozr0V4zteF2zi8iJjiz+cVBF+wa4AxwN5xy+9F\n0GxzXfh3z5h5LcP3WW643VxiEi7BF9Vygqqm92P3G87vSNCSZU14Pk5M9PyF8z8Ebi1gelq473nk\nt+w6p5DzUabvgWQbFB50oSR1J/hZkw48ZWZ3x80/Ebgt5h9/tZl9Gs6bE57wLUCOmXUtcmfOOedK\nTZEJPmxeNB3oRlBH+g1BKWRqzDI1LWyyJWkP4E0zaxOOzwb+YmbLy+4QnHPOFaS4i6xdgZlmNseC\nq+QvE9xt9gfbuj1uLYIbSGKl/oUM55xLQsUl+KZs3axsPls3ZwNAUk9JUwlurx4QM8uAjxU806LX\n9gbrnHMucX96PkecRJrpYWZvAW9JOhh4Hsi7eeFAM1skqREwUtI0M6s0D/pxzrkoFZfgFxDcVZen\nGfk3LvyJmY2VlCFpBzNbZmF7ZDNbKulNgiqfrRK8pIS+RJxzzm3NzIquAi+qiQ3BF8AsguZRVQke\nltUhbpnW5F+s3RuYFb7OBGqHr2sS3EZ8VAH7sKibEhUQ0y1Rx1BR4vKYPKbKEFeSxmTFLVNkCd7M\nNkvqR/BUuHSCZ4hMjXmM5+PAKcD5knII2reeGa6+EzAsvFksAxhqZh/F78M551zZKK6KBjP7gODi\naey0x2Ne/x/BTQbx6/1C4Q+Jcs45V8b8WTQFGx11AIUYHXUABRgddQAFGB11AAUYHXUABRgddQCF\nGB11AAUYHXUA26LYO1nLPADJrLgLBc4557aSSO4storGOedKm7eeK5ltLQR7gnfORcJ/uSdme74M\nvQ7eOedSlCd455xLUZ7gnXMuRXmCd865FOUJ3jnnypCklpJyJZV7vvVWNM65pCJ16gHNBkDt6pC9\nAeY9YPbT8PJav2SxKt3MtsSMC8KHxBSweKLbKS2e4J1zSSNIzvsPgifb5E/t1VrqRCJJenvXz9+O\nmgAPAgcTPGPrf2b2oKRbgE4EHc+fAPxD0nkED1PMArqE83+J2dyY8O/KsMnjUQSdn/ci6I/5fOAR\n4KZE40uUV9HEkaRW0sC8b2LnXHlqNmDr5AzBePP+5bM+hFUp7wITgSYEHaRfKemocJETgNfMrC4w\nNJx2DnApQa92c+M2eXD4t66Z1TGzr8PxrgRP690RuDPR+ErCS/BxWsPJR8JlI4P+Z4dFHY9zlUvt\n6gVP79pdSqQDov0KmV6rRgmC2BdoaGZ3hOOzJT1F8KTcX4EvzewdADPbEJbKn7P8vqpz47ZXWGFx\noZk9HL7eUIL4EuYl+FATqXd7aXIW3PkI1MmCge2lyU2k3lHH5lzlkV1Iohv/oRkqboBxIwpef836\nEgTRAmgiaUXeAFxPUNKGgjs9+qNrU0lrJGVLWi1plyL2M6+IeaXCE3xoETyZAzdXh3oCMqDGZrhp\nETwZdWzOVR7zHoBeM7eeduksmPtg+awPBFUss82sfsxQx8yOC+cX9Evij2lmVsvMaofrzC9k+cK2\nU6q8iiZkZtZY9bqksWrHAWCgpqup08Vs5RtRx+ZcZWH203CpE9Cjf1CtsmY9zH0w0Quk27t+aDyQ\nLekaggutm4AOQCHVR0ARLWSApQTVNq2BGSWIY7t5gg9JnXpkUb9/R+rxBbdqNx6UsaWP1OnLsmpi\n5Zz7s/Dzts2fue1f33IlHQfcR9AaphowDbgxb5GCVitie+sk/Qf4QlIGcEy4fJmX4P158KF0df9w\nEguO/hd3Mpos5tGMNsxkGed/aDb8mKjjcy6VJMvnviIo7Fwlcg69Dj7Ul9+ar6Um73Eca6jNBxzD\nqbxOCa++O+dc0vAEDyBVuZ4pLf7FneRVpb3I2ZzNi5Tw6rtzziUNT/CBC9PJnTmKEZvyJozgaDox\nLvdkfnolysCcc25b+UVWqTpw44088gY03gI9lkCtGjnssvu3tJj/BjMaRx2ic85tC7/IKl25lsxj\narH2L8BBZkwLJnPmCbz9j7fpWQ2zPSOLz7kUFPnnvgLZnouslTvBS7WAmd0YOf4Tuk0z45r8WdRI\nY8v8jVTbkMGWozH7KZIYnUtBnuAT561ott2A+TT98RO67Q3cHjvDjPW5pL82gqNnAmdHE55zzm27\nyluCl+obzNiT75f+SOfbzHjpz4vw130Z//I49kPQiqhPlnMpwkvwifMS/La56hv2nf4jnZcALxey\nzLgJ7LN+LTVzgQPKMTbnnNtulTPBS41z0d/O4JX2QD+zgm8ZNsOMtOde5Ozf8Woa59w2iLLLvmJ3\nKKm7pGmSZki6toD5J0r6XtJESd9KOjzRdSN0/ducuGgOrYaYUdzF0xf+j2vaGZyGVKU8gnOustve\njnfKq+MeSenx+y1in+VfJWVmhQ5AOjATaAlUASYBHeKWqRnzeg9gZqLrhstZUTGU+gDNN5Gxqgnz\nF4PVTWQdsA8W0fhngx7lGqsPPqToUNznvjWc0hdWtYaTt2X727s+QU9ObwC/ETxwrH84/RbgdeB5\nYBVwCTAa+A9Bt33rgF3jtjWX4GmS2eGwXzj9YmAKsBz4EGheknOVSO4srgTfNUzYc8wsh6Cu+sS4\nL4i1MaO1gN8TXTcKueimJ+i9biFNrzZjVYKrPfcYfXPxahrnytT2drxTGh33lHGXfbXNbJykEwk6\nETkJaAiMhT839NhexSX4pmzd68j8cNpWJPWUNBX4ABhQknXLldR2I9XOuI2bfgVeKMGabz9G3x1z\n0fFINcsqPOcqu7yOd9KhuoBG0O5+6LgAHkey4oYF8Pj/oGMjaCcgHapvQ8c9f3TZZ2abzWw2kNdl\nnxHXZV847Tkzm2pmuWa2OW57BVXN9AUGmtl0M8sFBgJ7SWpWohNWjOISfELNAs3sLTPrABwPPJ+s\nHVavo8bAe7hav9G4j1niz2I2Y8MSdnplGu2XEHx7O+fKgJmZwNKg3ukwZQVk94NTZCYSGGSm/nDq\nSlh9OkwR1A83W5ImzuXRZV8LYFDM9peF00u1EFzcs2gWALHfKM0o+OAAMLOx4QPtG4TLJbSupFti\nRkeb2ehi4io5qXMOdXo8RL8hZny/DVt4dhBXnPQYfc9WGfyUcs4FqkLbkXDRLHizNZxUFdqW5/rk\nd9nXLn6GpJtJoMu+uHVaFLKP280s4VwiKQvISnT5vGCKutCQAcwiuFBalYIvsrYm/4apvYFZia6b\n6IWC0hh+o+Hoa7hrNVj9bbvoYqrLimk5pGcb7FAeMfvgQ6oO5fW538bY0oBvgWuAGgQNRjoB+wA3\nA8/HLT8KuKSI7WUCm4G2MdN6Aj8CHcPxusBpJTlXiZzDIqtoLKhL6geMILja+4qZTZXUR1KfcLFT\ngB8lTQQGEdRTFbpuMd83ZWKDqv91C+n7P80lV5mxYlu2YYatot4zX3LAb8BppRyicy5JWFAnfhyw\nF0ELmqXAEwRJGLahyz7CVjZhlUxXM3sLuBt4WdIqgmR/dOkdRaBSPKrgV7WY/BD9atzL1W3MyN3W\n7Ug0OYlh01/l9O8zbPNBpRmjc5WJP6ogcdvzqIKUfx78FHU8vgZpuw3h/AO3J7kDmLGwho75fDMZ\nB2VIzTGLbw7lnHNJI7UfVSCpCjlPPMWlo5dY43GlsckN1Hj2XY5fRVgV5ZxzySqlE/w7HN9vMxmN\nXuGMs0pzs4/Rt9ZGql5Yitt0zrlSl7IJ/jWdlrYb0we+zYlPzLQ2S0tru2Zs+IxDh26gelOkTqW1\nXeecK20pm+B/p+GgTVTVh3TvX9rb3kLGc0M4P9egNH8ZOOdcqUrJBH+Avqx/JCP7fsfe1422rC1l\nsIsJQzln+QaqX0SS3rXrnHMp2UxyoK4fcTzvdu7E5CaU0QGma8vVv9LiX7uw4FjMviyLfTiXqiR5\n72glsK3NJFOuBN9WM/Y6h6Hd1lLz0rJK7gC5pL/wJL2qb6Da+WW1D+dSlZnJh8SHbT3PKZXgJXQ2\nL766gepT97Nx75flvsxY9CYnfWXoLO8IxDmXjFIqwTdj7vn9eKjVjvx2YXns70c6PzqDtgZ0K4/9\nOedcSaRMgpeodSHPPbCJqmPq2coJ5bTbd57loowV1OtVTvtzzrmEpcxF1maad/9kdu9bh+y9MJtW\nGrEloo1mPv0je5xdgw0N2bp3K+ecKzOV5iKrRPvLeKQX8FZ5JneAWbR59Cv2t1x0fHnu1znnilPh\nE7yEmjL/sSu5nzpkXxdBCN++xmnLltC4XwT7ds65QlX4BA+cfCO3d6pCzhDM5pT3zs2wdzjhsTqs\n3hdph/Lev3POFaZC18FL1GzBnOk/0652VXLaY7aotONLMI6dX+PUX4/gk6vq24oHoojBOVe5VIY6\n+OsHccX6quQ8HlVyh6BN/HB6fL+WmpdFFYNzzsWrsCV4ibZt+XncVDqQTm5bzJYVv1bZaaDlZ8ym\n1Qt1Wd3aOwJxzpW1lC3BSwgYNJRzZqeTOyjq5A6wggZvDePkzbNp+beoY3HOOaigCR44vgvf7bYP\nE3YB/hd1MABmbPyUw4dXZdOFUcfinHNQAatoJGoAk+fSbEEz5r+N2b1lGF6JZGrdPrNo/XVdVu2V\naet+ijoe51zqStUqmmuOYsQvzZi/K/Bw1MHEWk/mt2/Rc8XPtLs26licc65CJXiJVkD/NzglA7gD\ns/VRxxTLDBtN1pAmLOzpHYE456JWoRI8cH9vHh9Wi7XNgaejDqYg73HcPctpkDmT1odHHYtzrnKr\nMAleoofI7fgIl3UEbsFsU9QxFWSt1Vw8kiN/XsYO10cdi3OucqsQCV6iGjDoQfoPSSe3ATA06piK\nMpqsB9rx8yHeEYhzLkoVohWNxL/T2LLvFjKaA//B7I1yCm+bSFQdR9e166nR61D77Lmo43HOpZ6U\naEUj0Rz4+1fs/xFgwLCIQyqWGZu+Yv8vqrLp71HH4pyrvJI+wQP/rcKmh7ryTT/ghrLsSLs0fcGB\nd+zO5E5T1LFm1LE45yqnYhO8pO6SpkmaIelP7bslnSPpe0k/SPpCUueYeXPC6RMljS9pcBJHAnsv\npdE8YBnwYUm3EZXXOP2TSey1bgodr4k6Fudc5VRkgpeUDjwEdAc6AmdJ6hC32C/AIWbWGbgdeCJm\nngFZZtbFzLqWJDCJqsCDDVh2VV1W3wD8u6KU3iFoEz+RLu82Y95FUcfinKuciivBdwVmmtkcM8sB\nXgZOjF3AzL4ys1Xh6Dhgl7htbOsNP1cCs5bSqDEwHbMx27idyHzPnv9uz7Rmg3VB86hjcc5VPsUl\n+KbAvJjx+eG0wlwCDI8ZN+BjSRMk9Uo0KIldgGv258tr07AbgBsSXTeZPGMXz57APgvTyL016lic\nc5VPRjHzE64SkXQYcDFwYMzkA81skaRGwEhJ08xsbAHr3hIzOhqsD/DYlxzYHfgaswmJxpFsprPb\nkP0Y9zfAq2qcc9tMUhaQVZJ1ikvwC4BmMePNCErx8TvuDDwJdDezFXnTLexlycyWSnqToMrnTwne\nzG7J3xaHAQc8Rp8rgR+BwxI9mGT0Ow3vaMXsa2/SbQfeZjd9EXU8zrmKycxGA6PzxiXdXNw6Rd7o\nJCkDmA4cASwExgNnmdnUmGWaA58C55rZ1zHTM4F0M8uWVBP4CLjVzD6K28cfjfUlqgATgZsM7QG0\nwey84g4i2X2ko35cSJPFF9pzR0Ydi3MuNSRyo1ORJXgz2yypHzACSAeeNrOpkvqE8x8HbgLqA4+G\nD1DMCVvM7AQMC6dlAEPjk3sB+gELs6n1GUFrnP2KWb5CWMxOg/blm0ck0szIjToe51zlkDSPKpDY\nGfgBOMjQJUAdzPpGGlwpeU2npR3M2A13c22f/9nfn406HudcxZfQY16SKMEPARYaeoCg7r0zZgsi\nDa4UjdSRny6kyQ4X2OA9o47FOVfxbXcVTXmROIjgYmoH4G7guVRK7gDC7jiEMSN30LLay2yH7Kjj\ncc6lviR5Fs3Vw+GeoYYaAWcCd0UdUWnrxiej0shdew5DvTs/51y5SJIEf09t+PmUn6n6BPAwZkuj\njqjUmdlsWr3blfHeHt45Vy6Sog4ejPZM5Ss659RjcyPyH32QUmarVfuarJ2yH+M6zLZW06OOxzlX\ncVWo58Hfys08Tut5qZrcAVrZ7GnZ1F5yAu/cFHUszrnUlxQJfi8mchCfM5Bms6KOpaytoP7gA/jy\nRCk5zr1zLnUlRRXNe/TgM5b+fg/rLjD7aXjxa1VgUuPV1F7QhYnHz7LWH0QdjnOuYqowVTR7M3L9\nYFZfkvLJHcBsyRIaz+jJW9dFHYpzLrUlRYJ/hpynl9i0d6KOo7xUIeehwxh1gESdqGNxzqWupEjw\ns+Ho9tLkJlLvqGMpDy359fnDGEVHJl8QdSzOudSVFAk+HapthpsWBY8cTn1mqxez07jjeG9A1KE4\n51JXUiR4BU+jNIv6im852onF/z2ed1tItIk6FudcakqKBP8xXFgV2kYdR3mqybr3uzBxy9582z/q\nWJxzqSkpEvxMs2FTzO6OOo5yZbZxJfWG92D4+d4m3jlXFjyxRKgpCx88mxerUcG7JXTOJSdP8NEa\nuwvzNx3E2CuiDsQ5l3o8wUfJbMsW0ocey/tHSdSNOhznXGrxBB+xeqx66gIGb0ljy2lRx+KcSy2e\n4KM3KZN1yw/nU29N45wrVZ7go2ZmNVj/1Km83laqXE1FnXNlyxN8EqhKzgtn8ZJVY8OFUcfinEsd\nnuCTgdksYbOO4qNeEulRh+OcSw2e4JNEbdY8dRHPpuNt4p1zpSQpOvwo7qH1lYLUeAPVZu/E4ndW\nWr0zow7HOZfcKkyHHw4wWwJ83YPhx3ubeOdcafAEn0Sqs3HwZTyyCjg96liccxWfV9EkE6lODhmL\ndmH+T0us8X5Rh+OcS16lUkUjqbukaZJmSLq2gPnnSPpe0g+SvpDUOdF1XRyz1WnkDj+V19tLtIs6\nHOdcxVZkgpeUDjwEdAc6AmdJ6hC32C/AIWbWGbgdeKIE67o46eQO7c+DqwHvzs85t12KK8F3BWaa\n2RwzywFeBk6MXcDMvjKzVeHoOGCXRNd1BfqgDTNrt2T2Rd4m3jm3PYpL8E2BeTHj88NphbkEGL6N\n6zoAs40ZbHntIp7dAhwRdTjOuYoro5j5CV+BlXQYcDFw4Dase0vM6GgzG53ouinqxd480f1mbrsQ\n+CjqYJxz0ZOUBWSVZJ3iEvwCoFnMeDOCknj8jjsDTwLdzWxFSdYFMLNbEoy3shjTiKUZu/PT8VKn\nemasjDog51y0woLv6LxxSTcXt05xVTQTgLaSWkqqCpwBvBO7gKTmwDDgXDObWZJ1XSHMtqSTO/RK\n7p+Ht4l3zm2jYtvBSzoGuB9IB542s4GS+gCY2eOSngJOAuaGq+SYWdfC1i1g+94OviDS3muo+X4d\nVs/OtbQDog7HOZdcEsmdfqNTspJkMPUQxjT8nIMPNGN61CE555KHP4umIgv+c0Ov465f8Tbxzrlt\n4CX4ZCa1ySHj60zWbdhMlRZmbIk6JOdccvAqmlQgjevJjU3fZtZiyFkD2Rtg3gNmPw0vfmXnXKpK\nJHcW10zSRWwENSeeziud32Z6zE1ivVpLnfAk75writfBJ7m+/KXdMfxWvQbrYqY+2Qaa948sKOdc\nheAJPskc74NBAAAb7UlEQVTNoXHaN+zLsbxLPa4j/wbh3TpLHCtRI8r4nHPJyxN80sveMJRz2IP7\nOJlHyGRYOH1NNnANsETiXYm+0lZ3DjvnKjlP8EmuPp///CX9c9YxgafI5jT604Y6OfV5+SMzDgVa\nAEOBg4BJEpMk7pDY359G6Vzl5gk+ya1gzRWrSLtnKRkbBdRnce59ZOf+zpospAcNdTM0yoxzgcZA\nP4I7h58AFkkMljjN+3l1rvLxBJ/kzMzqsOq7quRsPB2mrMfW/gvOS4NLCR4PcT4wDWmaoYcNtTD0\nsBl7EDyTfzzBUz7nS3wq8Q/vLcq5ysHbwVcAHaXrNsHPs+DN1nBSVWg7xezuPxYIes/aAzgUOCQc\nsoHPgDHAZzuyZMlSdjwCOA44FlgLvAe8D4w1Y1O5HpRzbrv4jU6VlSSgA/kJ/1AglzDhb6LKmFqs\nqZ5D1eMIEv5uwEiChP+BGb9FE7hzLlGe4F0gSPit2TrhZxKW7j+g+08n8nbLHKoeC3QDphEk+/eA\n780S77zFOVc+PMG7wkktyK/OORRoCHy+gWqfD+T6NXdzbbuNVD8OqE5QjfMe8IlZcMeV1KkHNBsA\ntav74xOcK3+e4F3ipJ3JT/aHAM0MvppB28n38c+qg7lgj41U/wswBh6eAz/2gEdb1uN6VjIQ6D0T\nvrrCk7xz5cMTvNt2UiOCtvV5Sb/dJqp89wUHLr6LOoeN4eWGabzPmVzMyzzLOk4BenxoNvyYSON2\nrpLwBO9Kj1SPoEP1Q++kev/P2Vh9N6rxXzZwGm35liosJf3nNfbDblGH6lxl4B1+uNJjthKz9zG7\n5t8c+tk4BrORqgioxgZ+41bW0vSXqMN0zuXzBO+2wfwHNvH0oo0YA0inBstJ46HFMO/BqCNzzuXz\nKhq3TVqo5rMraLXn7WxqNJdNTV4j+6W5tuzcqONyrrLwDj9cmfnV1l4EgNRoDTXnj2LMzIhDcs7F\n8Soat33Mlv7Gjm+dz5C/S1SJOhznXD6vonHbT2q5ijoz9mNc72nW/tmow3GuMvBWNK58mM1ZQuMv\nzmfIbRL+Ze1ckvASvCsV61Vjz9XU+bYr44/+1Vp8EnU8zqU6L8G7clPD1n+/irrTLmDwvVHH4pwL\neIJ3paYaG/9xAYP3aKjfvUMR55KAJ3hXalowd6SwxRfzzKCoY3HOJZDgJXWXNE3SDEnXFjC/vaSv\nJG2Q9M+4eXMk/SBpoqTxpRm4S0Jmtpo6N5zJy0fW0eoGUYfjXGVXZIJX0BXcQ0B3oCNwlqQOcYst\nA/oDBdW9GpBlZl3MrGspxOuS3F58P6QRS9dczDP/F3UszlV2xZXguwIzzWyOmeUALwMnxi5gZkvN\nbAKQU8g2vIVMZWKW+yst7uvJW+dKVI06HOcqs+ISfFNgXsz4/HBaogz4WNIESb1KGpyrmA7ii7s7\nMoU+PPanKj3nXPkp7lk029tI/kAzW6Sg84iRkqaZ2dj4hSTdEjM62sxGb+d+XZTMNv2gboOP591/\nSH3v8D5dndt+krKArJKsU1yCXwA0ixlvRlCKT4iZLQr/LpX0JkGVz58SvJndkug2XcXQitlX1WfF\nJZfy5LnQ6/mo43GuogsLvqPzxiXdXNw6xVXRTADaSmopqSpwBvBOIctuVdcuKVNS7fB1TeAo4Mfi\nAnKpobXNyv6GfT84hg/uiDoW5yqrIhO8mW0G+gEjgCnAK2Y2VVIfSX0AJO0kaR7wd+AGSXMl1QJ2\nAsZKmgSMA94zs4/K8mBccjF02WGMatZHjx8cdSzOVUb+LBpXpkbqyHG/07DqWfZSl6hjcS6V+LNo\nXOQW0PSyoxmx5xUatGvUsThX2XgJ3pW5UTpsxnx2mX2ePX9U1LE4lyq8BO+Swg90vuZIRh5xm26q\nG3UszlUmnuBdmbvCBr05i9bLWjLHH0LmXDnyBO/KxVfsP/AQxpw1WlnpUcfiXGXhCd6VizXUun8F\n9TfPo9ktUcfiXGXhF1ldublXV91/HO9d1J7p9Yj6jedcBecXWV1SGcbJ1wnLfI1TL4k6FucqA0/w\nrtx8aQdseJ9j327LjFujjsW5ysATvCtXb3Niv4b8vvNHOsrbxDtXxjzBu3L1mR26+E1O+rI+K+6P\nOhbnUp0neFfu3uCUfq2Y3X6S9toj6licS2We4F25G21Zk97glBkbqfZw1LE4l8o8wbtIvMZpV+/G\n9AOXq0GTqGNxLlV5gneR+IRu775Fz2WzaO2PL3CujHiCd5Eww17l9Dva8XNPJH8ImXNlwO9kdZGR\nqPIap65sw8wn97JJV0Ydj3MVid/J6pKaGTkvcdbjzZnbG6l61PE4l2q8BO8iJVFvBEf91orZ/25r\nM+6JOh7nKgovwbukZ8bKIZz/dl1WXY/kjxJ2rhR5gneRe51Tr51Dy9pL2PGsqGNxLpV4gneR22DV\nf3mKS8fnknYnklfXOVdKPMG7pPA8512zhlo7rad6t6hjcS5VeIJ3SWG91fjiYS6fu5wG90Ydi3Op\nwhO8SxpDOP/GDDa3R/pL1LGkOklqJQ2UV4mlNE/wLmmsoMFrg7hizUJ2vivqWFJdazi5O1zWGk6K\nOhZXdjzBu6RhxubnuPDumqw9GKlN1PGkoiZS7w7S5KPg/kegThYMbC9NbiL1jjo2V/o8wbuksogm\njz3K33KX0tC79StNUlWkY+ZD17thl1qwo4C6sEsVuG0RPBl1iK70FZvgJXWXNE3SDEnXFjC/vaSv\nJG2Q9M+SrOtcPDNWP8Wlz2ay7mSknaKOp0KTaiD1RHoeWAzckAZT7qLm88tJSz+WzHUbocYA0u4w\n8EdFpKAiE7yCOwsfAroDHYGzJHWIW2wZ0B+4dxvWde5PZtHm7hc4l2xqXRN1LBWOVBvpTKTXCJL6\nAOBroBNmB4rdp82g7Zkv8Wr6cNZkDuFl/UCdZktIH4dUJ9rgXWkrrgTfFZhpZnPMLAd4GTgxdgEz\nW2pmE4Cckq7rXEHMmPsofxuZweY+/ijhBEgNkC5AegdYAJwPfAi0xuxwzB7GbGGwcLMBvzNxh3Wc\nAohszuBhllUbQ6N6wCdIO0R1GK70FZfgmwLzYsbnh9MSsT3rukrue/a67X2OtRwy/hZ1LElJaozU\nB+kjYA7QE3gVaI5ZD8yeFrZMoolED4l/SbwGex8SvykjjTM46BfgE2AMkveylSKKS/Db86jJaB9T\n6So0Myb8l39Mz6HKNf4o4ZDUDGkA0mfAdCALeAJoIuxUYROEdZe4W2IEQRXND8DfgfrAWzD9u4I2\nbazdgNl1wPPAWKRW5XFIrmxlFDN/AdAsZrwZQUk8EQmvK+mWmNHRZjY6wX24FPYVB9w+gX2eO4jP\nz0urrK08pNbAKeHQBnj3Nxo98Fe+XjKbXTsCRwD/BPYAlgCTwuHB8O8Cs/zCljRtBfQaBE/GNEO9\nej1cUlWiipndhbSaoCR/NGZTyudAXXEkZRF8qSe+TlHPg5eUQVBSOAJYCIwHzjKzqQUsewuQbWb3\nlWRdfx68K4xE+pF8NOctepLJ+paYbYk6pjIX3FnakSChn7yFtJ1n0Hb8k/Ra+DCX195I9T2BVgSf\nrUnAxPDv92asSmwXnXpA8/5QqwasWQ+rH4fPexFcRzvTjA1I5wL3AMdh9m0ZHKnbTonkzmI7/JB0\nDHA/kA48bWYDJfUBMLPHFTRl+waoA+QC2UBHM1tT0LrbEqSrvNK15fIf2eO2jkztg9nrUcdTUkEy\nbTYAaleH7A0w7wGzn4bHLaQcMrosofEltck+0VDtDzhm2dNcUms0WdW2kBGbyCcBU8zYVLpxUhV4\ngaAqp6cZa5F6ElQBnYLZ2NLcn9t+pZLgy5oneFcUiZqn8PrC5znv1xps2JOo37AlECT3/QfBE23q\ncT0rGQj0ngk//DOTT+f14slTD+WzY/ZjXIcNVK/6LsevHU6P70dy5CgjLS+Zz46tYinbeEknSOjt\ngWPNWInUDXgROB+zD8sjDpcYT/AuJWRo851zad63CYtOw+yTqONJlHTMh/DB0Zm8zplczKs8xT40\n4lSutlOZt2Uj1dZOpMv4zzno9cfoOyzbav8efcykAf8FDgWONuM3pP2Bt4DLK+KvqFTlCd6lBIkm\nl/LkjIfoN66abTw86ngSVUudpzdiS7sDWMsL/Mp1pDGBqjSm9rIXWXogZtOjjrEgEgJuBU4Hupkx\nH2lP4APg35g9G2mADvA+WV2KMGPhUM55cwPV964ojxKWaFWPfRudyo40ZR4CFtGYr3mel/jLN8ma\n3AHMMDNuAp4Gxkq0xux7ghYctyBdEWmALmGe4F2FsJ7Me+/iOjaTfl3UsRRFIn0Xzb/uLq6dPJWh\nmeuYuWYZmRxNRzJYQxoPLYZ5D0YdZyLMuAe4C/hMYnfMfgYOBi5Husm7V0x+XkXjKowGWj56IU32\nrs7GvTGbGXU88ZpoYZfzGTLsKu5tAgxvyLIrWpB56wpa7ZlNh9W1mVqnAbMnzbG1F0cda0lInE1Q\nL3+cGROQGgMfAR8DV1WkC9+pxOvgXUqROPZ+rnhqAA+8LbO+UceTp6XmVDuDV4ZcwtOnVGXTz41Z\ncm4NW1/gHaMVlcSJBDebnWLGWKT6wHDgJ6BvpbhHIcl4gncpRSJtJxZNm0vznaqwuR1mi6OO6S5d\nd8lhjHqgMUs2/07Dy/bh2xdTtUQr8UeTSTM+RKoFvA0sJWhGWapt813R/CKrSylm5C5m53uHcfIy\nINILfSN0dKcxOuTnCxj8xK+0GNySX3fYxyYMTdXkDmDGxwQPNRsscQpma4BjgRrAm0g1Ig3Q/YmX\n4F2FIlGjNTPnTqN9RgZbWmKW0O35pRhA/e/o8lQrZvccxsmTxtP1hMetz4JyjSFiEnsRVM9cb8Zg\npCrAs8AuwAmYrY40wErCS/Au5ZixfhZtHvmCA38H+pTbjqVqC9XkxlXUWfQTnbqdx/NnXGJP/6Wy\nJXcAMyYBhwO3S/Qj6O/hfGAq/kz5pOIleFfhSDTei4kzJrDPunRyW2K2oQx3plx0+hpqPfgV+9e9\nj3++MJKj+puxrsz2WUFItCRoSfO0GQPDZpMDgeOBI/M7GXFlwUvwLiWZsWQSXV6dRvvVwHlltiPp\noPVU/24GbZ84mxdXd2fEAR/ZUZd4cg+YMYegXfw5EncJw58pn1y8BO8qJIndj+DjMR9x1PI0rH2p\nNtOT2uWiu7OpfegVDKoylHPu3EyVe83+1C2lAyR2IOgi8Bugnxm5SJcB1wP+TPky4iV4l7LMmPwp\nh3+zhMYCTiqVjUqNkB7aTPrXd3Hd3k1Y+NNgLvxLjlUZ6Mm9cGYsI+j3YXfgOYkMzB4hSPCfVJTH\nSxRGklpJA1UB79z1BO8qLCPtvqu4t4rBtdt127yUifQvg6kjOGqfJizM/Td3/mcdNbPM+LkUQ05Z\nZqwGjgEaAa9KVMPsBeBvwAdIB0ca4HZoDSd3h8tal1ZBohx5FY2rsCSUxpYfVlKvbm3WXFTiRwlL\n6cC5wO0L2XlmNz5uNpWOk4HLzah0rWNKQ9hxyFCgLnBS2HFIxXumvNTkb3DbVDjlr5A5EKr+E1Z+\nDmtrwqOj4M6o73nwKhqX0sywXNL/ezs3rgKuLdHKQdL5djPpfzuDl8c1ZWH7qXS8niApeXLfRmFP\nU2cR9Mk8QqIeZh8DJwKDkU6NNMDCSI2RzkB6DGk68OPD0KAHvLkalomgu7r+MPeToHnu70gfIN2C\n1AOpYbQHUDAvwbsKTaJaVTbOyaY2Vckpvv9QaQ/g/4A2z3HBqxfzzLlG2sfA1WYsL4+YK4Ow45D/\nEbSyOdqMpUn1THmpEUGnJoeFw87AGGA0MAr4AbPcHVXvPyeQfe1Cqm9swoZq71D7rt9s5Q1IOwNd\nw2E/YF/gd4K+p8eFfyditr7sDsGfReMqAYkb7ueK46/ggTmYnVHIQk2A24DjF7Lz/W2Y2Xk9mV2B\n3mZUmF6iKpKw45DbgFMJOg5ZgNQOGAn8F7NB5RjMDsAh5Cf05sBY8hP6pPiWWFKnHg2pMmQdN+yw\njpPJZBiZ3PH77+RcUEC/umnAbuQn/K5AB2Aa+Ql/HDAds9zSOSRP8K4SkGhYm9UzVlB/Szq5f93q\nUcJSbeBq4HKDJzsw9ZfptL+NoK32zd6mvexJXAP0JUjyvyA1J7hB6gXg9jKpy5bqsXVC3xX4kiCZ\njwK+w2xz0Zs44VN457A/z+nxodnwYxKIoQawF/kJfz9gB2ACQcIPkr7ZokQPa+vNJ5A7Ley+Jaoh\nCCHaGHyo+APYo69xypgt8FhLGPgNZBj0MVhk8PxArj0A7AOw78H2iTreyjaA9QWbB9bRzDBobPC9\nwX0WFjS3a4C6Bsca3GvwrUG2wUiDfxnsb1AlgRgbgPUEGwT2A9y4Gcz+PFy9HOw6sL+CFbvduDgb\nGvQwuMXgA4NlBvMMXje4xuBQg1qJnVOsuGW8BO9SgsRuO7Lk86fYqfoISO8Gy3rCjDXUvLY2a/YH\nbgLuA/yGpYhInAPcCxxrxnfb9Uz54JfZgeSX0DsSlIjzSujjKebxxRJ5pfyscButCUr5o4Nt9LgN\nhh/15zXP+w6eHxuut+vW6/Bdid5fQfPeXdm6lN8ZmMXW9fmTifnFEbbJzy0ud3qCdymhidS7Bo3v\n787yag+Rk3YJVTaOJW35Im5dv5ZrFwC9zEjaflArC4mewBPAyWZ8Hv9MeUFOS7hzDvzLYpOTVBM4\ngPyEvgfwLfkJfRzFPJNIoi5wUMw22gFfx2xjQmxyljr1gP0HwZNt8rdy6Sz4ekBeHXx4F+/B5H9J\ntCQ/4Y8GvjWjyKqgAgKtSpDkY+vzdwEmEib9nrDD2/CoJ3hXKUi796jJzkNO5OcdhjKPc2jG2/wl\ndy2HPwL9rzCjVC5sue0ncSRBW/lzzfgIqTrwCpCxN7ywHzw2Hvp8GyT9LILEuRcwifxk/BXFtFCR\nqE2Q0PO20YEgQY4OtzHegmadRWyjUw9o3h9q1YA162Hug3+6wLr1PvMu5maFQ0vgC/IT/nclTvjB\nhusB+/wLLv8ajtgPMu+CdE/wrlKQjvkwk0uOPpOLmU8zmjKPV3iWdTyd2AUxV64kDgSGAX3NeLOl\n1LcV3NUVqt0F1f8N9g1sOABG3xo0t/wSs7XFbLMmW1fbdCK4oPlHtY0ZZffk0YJjasjWCb8F8Dn5\nCX9iSRK+JO0Kp3SD+56A5sXlzoxtC9u5ZFO7enVm8DLPktekrTozWEct72UoCZnxhUR3YLhELeDx\nKrBsD3hMUH05LJgNV34Mw24tpBQqkUlQbZNFkND3BL4jSJzXA1+bUWbt0BNhxu8EX2TDACQakZ/w\nnwGaSVsl/ElFJXwzszaSpUG9RPbvCd6liOwNy7n+j7F1nBK2f+wR6QfcFc6MiRKHAx+B1V5Fvabr\nyK7bg+rrmrBh59XU7mK28o285SVqAH8lSOZZwN7A9wSl85uAryzJm72asRR4IxzyEv6hBMfzHLBL\nmPBHkZ/wt7r4nEPmcUNpOQumdCluf15F41JCIhfEXHKSaAUjv9iBm+qu56rM/JuK/rPsd7r/H9yZ\nSZAA9wF+JD/5fWFGkdU2FY3EjuQn/CygKcENWeExd2kC+/wveJ+L7a6Dl9QduB9IB54ys7sLWOYB\ngifJrQMuNLOJ4fQ5wGpgC5BjZl0LWNcTvCsVJb0g5pKH1HMUvJX15zlXr4J7HidIcF+YkV3OoUVK\nojH5N2xlwfVtYWBY81J8gi+uIX06MJPgSnAVgqvYHeKW6QEMD1/vB3wdM2820GB7G+uX9wBkRR1D\nRYnLY/KYSieW00bn30g0KuamotNGRx1bMp0rOOuL/HODFbd8cU+T7ArMNLM5FnSs+zLBU+FinQAM\nDjP1OKCepMYx8yti6Twr6gAKkRV1AAXIijqAAmRFHUABsqIOoABZUQeQLzumdcvomOlrkuUaSlbU\nAQRWlOgXTHEJvikwL2Z8fjgt0WUM+FjSBEm9ShKYc64ymfcA9Jq59bRLZ8HcB6OJJ1kVdJ4KV1wr\nmkSvwBZWSj/IzBYqeDTnSEnTzGxsosE55yoHs5+GS52AHv1hRgcYP9WvofzZ1ueJ7sUtX+RFVkl/\nBW4xs+7h+PUEzz+4O2aZx4DRZvZyOD4NONTMlsRt62ZgjZndFzc92mY8zjlXQdl23ug0AWgrqSWw\nEDiDoLeWWO8A/YCXwy+ElWa2RFImwa202QqeI3EUcGtJA3TOObdtikzwZrZZUj9gBEGLmqfNbKqk\nPuH8x81suKQekmYCa4GLwtV3AoaFfSFnAEPN7KOyOhDnnHNbi/xGJ+ecc2Uj0k63JXWXNE3SDEkl\n6zS5bOJ5RtISST9GHUseSc0kjZI0WdJPkgYkQUzVJY2TNEnSFEkDo44pj6R0SRMlvRt1LHkkzZH0\nQxjX+KjjAZBUT9LrkqaG/8O/RhzPbuH5yRtWJcl7/frws/ejpBclVUuCmK4I4/lJ0hVFLhzhjQPF\n3kQVQUwHA12AH6O+oSEmpp2AvcLXtYDpUZ+nMJbM8G8GwTO1D4o6pjCefxA8ivadqGOJianYG/4i\niGkwcHHM/7Bu1DHFxJYGLAKaRRxHS+AXoFo4/gpwQcQxdSJ4XEP1MIeOBFoXtnyUJfhEbqIqVxY0\n4VwRZQzxzGyxmU0KX68BpgJNoo0KzCzvoU5VCd5oyyMMBwBJuxDcWf0UyXeDXdLEI6kucLCZPQPB\ntTYzWxVxWLG6AbPMbF6xS5at1UAOkCkpA8gEFkQbEu2BcWa2wYIesD4DTi5s4SgTfCI3UbkYYWum\nLgTdeEVKUpqkScASYJSZTYk6JoLnhl8NSde5R7Ld8NcKWCrpWUnfSXoybPWWLM4EXow6CDNbTtDN\n41yCVoQrzezjaKPiJ+BgSQ3C/9mxBL09FSjKBO9Xd0tAQddmrwNXhCX5SJlZrpntRfDmOkRSVpTx\nSDoO+M2CB90lTWk5dKCZdSF4IN/lkg6OOJ4MgkftPmJmexO0frsu2pACCrqrOx54LQliaQ1cSVBV\n0wSoJemcKGMys2nA3cBHwAcE3fgVWqCJMsEvAJrFjDcjKMW7OJKqEDw/+gUzeyvqeGKFP+3fJ3iU\na5QOAE6QNBt4CThc0pCIYwLAzBaFf5cCbxJUT0ZpPjDfzL4Jx18nSPjJ4Bjg2/BcRW0f4EszW2ZB\nh9fDCN5nkTKzZ8xsHzM7FFgJhfc1HGWC/+MmqvBb+wyCm6ZcjLD39KeBKWZ2f9TxAEhqqKCPSCTV\nAI4kKElExsz+ZWbNzKwVwU/8T83s/ChjApCUKal2+Drvhr9IW2mZ2WJgnqR24aRuwOQIQ4p1FsEX\ndDKYBvxVUo3wc9gNiLwqUtKO4d/mwEkUUZ0VWY9OVshNVFHFAyDpJYKH7e8gaR5wk5k9G2VMBH1M\nngv8ICkviV5vZh9GGNPOwGBJaQSFhOfN7JMI4ylIslQBNgbeTMIb/voDQ8PC1Szyb1CMTPgF2A1I\nhusUmNn34a/ACQTVIN8BT0QbFQCvS9qB4ALwZWa2urAF/UYn55xLUZHe6OScc67seIJ3zrkU5Qne\nOedSlCd455xLUZ7gnXMuRXmCd865FOUJ3jnnUpQneOecS1H/D/Nx5BTiYq37AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x16e5e320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.plot(err_list_tr,'bo-',label='err-tr')\n",
    "plt.plot(err_list_val,'r*-',label='err-te')\n",
    "plt.legend()\n",
    "plt.title('best validation err = ' + str(best_err))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}