{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LR Search.ipynb",
      "version": "0.3.2",
      "views": {},
      "default_view": {},
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "metadata": {
        "id": "m0HxmCDXrCZJ",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "%matplotlib inline"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-gr4suLdrEBI",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "plt.rcParams['figure.figsize'] = (7,7) # Make the figures a bit bigger\n",
        "\n",
        "from keras.datasets import mnist\n",
        "from keras.models import Sequential\n",
        "from keras.layers.core import Dense, Dropout, Activation\n",
        "from keras.utils import np_utils"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "kQApfw9IrFvP",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "d6ff9f55-5e2d-43fe-e89b-c26af744bf0d",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528032009983,
          "user_tz": -120,
          "elapsed": 710,
          "user": {
            "displayName": "Jannes Klaas",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "115843019312458301179"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "nb_classes = 10\n",
        "\n",
        "# the data, shuffled and split between tran and test sets\n",
        "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
        "print(\"X_train original shape\", X_train.shape)\n",
        "print(\"y_train original shape\", y_train.shape)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "X_train original shape (60000, 28, 28)\n",
            "y_train original shape (60000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "oDUpOuX0rH0_",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 443
        },
        "outputId": "be281d82-334b-4910-ec13-95558840341f",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528032011395,
          "user_tz": -120,
          "elapsed": 1218,
          "user": {
            "displayName": "Jannes Klaas",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "115843019312458301179"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "for i in range(9):\n",
        "    plt.subplot(3,3,i+1)\n",
        "    plt.imshow(X_train[i], cmap='gray', interpolation='none')\n",
        "    plt.title(\"Class {}\".format(y_train[i]))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAGqCAYAAABERjBFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmcFNW9///XMG6AgqBGUIO4wAd1\ncE2ieFUwsihB0CBqgrjEiD+Non6jN7kRFUVxIcYoqMQYRNHE6FVZjEZRcUlcggteUDkCSVACBhJE\nBBEE5vdHd5M+h57unpleTs+8n4/HPB71qaru+nT3Z/p01ak6VVVbW4uIiEi5tSh3AiIiIqAGSURE\nIqEGSUREoqAGSUREoqAGSUREoqAGSUREorBVuRMoFzOrAi4DfgBsTeK9eAb4H+fcZ2Y2CVjgnLu+\niDm8COwNfJE2+zjn3D+KtU2pn0jqZEdgIlADrAeuc849UqztSf3FUCdpubQG3gMmOedGFXt7hdSc\n95BuAk4D+jnnDDgQ2AZ4MllcpXKmc65b2p8ao7jEUCc3AR8557oCxwPjzWz3Em1b8hNDnaSMKvH2\nCqZZ7iGZWXtgBHBIqgFwzq0xs4uAPkBVsH4PYDzQGtgEjHDOPWdmWwETgKOBauD/gLNJ7PFsMd85\nt6r4r04KJaI6GQL8V3L7i5N71gOBuwv8kqUBIqoTzOxA4DjgwcK/0uJrrntIRwCLnXPz0mc65750\nzk13zm0K1r8HGOuc60bil9CE5Px+wF5AN6ALid3kHlnmZ/L/zOwdM3vXzH7Y+JcmBVT2OjGznYD2\nwMK02QuTj5E4lL1OYPNhw7uBi4ANhXlppdUs95BI/IP/sx7rHwykxlh6hUS/D8ByYH/gZOAZ59xV\nAGb2rUzzM/gDiS+XJ5LrzzSz+c65l+qRmxRPDHXSCtjknPsqbd5aYJd65CXFFUOdAJwPvO+ce9XM\n+tbvJcShue4h/QuozzH4ocBfzMwBM0jugjvn/gJcnPz7xMx+a2Y71jU/fFLn3Fjn3OPOuVrn3HvA\nw8B3GvXKpJBiqJM1QAsz2yZtXitgdYNekRRD2evEzHYlcVLFTxr9asqouTZIrwO7mtmh6TPNbGsz\nu8HMWqXN2x34NfDDZGflCemPcc79r3PuWGBPEl8UV2Sbn/a81WZ2UJDXVsBXSCzKXifOuRUkfjnv\nkza7C/B+YV6iFEDZ6wToDXwNeN/MPgEuBy43swlUkGbZIDnnVgK3AA+Y2b4AyaK5h0THZPpp2LuQ\n+JU6L9npODy5/vZmdo6ZXZV8zhXAPKC2rvkZUnnSzIYkn+/rwHdJHMaTCERUJ48Alyafb3+gJzC1\n0K9XGiaGOnHOPeSca+ec6+Cc6wD8HPi5c+7/K94rL7zm2oeEc26Uma0ApplZNYmzXaYCFwSrvgs8\nBXxI4jjxj4GjgJdInEEz0czmk+hEnE/irBiyzE9tf6OZfRcYZ2bXk9gzutI592qBX6o0QrnrJOln\nwCQzWwB8CZzrnKtPn4UUWSR1UvGqdD8kERGJQbM8ZCciIvFRgyQiIlFQgyQiIlFo8EkNZnYbiSuU\na4FLnHOzCpaVNBmqE8mH6kQAqK2trfdf165de3bt2vXJ5PR+Xbt2fS3b+iSKrHbOnDm1qekY/5Rf\n5r+G1IjqpPnlpzqJ43OIPb9sn21DD9kdB0wBcM59ALQzsza5HlRTU9PAzZWG8is41UkZxJ5fBqqT\nMogxv4YesusAvJUWL0/Oyzia9Zw5cza/+NhPM1d+BaU6KZPY8wuoTsoktvwKdWFs1vt9dO/eHUi8\n+KqqUt8aJH/Kr+7tFojqpARUJ3FQfnVvty4NPWS3hMQvmJTdgKUNfC5pulQnkg/ViQANb5CeBU4B\nSA4ouMQ593nBspKmQnUi+VCdCNCIoYPM7CbgGBJjNv3IOfdunRupqkqcGqNd2EYp4y52gzeqOik9\n1UkclF+d261zoyUZy04FVBgxFlAhqU4KQ3USB+VX53br3KhGahARkSioQRIRkSioQRIRkSioQRIR\nkSioQRIRkSioQRIRkSioQRIRkSioQRIRkSgUanBVEamnww47zIsvuugiLz7zzDO9+IEHHvDicePG\nefHbb79dwOxESk97SCIiEgU1SCIiEgU1SCIiEgX1IeWhurrai9u2bVuvx4d9A61atfJiM/PiH/3o\nR17885//fPP0b3/7W773ve95y7/88ksvvummm7z42muvrVe+UhwHH3ywF8+YMcOL27Tx79odDnw8\nbNgwLx44cKAX77TTTo1NUZqB4447zpt+6KGHvOU9e/b0YudcSfIC7SGJiEgk1CCJiEgU1CCJiEgU\nmkUfUqdOnbx4m2228eIjjzxy8/SZZ57JUUcd5S3fcccdvXjw4MEFzW/x4sVefMcdd3jxySefvHn6\ntNNO4/PP/bs7v/uuf3PNl156qaD5ScN861vf8uLHHnvMi8O+yLDPKPyc169f78Vhn9ERRxzhTYfX\nJYWPb66OOeYYLw7fxyeeeKKU6ZTcN7/5TW961qxZZczGpz0kERGJghokERGJghokERGJQpPsQwqv\n93jhhRe8ONt1RPfdd19Rckq3adMmLx45cqQXr1692otT1wk8/vjjnHLKKSxdutRb/umnn3pxKa8b\naM7C68kOPfRQL37wwQe9uGPHjvV6/vnz53vxLbfc4sUPP/ywF//5z3/2psO6uvHGG+u1/aaqV69e\nXtylSxcvbmp9SC1a+Psde+21lze95557esurqqpKklcm2kMSEZEoqEESEZEoqEESEZEoNMk+pI8+\n+siL//3vf3txfceiy+WNN97w4pUrV3rxscce68Xh9SCTJ0/Oe1tN7fh2JfvVr37lxeEYg40V9klt\nv/32Xhxebxb2jRx44IEFzaepCO8z9dprr5Upk9II+y7PO+88bzrs65w3b15J8spEe0giIhIFNUgi\nIhIFNUgiIhKFJtmHtGLFCi++4oorvHjAgAFe/M477wAwfvx4RowYscVYcqHZs2d7cZ8+fbx4zZo1\nXnzAAQd48SWXXJL1+SVOhx12mDf9ne98x1ue6/qNsM9n+vTpXpx+3yuAJUuWeHGqTlPC68++/e1v\nb55u0aJFWa8niVl4XU5Td++992ZdHl7vVk7N65MREZFoqUESEZEo5HXIzsxqgKnAbc658Wb2dWAy\nUA0sBYY559YVL02pBKoTyYfqROqSs0Eys9bAOOD5tNnXAXc65x41szHAD4C7i5Ni402ZMsWLw7Ht\nUvedGT9+PHfddRcHHXSQt/zcc8/14vBYf9hnFHrvvfe8ePjw4bmTrjBNoU5C4ZiIM2bM8KbbtGnj\nLQ/vZ/T00097cXidUs+ePb04HHsuPPa/fPlyLw7vg5UaI7FFixZs2rRpiz6u8Lqm8H5JpVCOOgmv\nx9p1110L9dQVIdd1l+l1XW75HLJbB/QH0ntYewHTktPTgd6FTUsqkOpE8qE6kTrl3ENyzm0ANphZ\n+uzWabvUy4CswxjPmTOHmpoaYMtfkbEJR+LO5Le//W3WuJhiff+aW520a9cu5zrh2ZzhHWBzGTNm\nTL3WT9eiRYstRnZ46623Gvx8hRJjnZxxxhlZ48aIvY6rqqqiGqmiEKd95zy3tHv37kDiw4nhVNTw\nUEvqi2LTpk20aNFiiyFhwkN2YcH+7ne/K0KWWyrX+1egf6qKq5O6bmPSrl07Pv300y3qKFTfQ3bh\noaVch+xCGzduBP5zyO6LL77Iur1CH7KLtU7C9zX8An788ce9eNiwYTmfMx+x1PGrr77qxalb3VdV\nVVFbW8uRRx7pLX/99deLmk+2Omlog7TazFo659YCu+Pvfkdv1apVdS6rra3ls88+y/r49LGgAH7/\n+997cT57Wc1ERdVJ165dvTi8fi39WHzbtm3517/+5S0P71N1//33e3F4n6s//OEPWePGatmypRf/\n+Mc/9uKhQ4cWdHuNUNQ66d+/vxeH70tTE/aRpd//KJN//OMfxUynXhp62vdzwODk9GDgj4VJR5oY\n1YnkQ3UiQH5n2R0G3Ap0Br4ys1OAocAkMzsfWATcX/czSHOgOpF8qE4km3xOaniLxFkwoT4Z5kkz\npTqRfKhOJJsmOZZdY40aNcqL08cwgy07h3v39s9SffbZZ4uSlxTWtttu68Xh9WVh30Pq5Je2bdvy\n+eefb3FfnTfffNOLY+ur6NSpU7lTKIvgjL4thNcJVrqwjsM+pQ8//BBIvC8ffvhhvc/+LCYNHSQi\nIlFQgyQiIlFQgyQiIlFQH1IG4dh04XVH4QWFv/71r7145syZXhz2Ldx5551eHPvV3E3VIYcc4sVh\nn1Fo0KBBALz44osMGjRoi/sbSWWaNWtWuVPIKrwA+/jjj/fi8EL9vn37Zn2+0aNHA/Dggw8yevRo\nVq5cWYAsC0N7SCIiEgU1SCIiEgU1SCIiEgX1IeVh4cKFXnz22Wd78X333efF4eCMYdy6dWsvfuCB\nB7w4HBNNiuMXv/iFF4cDYYZ9ROlxjP1HLVq08KY1pmJ+2rdv36jHh/dPS6+jgw8+eIvrFPfYYw8v\n3mabbbw4HGMw/XMFWLt2rRe/8cYbXrxunX9vw6228r/m00d9j2EE+HTaQxIRkSioQRIRkSioQRIR\nkSioD6kBnnjiCS+eP3++F4d9E8cdd5wXh3cC3XPPPb34hhtu8OKY7ldSycI7uIY34AuvB5s2bRqV\nJNVnlOo/Cl/P7Nmzy5FW2YV9LuH7MmHCBC/+2c9+Vq/nD28AmN6H9Pbbb7NhwwZveXjjxPfff9+L\nJ06c6MXhdYxh/+U///lPL168eLEXh2Mqzps3L+N0DLSHJCIiUVCDJCIiUVCDJCIiUVAfUgHMnTvX\ni0899VQvPvHEE704vG7p/PPP9+IuXbp4cZ8+undZIYTH0sPrP5YtW+bFv//974ueU32E928K79sV\neuGFF7z4f/7nfwqdUkW48MILvXjRokVefOSRRzbq+T/66CMvnjJlCpDoCzr33HP54IMPvOWvv/56\no7YXGj58uBfvsssuXvzXv/61oNsrJu0hiYhIFNQgiYhIFNQgiYhIFNSHVATh/UUmT57sxffee68X\nh2NNHXPMMV7cq1cvb/rFF19sfJKyhXAMsHKPKRj2GY0cOdKLr7jiCi9OXX/SqVMnFi9ezK233uot\nX716dRGyrDw333xzSbYzceLELfqLiyG8zjH02GOPFT2HQtEekoiIREENkoiIREENkoiIREF9SAUQ\njmV1yimnePE3v/lNLw77jELh2FYvv/xyxmkprHKPXReOrRf2EZ122mlePHXqVC8ePHgwkBirLRwf\nUZqvcOzNmGkPSUREoqAGSUREoqAGSUREoqA+pDyYmRdfdNFFXvzd737Xizt06FCv59+4caMXh9e/\npO5zE05L/aTfpyZTfNJJJ3nxJZdcUtR8LrvsMi++6qqrvLht27Ze/NBDD3nxmWeeWZzERMpEe0gi\nIhIFNUgiIhKFvA7ZmdktwNHJ9W8EZgGTgWpgKTDMObeu7meQ5kB1IrmoRiSbnA2SmR0L1DjnepjZ\nTsA7wPPAnc65R81sDPAD4O7iplo86X0+HTp04Hvf+563POwz6ty5c6O29+abb3rxDTfc4MXlvh6m\nISqhTmpra7PGYd/fHXfc4cUTJ07cPH3wwQfz73//21t+xBFHePGwYcO8+KCDDvLiPfbYw4vD++o8\n88wzXnzXXXdRySqhRpqCsG+0a9euXlzo+zEVUj6H7F4GhiSnVwKtgV5A6ltzOtC74JlJpVGdSC6q\nEcmqKvyVmI2ZDSexu93POfe15Lx9gMnOuTpvuzh37tzampqaxuYq5VOVe5X/UJ00W3nXSUNrBFQn\nTUCddZL3ad9mNgg4F+gLzM/nyVO6d+8OJA6RhLuTMUgdqlm6dCkdO3aM9pBdud6/ev5oibZOhgwZ\n4sW/+93vvDg8/f5Xv/qVF6cO2b3zzjsccsghRT9kFx5auf3227MuT4m9ThpTIxD/90lKqfL7/e9/\n78WnnnqqF5911lle/MADDwBx1km+JzX0A64EjnfOfWZmq82spXNuLbA7sKQgmRbJrrvu6sX777+/\nF48fP37z9PPPP0+3bt0atb033njDi8eOHevF4RhkTeXaokqvk+rqai++8MILvTg1VhzAH/7wB1at\nWuUt79KlS7229+qrr3rxzJkzvfjqq6+u1/NVgkqvkUoQfuG3aFE5J1PnzNTM2gJjgQHOuRXJ2c8B\nqf/OwcAfi5OeVArVieSiGpFc8tlDOg3YGXgkbcSCs4B7zex8YBFwf3HSkwqiOpFcVCOSVc4GyTl3\nD3BPhkV9Cp+OVCrVieSiGpFcmsRYdu3bt/fisDM6vM/M3nvvXedz5dN/FB77v/XWW704vH5k7dq1\nOZ9Tiu+1117z4lmzZnlxeN+qUHi9Wtg3GQpPenj44Ye9uNhj5YkA9OjRw4snTZpUnkTyUDm9XSIi\n0qSpQRIRkSioQRIRkShURB/S4Ycf7sVXXHGFF3/rW9/y4t13371R2/viiy+8OBzTbMyYMV68Zs2a\nRm1PSmPx4sVeHN7H6vzzz/fikSNH1uv5wwtX777bH5JtwYIF9Xo+kYaI+WLhXLSHJCIiUVCDJCIi\nUVCDJCIiUaiIPqSTTz45a5zL+++/78VPPvmkF2/YsAFI9BmMGTNmi+uKVq5cWa/tSWVYunSpF48a\nNSprnFJbW7vFuHci5fL00097cTiIcCXRHpKIiERBDZKIiERBDZKIiEShXneMbfBGqqpqQTfUaqwy\n3lCrJBtVnRSG6iQOyq/O7da5Ue0hiYhIFNQgiYhIFNQgiYhIFNQgiYhIFNQgiYhIFNQgiYhIFNQg\niYhIFNQgiYhIFNQgiYhIFNQgiYhIFNQgiYhIFEoylp2IiEgu2kMSEZEoqEESEZEoqEESEZEoqEES\nEZEoqEESEZEoqEESEZEoqEESEZEobFWqDZnZbcARQC1wiXNuVqm2nY2Z1QBTgducc+PN7OvAZKAa\nWAoMc86tK1NutwBHk/icbgRmxZJbscRYJzHXSDI/1YnqJJ/8oq+TkuwhmVlPoItzrgdwLnBHKbab\njZlVmdlPgdeBnYBrzOwuEh/UncBCYFfgB0XOo6+ZLTOzkcH8Y4Ga5Ht2PPBL4DrgTufc0cCCYudW\napHWSWvgCaAdcJWZLQSeBe5Lfg7dgEeLnMO+Zva8mc03s3fN7JC0ZaoT1Ukqh93NbLqZfWBm88zs\ngrRlFVEnpTpkdxwwBcA59wHQzszalGjbdbkJOBk4iMSHcyOwDXAKMC25zjygd7ESMLPvA9cAb2dY\n/DIwJDm9EmgN9ErLbXoxcyuTGOvkGuBT4D5gNHAgsAdwnplVAR8D+xQ5h4eAJ5xzXYD/Bh5JbhtU\nJ6qT//gV8JZzbj/g28AYM7Pksoqok1IdsusAvJUWL0/OW1Wi7XvMrD0wAjjEObcw+ZmtBy4i0SB9\nlVx1DdDdzHoA40l8iJuAEc6558xsK2ACid3gauD/gLOBLzLNd86Fr3cecCxwT5ijc25jcvuQ+BX4\nFNAvbZd6GdCx4e9ClGKsk4uBQ4DTAZxza8xsLXAzUAV8CeyQXL/gdZL8ov0WiS8YnHPPmNlXJH5I\nzVadAKqTlF8Bf0puf4mZ/Q3YLxFWRp2U66SGqtyrFNURwGLn3Lz0mc65L4H1zrlNwfr3AGOdc91I\n7FlNSM7vB+xFYne8C/Ae0CPLfI9z7m3n3PpsiZrZIBIFdFGwqNzvYSmU+zVmrBMA59z0EtVJarDJ\n9P/V1cC+6SupTsoqhjpJbetTADPrBHQlOPoSe52UqkFaQuIXTMpuJDrRyqU98M86lq02s5bJ6TYk\ncj8YeCQ57xVg7+T0cmB/Eof+WjnnrnLOPZNlfr2YWT/gSuAE59xnQW67J3NrSiqlTtI/h1b855d5\nwevEOfc58Abw/5L9nr2BGmC71DqqE9VJOjPbEXgMGOOc+yhtfvR1UqoG6VkSh8Iws0OBJcl/tHL5\nF4kPIJPngMHJ6QOAPwJDgb+YmQNmkPw14Zz7C4ld9YuBT8zst2a2Y13z65OgmbUFxgIDnHMrMuQ2\nOJlbU1IpdZL+OewJzE9OF6tOhgL/BTjgVBKHZVaC6kR14jOzDsBM4Cnn3Ji0+RVRJyW7/YSZ3QQc\nQ+KY6Y+cc++WZMOZc9mRxK+B85J/nUn0Gy0BZpP4BVpD4lfXIOBD4HDn3Gwz6wJ86JyrCp6zPTAR\neM85d2Wu+cFjJwELnHPXp80bDoxKbjvlLOBeEr+OFwHnOOe+ogmJsE6WkjhEsjOJGvkHic9hZnJZ\nB+B+4DfAXylinaStuxD4tnNukepEdZK2rA2JPa5JzrnbgmUVUSfN9n5IZjaKxK+sk5xzC8ysFYnT\nvXd1zvVPNRLAk8ALJA4LbCBxNt7lJDoohwB7OOdGJ5/zpuQ6CzPNd855p3an5TKJoEGSOMRQJ2Y2\nDbjfOfeYmQ0DfuCcO7a4r1zqI5I6uQPY6Jy7rMgvt2hKdmFsbJxzo8xsBTDNzKpJ/NKaClwQrPou\niTNSPiRxnPjHwFHAS0AfYKKZzSdROPNJnBVDlvmbmdlE4EgSZ7esN7MzgPHOufGFe6XSGDHUCYkz\nte4xs7EkfsmeWajXJ4URSZ2cDywxsxPS5v3SOTchw7pRarZ7SCIiEheNZSciIlFQgyQiIlFocB+S\nRTi4ocRHdSL5UJ0INHAPySIc3FDiozqRfKhOJKWhh+zqNbhhVVVVbVVVVe3cuXNrU9Mx/im/zH8N\nrBHVSTPLT3USx+cQe37ZCqGhDVIHEsNZpKQGN8yqpqamgZsrDeVXcKqTMog9vwxUJ2UQY36Fug4p\n68B8c+bM2fziYz/NXPkVleqkRGLPLwfVSYnEll9DG6R6DW7YvXt3IPHiq6qiGFQ2I+VX93YbSHVS\nBqqTOCi/urdbl4YesottcEOJk+pE8qE6EaARIzXUZ3DDVEeWfjE0Thl/0TR4o6qT0lOdxEH51bnd\nOjdakqGDVECFEWMBFZLqpDBUJ3FQfnVut86NaqQGERGJghokERGJghokERGJghokERGJghokERGJ\nghokERGJghokERGJghokERGJghokERGJghokERGJghokERGJghokERGJQqFu0CeNMHLkSC++9tpr\nvbhFi//8bqitraVXr17e8pdeeqlouYlIee2www5evP3223vxd77zHS/eZZddvPgXv/iFF69bt66A\n2RWW9pBERCQKapBERCQKapBERCQK6kMqg7PPPtuLf/KTn3jxpk2bMj6uRYsWbNq0Kes96UWksnTu\n3NmLw++DHj16eHFNTU29nr9jx45ePGLEiHo9vpS0hyQiIlFQgyQiIlFQgyQiIlFQH1IZ7Lnnnl68\n3XbblSkTKaTDDz/ci8844wwv7tmzpxcfcMABWZ/v8ssv9+IlS5Z48VFHHeXFDz74oJfLG2+8kT1h\nKYlu3bp58aWXXurFQ4cO9eKWLVt6cVVVlRd//PHHXvz555978X777efFp556qhffddddXm7z5s2r\nK/WS0x6SiIhEQQ2SiIhEQQ2SiIhEQX1IJdC7d28vvvjii7OuHx7THTBgAAB///vf2XvvvfnnP/9Z\n2ASlQU477TQvvv32271455139uKwL+DFF1/04nAMsrFjx2bdfvh86Y+/7LLLOP3007M+Xgqjbdu2\nXnzzzTdvnp4wYcIWdRKOTZfL/Pnzvbhfv35evPXWW3tx+P0R1mF6HC4rN+0hiYhIFNQgiYhIFNQg\niYhIFNSHVATh9SH33XefF4fHnENh38GiRYsyTktxbbWV/+/xjW98Y/P0EUccwa9//WtveatWrbz4\n5Zdf9uLRo0d78Z/+9Ccv3nbbbb34kUce8eK+fftmzffNN98EEn1bqWkpvpNPPtmLf/jDH2acztfC\nhQu9uE+fPl4cXoe077771nsbsdIekoiIREENkoiIRCGvQ3ZmVgNMBW5zzo03s68Dk4FqYCkwzDkX\n731xpSRUJ5IP1YnUJWeDZGatgXHA82mzrwPudM49amZjgB8Adxcnxcpz1llnefFuu+2Wdf3wepQH\nHnig0CkVXVOsk3AsunvvvXfzdNj/AzBjxgwvDq8/WbVqVdbthevn6jNavHixF99///1Aog8yNR2b\nplgnQ4YMqdf6f//737141qxZXhzeDynsMwqFY9dVsnwO2a0D+gPpIzv2AqYlp6cDvZHmTnUi+VCd\nSJ1y7iE55zYAG8wsfXbrtF3qZUDHLR4ozYrqRPKhOpFsCnHad1WuFebMmbP5trux3367HPl9+9vf\nzjuH2N+/LJpUnVRXV28x7/jjj/fizz77rKg5dOrUyYuXLVuWcbrCNPk62WeffbLGxR7y6ZVXXsk4\nHYOGNkirzaylc24tsDv+7vcWunfvDiSKJxx/KyYNzS8cDyoca27Tpk1evHLlSi8O71cyc+bMgubX\nWI34p6+oOgmvE/rZz37mxan3obq6mo0bN3r3lQEYOXKkF+fqMwp98MEHXtylS5es6w8ePNiLp06d\nujlP1UnphH3Ew4cPB+Caa67h2muv5dlnn/WWL1iwwIsb++MhvNZpwoQJWdfv1asXkGiMjj766Iz9\nocWUrU4aetr3c0Dqv2Ew8McGPo80baoTyYfqRID8zrI7DLgV6Ax8ZWanAEOBSWZ2PrAIiPOUHikZ\n1YnkQ3Ui2eRzUsNbJM6CCfXJME+aKdWJ5EN1ItloLLsG6Ny5sxc/9thj9Xr8uHHjvLiuPiMprquv\nvtqLwz6j9evXe/EzzzwDwKBBg3jyySe3uF5k7dq1Wbe33XbbeXF4nVF4kkLYP3L99dd7carPSMpr\nyRK/y2vUqFFAog8pNV1MPXr0KPo2SkVDB4mISBTUIImISBTUIImISBTUh9QA4QWQBx54YNb1n3/+\neS++/fbbC56T5Lbjjjt68YUXXujF4fURqT6jlJNOOmnzeqnpbML71Dz00ENefNhhh2V9/P/+7/96\n8S233JJzm1J5RowY4cWtW7eu1+NT12XV5dVXX/Xi1157LeN0DLSHJCIiUVCDJCIiUVCDJCIiUVAf\nUh7C/oKbbrop6/rh2FDh/ZGKPeimZLbNNtt4cTgGYSg8tv+1r33Nmz7nnHO85QMHDvTi1ACgKdtv\nv70Xh31WYfzggw968Zo1a7JViJuxAAAaZ0lEQVTmK3Fo1aqVN73//vt7y6+55hov7t+/f9bna9HC\n328Ix8YMhddFhXW6cePGjNMx0B6SiIhEQQ2SiIhEQQ2SiIhEQX1IGTR2rLq//vWvXhzeH0nKIxyb\nbvny5V68yy67ePHf/vY3L07v4wmP02cSrhPeH6ljR//GqP/617+8ePr06Tm3IaW39dZbe/Ehhxzi\nxenfF865LT7ncMzDsE7Ca4PC6x7T+6gy2Wor/2v9u9/9rhenXwe5zTbbbPF/UU7aQxIRkSioQRIR\nkSioQRIRkSioDymD8D43uc77D+W6TknKY+XKlV4cXl/25JNPenH79u29eOHChQB07dqVhQsXbnE/\nokmTJnnxihUrvPjhhx/24rBvIVwucQivXwv7dB5//PE6H9uxY0euvfZab94LL7zgxX/+85+9OKy7\ncP3w+rZQ2Bd64403evFHH320efrkk09mypQp3vJ169Zlff5i0h6SiIhEQQ2SiIhEQQ2SiIhEQX1I\nwMEHH+xN9+3bt16PD/sSnHMFyUuK64033vDi8Nh7XWprazGznOsdc8wxXtyzZ08vDvsmw+vXpDzC\n64zCPqArrrgi6+OffvppAAYMGMDTTz/NuHHjvOVhX2ZYd0899ZQXh/c7Cq8bCu+TFfYxDRo0yIvT\n78v10EMP8dxzz3nLb775Zi/+9NNPyWb27NlZl9eH9pBERCQKapBERCQKapBERCQKVeE9WIqykaqq\nWkgce6+qqir69upr2bJlQOJY7vLly2nXrl3W9V9//XUvPuGEE7x49erVhU0wqVzvX21tbUk2Gnud\npOSbX79+/bw47BsI//fC65LCsfYKnV+hVWqdVFdXe/ENN9zgxZdffrkXh/el+ulPf+rFqevJVqxY\nQfv27bfog/nGN77hxePHj8+6fMGCBV58wQUXePHMmTO9uE2bNl585JFHevHQoUMBOOOMM3jwwQe3\nuI9X69atyebjjz/24r322ivr+qFsdaI9JBERiYIaJBERiYIaJBERiYL6kPjPfeVbtGjBpk2bco5d\nd+aZZ3rx7373u6Lllk59A3FoaH6pOkt/nnTqQ8pPoesk7JMJrxv64osvvHj48OFe/Oyzz3rx4Ycf\nDiT6DPv3788555zjLQ/7nFu2bOnF1113nRffd999Xhz24TRU6v373ve+583//ve/n/Vxl112mReH\nfVx5bFd9SCIiEjc1SCIiEoW8hg4ys1uAo5Pr3wjMAiYD1cBSYJhzrnxjlksUVCeSi2pEssnZh2Rm\nxwJXOOf6m9lOwDvA88BTzrlHzWwM8LFz7u46NxJZ30B4TPbss8/24lx9SHvvvbcXL1q0qCB55RJz\n30BTrJO66DqkOrebdaOFqBEofJ0sXbrUi8Ox5cL7A82bN8+Lw+t29t13XyBxfVPYb5jJqFGjvDi8\nf1E+z9EQMdZJPofsXgaGJKdXAq2BXsC05LzpQO9G5CdNg+pEclGNSFY5D9k55zYCqUuTzwWeAvql\n7VYvAzpmeqw0H6oTyUU1IrnkffsJMxtEooj6AvPTFuXc55szZ87mIdFLcZp5Y7VokX3H8e9//3tp\nEskg9vevudRJMfJLDWFVCDG/f42pESh9nbRq1cqLDz300LwfGw5LlMno0aOzxsUUW53ke1JDP+BK\n4Hjn3GdmttrMWjrn1gK7A0uyPT51P49yHbNMv98RwPTp0714t912A/5zHVJ4v5E777zTi0eOHOnF\nX375ZaFSzaqMx3zzWq/S6yRf+eYXXt8SjlnWBPuQcq7T2BqBwtfJO++8k/H5GyrVV3jiiScyffp0\nXn75ZW/5lClTvDj8gbthw4ZGbT9fMdZJzj4kM2sLjAUGOOdWJGc/BwxOTg8G/tjIHKXCqU4kF9WI\n5JLPHtJpwM7AI2l3yTwLuNfMzgcWAfcXJz2pIKoTyUU1Ilnlc1LDPcA9GRb1KXw6UqlUJ5KLakRy\naRZj2fXq1cuLZ8yY4cWpkxhSfUh/+9vfvOWp6wrKLcbrBgqp3HWSr3zzS3W8p8yZM8eLw+vdOnTo\n4MUV2IdUkXWyww47ePFJJ53kxeFJDOHJJxMnTvTi1P2P1q1bx7bbbrtFn3QsYqwTDR0kIiJRUIMk\nIiJRUIMkIiJRyPvCWBGpn7lz53rx/PnzvTgcE3Gfffbx4ob2IUn9fP755148efLkrHF9xNp/FCvt\nIYmISBTUIImISBTUIImISBSaRR9SeP+SV1991YuPOuqoUqYjzdSYMWO8+N577/XiG264wYsvvvhi\nL37//feLk5hIJLSHJCIiUVCDJCIiUVCDJCIiUWgWY9nlS/nVud2KHKOsWBqaX5s2bbz4kUce8eLe\nvf27dz/++ONefM4553jxmjVryER1EgflV+d2NZadiIjETQ2SiIhEQQ2SiIhEQX1IaZRfndtV30Ca\nQuUX9imF1yFdcMEFXnzggQd6cV3XJalO4qD86tyu+pBERCRuapBERCQKapBERCQK6kNKo/zq3K76\nBtIovzq3qzpJo/zq3K76kEREJG5qkEREJApqkEREJAol6UMSERHJRXtIIiISBTVIIiISBTVIIiIS\nBTVIIiISBTVIIiISBTVIIiISBTVIIiISha1KtSEzuw04AqgFLnHOzSrVtrMxsxpgKnCbc268mX0d\nmAxUA0uBYc65dWXK7RbgaBKf043ArFhyK5YY6yTmGknmpzpRneSTX/R1UpI9JDPrCXRxzvUAzgXu\nKMV2szGzKjP7KfA6sBNwjZndReKDuhNYCOwK/KDIefQ1s2VmNjKYfyxQk3zPjgd+CVwH3OmcOxpY\nUOzcSi3SOmkNPAG0A64ys4XAs8B9yc+hG/BokXP4upk9ZWYfmNk8M7swbZnqRHWSnkdFf5+U6pDd\nccAUAOfcB0A7M2uT/SFFdxNwMnAQiQ/nRmAb4BRgWnKdeUDvYiVgZt8HrgHezrD4ZWBIcnol0Bro\nlZbb9GLmViYx1sk1wKfAfcBo4EBgD+A8M6sCPgb2KXIO9wLPOuf2A/oAY8zsgOQy1YnqBGga3yel\nOmTXAXgrLV6enLeqRNv3mFl7YARwiHNuoZkBrAcuItEgfZVcdQ3Q3cx6AONJfIibgBHOuefMbCtg\nAond4Grg/4CzgS8yzXfOha93HnAscE+Yo3NuY3L7kPgV+BTQL22XehnQseHvQpRirJOLgUOA0wGc\nc2vMbC1wM1AFfAnskFy/WHXyKxK/tnHOfWxmC4CuwHuqE0B1klLx3yflOqmh3DcJOQJY7Jyblz7T\nOfclsN45tylY/x5grHOuG4k9qwnJ+f2AvUjsjncB3gN6ZJnvcc697Zxbny1RMxtEooAuChaV+z0s\nhXK/xox1AuCcm17COnncObcaNn+ZdQT+lL6O6qSsYqmTiv8+KVWDtITEL5iU3Uh0opVLe+CfdSxb\nbWYtk9NtSOR+MPBIct4rwN7J6eXA/iQO/bVyzl3lnHsmy/x6MbN+wJXACc65z4Lcdk/m1pRUSp2k\nfw6t+M8v86LViZl1MrO/kfhle7FzbnnaMtWJ6iSnSqiTUjVIz5I4FIaZHQoscc59XqJtZ/IvEh9A\nJs8Bg5PTBwB/BIYCfzEzB8wg+WvCOfcXErvqFwOfmNlvzWzHuubXJ0EzawuMBQY451ZkyG1wMrem\npFLqJP1z2BOYn5wuWp045z5yzu0FHEqiD6k/qE5UJ/mplDop2e0nzOwm4BgSx0x/5Jx7tyQbzpzL\njiR+DZyX/OtMot9oCTAbqEn+LQUGAR8ChzvnZptZF+BD51xV8JztgYkkjutfmWt+8NhJwALn3PVp\n84YDo5LbTjmLRAf3dsAi4Bzn3Fc0IRHWyVISh0h2JlEj/yDxOcxMLusA3A/8BvgrBa4TM9sWOAOY\nlOwHwMxuBaqdc5eqTlQnGfKZRIV+nzTb+yGZ2SgSv7JOcs4tMLNWJE733tU51z/1oQJPAi+QOCyw\ngcTZeJeT6KAcAuzhnBudfM6bkusszDTfOeedipmWyySCApI4xFAnyVOIr3fO3Wdm2wMvkThdd2JR\nX7zkLYY6SctlEhX6fVKyC2Nj45wbZWYrgGlmVk3il9ZU4IJg1XdJHLf/kMRx4h8DR5H4UugDTDSz\n+SQKZz6Js2LIMn8zM5sIHEmik3q9mZ0BjHfOjS/cK5XGiKFOgO8C48zsJyT+Z6cBkwrzCqUQYqiT\npvB90mz3kEREJC4ay05ERKKgBklERKKgBklERKLQ4JMaLMLRdiU+qhPJh+pEAKitra33X9euXXt2\n7dr1yeT0fl27dn0t2/okiqx2zpw5tanpGP+UX+a/htSI6qT55ac6ieNziD2/bJ9tQw/ZNWi03Zqa\nmgZurjSUX8GpTsog9vwyUJ2UQYz5NfSQXb1G250zZ87mFx/7aebKr6BUJ2USe34B1UmZxJZfoS6M\nzTpSbPfu3YHEi6+qimJQ2YyUX93bLRDVSQmoTuKg/Orebl0aesguttF2JU6qE8mH6kSAhjdIsY22\nK3FSnUg+VCcCNGLooPqMtltVVZU4NUa7sI1Sxl3sBm9UdVJ6qpM4KL86t1vnRksylp0KqDBiLKBC\nUp0UhuokDsqvzu3WuVGN1CAiIlFQgyQiIlFQgyQiIlFQgyQiIlFQgyQiIlFQgyQiIlFQgyQiIlFQ\ngyQiIlEo1OCqTdrtt9/uxSNGjPDiuXPnevGAAQO8eNGiRcVJTESkCdEekoiIREENkoiIREENkoiI\nREF9SBl07tzZi8844wwv3rRpkxfvt99+XtytWzcvVh9S09S1a1cv3nrrrb34mGOO8eK77rrLi8M6\naqypU6dunn788cc5/fTTveXr168v6PakYcI6OfLII714zJgxXvxf//VfRc8pFtpDEhGRKKhBEhGR\nKKhBEhGRKKgPKYPly5d78csvv+zFAwcOLGU6UiYHHHCAF5999tmbp8eOHcuQIUO85S1a+L/vdttt\nNy8O+4wKfXPM9LocOHAgEyZM8JZfeumlXrxq1aqCbl/y07ZtWy+eOXOmF3/yySde3KFDh6zLmxLt\nIYmISBTUIImISBTUIImISBTUh5TBmjVrvFjXETVPN954oxf3799/8/Rll11W6nTq7cwzz/Ti3/zm\nN1785z//uZTpSJ7CPiP1IYmIiJSYGiQREYmCGiQREYmC+pAy2HHHHb34oIMOKlMmUk4zZszw4vQ+\npEyWLVvmxWGfTXidUq6x7MIxznr27Jl1fWkaqqqqyp1C2WgPSUREoqAGSUREoqAGSUREoqA+pAxa\ntWrlxZ06darX47/5zW968bx587xY1zVVhrvvvtuLp0yZAsBHH33EXnvttcX6X331lRc39nqRNm3a\nePHcuXO9OBwrL5TKN+XNN99sVD5SGuEYh9ttt12ZMik97SGJiEgU1CCJiEgU8jpkZ2Y1wFTgNufc\neDP7OjAZqAaWAsOcc+uKl6ZUAtWJ5EN1InXJ2SCZWWtgHPB82uzrgDudc4+a2RjgB8DdmR5fiZYs\nWeLFkyZN8uJRo0ZlfXy4fOXKlV48fvz4hqYWraZYJxs2bPDijz/+OON0sfTr18+L27VrV6/HL168\n2IvXrSv/d3xTrJNi+8Y3vuHFr7/+epkyKb58DtmtA/oD6d/SvYBpyenpQO/CpiUVSHUi+VCdSJ1y\n7iE55zYAG8wsfXbrtF3qZUDHbM8xZ84campqgMLfJbPQipHfuHHjssb1Eev7pzqJS3V1NZdccok3\nL4zLQXVSf4X8/gjF9v4V4rTvnONcdO/eHUi8+JiHxagrv6uuusqLw0NyuT7U8NbRDT1kV673r0BF\n2+TrpNDCW6RPnDjRi1u2bJnxcdXV1WzcuHGLOgvrsNBUJ7668tt55529OBxyKjRixAgvLtQh/xi/\nTxraIK02s5bOubXA7vi7303O6NGjvThXH5Js1qzqpLFOP/10Lz7vvPO8uK4GqC5XX311o3MqkWZV\nJ2Hf5GeffebFbdu29eJ99tmn6DnFoqGnfT8HDE5ODwb+WJh0pIlRnUg+VCcC5HeW3WHArUBn4Csz\nOwUYCkwys/OBRcD9xUxS4qc6kXyoTiSbfE5qeIvEWTChPgXPRiqW6kTyoTqRbDSWXQPU9742IgBD\nhw714p/+9KdevO+++3rx1ltvXa/nnz17NgCHHXYYs2fP3mJsPYlDeF3iK6+84sUDBgwoZTpR0dBB\nIiISBTVIIiISBTVIIiISBfUhNUDYZxTb1c5SGJ07d/biYcOGbZ6+6qqr6N27fiPcHHXUUV5c37pZ\ntWqVF4d9UE899RSQGGfvpJNOYu3atfV6fpFy0x6SiIhEQQ2SiIhEQQ2SiIhEQX1IIkmpEaRTpk2b\n5sWdOnXaPH3NNdeUJKd04fUq99xzT53rhvdCksq10047lTuFktEekoiIREENkoiIREENkoiIREF9\nSCJ1CG9elh435MZmjR0DMRzj7IQTTvDip59+ut45SfwGDhxY7hRKRntIIiISBTVIIiISBTVIIiIS\nBfUhNUB9+wKOOeYYLx4/fnzBc5LGmzt3rhf36tXLi8844wwArr/+eq6++mqeeeYZb/mXX37ZqO2f\ne+65XnzxxRc36vmkMsycOdOLdT8kERGRMlODJCIiUVCDJCIiUagqxb18qqqqaiFx/5eGXL9RKvnm\nt3Hjxi0eVx8HHnigF7///vt5Pa5c719tbW1JNtrU6qS+2rZt68X//ve/s65/4oknenHqOiTVSRzy\nzW/w4MFe/Oijj3pxeF+r/fff34sXLVpU1PwKLVudaA9JRESioAZJRESioAZJRESioOuQGmDChAle\nfP7559fr8cOHD/fiSy+9tNE5SeXr169fuVOQMtiwYUPW5WE/z7bbblvMdMpKe0giIhIFNUgiIhIF\nNUgiIhIF9SE1wLx588qdgjTA1ltv7cV9+/b14hdeeMGLw+s/Cu2cc87x4ttvv72o25M4TZ061YvD\n75du3bp5cdjnfOGFFxYnsTLQHpKIiERBDZKIiEQhr0N2ZnYLcHRy/RuBWcBkoBpYCgxzzq0rVpJS\nGVQnkotqRLLJOZadmR0LXOGc629mOwHvAM8DTznnHjWzMcDHzrm769xIExt7KvThhx968T777JN1\n/fB+Svvuu68XL1y4sKD5NVY+Y5TFWCdHHXWUF1955ZVe3KdPHy/ea6+9vPjjjz/O+Lz55te+fXsv\n7t+/vxePGzfOi3fYYYeszxf2aQ0cONCLU/fVibVOClEj0PS/T375y196cdjXuOuuu3pxQ+/DFWOd\n5HPI7mVgSHJ6JdAa6AVMS86bDvRuRH7SNKhOJBfViGSV85Cdc24jsCYZngs8BfRL261eBnTM9hxz\n5syhpqYGqP/I2KVWjvwWLFiQ97qxvn9NoU4++uijvNctR37bb7+9F4dnBaaLsU4KUSNQ/jqpj2Lk\nV8izP2N7//I+7dvMBpEoor7A/LRFOff5unfvDjTdXexmcMgu73VjqhMdsiutfOukMTUCTf/7pBkc\nsqtzWb4nNfQDrgSOd859Zmarzaylc24tsDuwpCCZVqj33nvPi/fee++s62/atKmY6ZRNbHUyfvx4\nL079qq7Lf//3f3vx559/Xue6Y8aMybn9sME79NBDvTjXF/iLL77oxXff7XetpBqgShJbjVSCsE7W\nr19fpkyKL2cfkpm1BcYCA5xzK5KznwNSd5UaDPyxOOlJpVCdSC6qEcklnz2k04CdgUfMLDXvLOBe\nMzsfWATcX5z0pIKoTiQX1Yhklc9JDfcA92RY1CfDPGmmVCeSi2pEctFYdgVwzz3+/9iJJ55Ypkyk\nMS644IK81w37mxpi2bJlXjx9+nQvvuSSS7y4oZ3XUtnatGnjxYMGDfLiJ554opTpFJWGDhIRkSio\nQRIRkSioQRIRkSioD6kA3n//fS/+4IMPvHi//fYrZTqSdPbZZ3vxxRdf7MVnnXVWQbcXXtD8xRdf\nePErr7zixWHf49y5cwuaj1SmU0891YvXrfPHmg2/X5oS7SGJiEgU1CCJiEgU1CCJiEgUct4PqSAb\naeL3LymVGO9fUkjFrpNtt93Wi8M+puuvv96L27Vr58VTpkwBYPDgwTz22GPMmDHDWz516lQv/uST\nTxqTboOpTuLQ0PwefvhhLw77oMNBdRctWlT/5IizTrSHJCIiUVCDJCIiUVCDJCIiUVAfUhrlV+d2\n1TeQRvnVuV3VSRrlV+d21YckIiJxU4MkIiJRUIMkIiJRUIMkIiJRUIMkIiJRUIMkIiJRUIMkIiJR\nUIMkIiJRUIMkIiJRUIMkIiJRUIMkIiJRKMlYdiIiIrloD0lERKKgBklERKKgBklERKKgBklERKKg\nBklERKKgBklERKKgBklERKKwVak2ZGa3AUcAtcAlzrlZpdp2NmZWA0wFbnPOjTezrwOTgWpgKTDM\nObeuTLndAhxN4nO6EZgVS27FEmOdxFwjyfxUJ6qTfPKLvk5KsodkZj2BLs65HsC5wB2l2G4uZtYa\nGAc8nzb7OuBO59zRwALgB2XK7VigJvmeHQ/8MpbciiXGOom5RkB1guokL5VSJ6U6ZHccMAXAOfcB\n0M7M2pRo29msA/oDS9Lm9QKmJaenA71LnFPKy8CQ5PRKoDXx5FYsMdZJzDUCqhPVSX4qok5Kdciu\nA/BWWrw8OW9VibafkXNuA7DBzNJnt07bbV0GdCx5YoBzbiOwJhmeCzwF9IshtyKKrk5irhFQnSSp\nTnKolDopWR9SoKpM262vsudpZoNIFFBfYH7aorLnVgKV8BqjyFF1Er0ocoy9Tkp1yG4JiV8wKbuR\n6ESL0Woza5mc3h1/F7ykzKwfcCVwgnPus5hyK5JKqZOoPgfVieokH5VQJ6VqkJ4FTgEws0OBJc65\nz0u07fp6DhicnB4M/LEcSZhZW2AsMMA5tyKm3IqoUuokms9BdaI6yUel1EnJbj9hZjcBxwCbgB85\n594tyYazMLPDgFuBzsBXwD+AocAkYDtgEXCOc+6rMuQ2HBgFfJg2+yzg3nLnVkyx1UnMNZLMT3Wi\nOsknv4qoE90PSUREoqCRGkREJApqkEREJApqkEREJApqkEREJApqkEREJApqkEREJApqkEREJAr/\nPwS/GururEs1AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f4c621507b8>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "mUxLKgForJkn",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "5f547a11-83f6-462d-8a51-642a3a2b1b45",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528032011919,
          "user_tz": -120,
          "elapsed": 498,
          "user": {
            "displayName": "Jannes Klaas",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "115843019312458301179"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "X_train = X_train.reshape(60000, 784)\n",
        "X_test = X_test.reshape(10000, 784)\n",
        "X_train = X_train.astype('float32')\n",
        "X_test = X_test.astype('float32')\n",
        "X_train /= 255\n",
        "X_test /= 255\n",
        "print(\"Training matrix shape\", X_train.shape)\n",
        "print(\"Testing matrix shape\", X_test.shape)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training matrix shape (60000, 784)\n",
            "Testing matrix shape (10000, 784)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "bJCFrPAzrLeu",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "Y_train = np_utils.to_categorical(y_train, nb_classes)\n",
        "Y_test = np_utils.to_categorical(y_test, nb_classes)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WSFwx8AxrN4-",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "model = Sequential()\n",
        "model.add(Dense(512, input_shape=(784,)))\n",
        "model.add(Activation('relu')) \n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(512))\n",
        "model.add(Activation('relu'))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(10))\n",
        "model.add(Activation('softmax')) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "9C7RojtlsRlp",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['acc'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "AOSWAK1FsTwp",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 207
        },
        "outputId": "1e838b2c-dcb1-4952-ec04-6a9e7328575b",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528032063838,
          "user_tz": -120,
          "elapsed": 49667,
          "user": {
            "displayName": "Jannes Klaas",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "115843019312458301179"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "model.fit(X_train, Y_train, batch_size = 128, epochs=4, validation_data=(X_test, Y_test))"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 60000 samples, validate on 10000 samples\n",
            "Epoch 1/4\n",
            "60000/60000 [==============================] - 13s 218us/step - loss: 0.2444 - acc: 0.9269 - val_loss: 0.1056 - val_acc: 0.9654\n",
            "Epoch 2/4\n",
            "60000/60000 [==============================] - 12s 197us/step - loss: 0.1025 - acc: 0.9692 - val_loss: 0.0824 - val_acc: 0.9748\n",
            "Epoch 3/4\n",
            "18688/60000 [========>.....................] - ETA: 7s - loss: 0.0715 - acc: 0.9783"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "60000/60000 [==============================] - 12s 195us/step - loss: 0.0724 - acc: 0.9774 - val_loss: 0.0684 - val_acc: 0.9792\n",
            "Epoch 4/4\n",
            "60000/60000 [==============================] - 12s 199us/step - loss: 0.0546 - acc: 0.9827 - val_loss: 0.0664 - val_acc: 0.9802\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f4c61c8e9e8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "metadata": {
        "id": "UWSozJxYtg5Q",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "model = Sequential()\n",
        "model.add(Dense(512, input_shape=(784,)))\n",
        "model.add(Activation('relu')) \n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(512))\n",
        "model.add(Activation('relu'))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(10))\n",
        "model.add(Activation('softmax')) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "I5Jlu4s-tkKQ",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        "from keras.optimizers import Adam"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Ub6p12C2uYJM",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 717
        },
        "outputId": "5d10f962-7845-4c86-eacc-d24b0bd09a80",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528032338945,
          "user_tz": -120,
          "elapsed": 273461,
          "user": {
            "displayName": "Jannes Klaas",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "115843019312458301179"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "init_lr = 1e-6\n",
        "losses = []\n",
        "lrs = []\n",
        "for i in range(20):\n",
        "  model = Sequential()\n",
        "  model.add(Dense(512, input_shape=(784,)))\n",
        "  model.add(Activation('relu')) \n",
        "  model.add(Dropout(0.2))\n",
        "  model.add(Dense(512))\n",
        "  model.add(Activation('relu'))\n",
        "  model.add(Dropout(0.2))\n",
        "  model.add(Dense(10))\n",
        "  model.add(Activation('softmax')) \n",
        "  opt = Adam(lr=init_lr*2**i)\n",
        "\n",
        "  model.compile(loss='categorical_crossentropy', optimizer=opt,metrics=['acc'])\n",
        "\n",
        "\n",
        "  hist = model.fit(X_train, Y_train, batch_size = 128, epochs=1)\n",
        "\n",
        "  loss = hist.history['loss'][0]\n",
        "  losses.append(loss)\n",
        "  lrs.append(init_lr*2**i)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/1\n",
            "60000/60000 [==============================] - 13s 213us/step - loss: 2.2732 - acc: 0.1354\n",
            "Epoch 1/1\n",
            "60000/60000 [==============================] - 13s 213us/step - loss: 2.2564 - acc: 0.1626\n",
            "Epoch 1/1\n",
            "50816/60000 [========================>.....] - ETA: 2s - loss: 2.1483 - acc: 0.2956"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "60000/60000 [==============================] - 13s 216us/step - loss: 2.1169 - acc: 0.3264\n",
            "Epoch 1/1\n",
            "60000/60000 [==============================] - 13s 220us/step - loss: 1.9342 - acc: 0.4650\n",
            "Epoch 1/1\n",
            "60000/60000 [==============================] - 13s 216us/step - loss: 1.5262 - acc: 0.6157\n",
            "Epoch 1/1\n",
            " 7808/60000 [==>...........................]"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "60000/60000 [==============================] - 13s 215us/step - loss: 1.0999 - acc: 0.7229\n",
            "Epoch 1/1\n",
            "60000/60000 [==============================] - 13s 216us/step - loss: 0.7590 - acc: 0.7997\n",
            "Epoch 1/1\n",
            "60000/60000 [==============================] - 13s 215us/step - loss: 0.5506 - acc: 0.8497\n",
            "Epoch 1/1\n",
            " 1152/60000 [..............................] - ETA: 1:36 - loss: 2.1497 - acc: 0.2899"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "60000/60000 [==============================] - 13s 215us/step - loss: 0.4071 - acc: 0.8853\n",
            "Epoch 1/1\n",
            "60000/60000 [==============================] - 13s 215us/step - loss: 0.3087 - acc: 0.9093\n",
            "Epoch 1/1\n",
            "60000/60000 [==============================] - 13s 216us/step - loss: 0.2476 - acc: 0.9260\n",
            "Epoch 1/1\n",
            " 1536/60000 [..............................] - ETA: 1:18 - loss: 1.1254 - acc: 0.6452"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "60000/60000 [==============================] - 13s 220us/step - loss: 0.2221 - acc: 0.9316\n",
            "Epoch 1/1\n",
            "60000/60000 [==============================] - 13s 219us/step - loss: 0.2312 - acc: 0.9293\n",
            "Epoch 1/1\n",
            "60000/60000 [==============================] - 13s 218us/step - loss: 0.2984 - acc: 0.9121\n",
            "Epoch 1/1\n",
            "  384/60000 [..............................] - ETA: 5:02 - loss: 5.3978 - acc: 0.1172 "
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "60000/60000 [==============================] - 13s 220us/step - loss: 0.6236 - acc: 0.8478\n",
            "Epoch 1/1\n",
            "60000/60000 [==============================] - 14s 227us/step - loss: 14.5157 - acc: 0.0977\n",
            "Epoch 1/1\n",
            "54912/60000 [==========================>...] - ETA: 1s - loss: 14.5015 - acc: 0.0985"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "60000/60000 [==============================] - 14s 227us/step - loss: 14.5045 - acc: 0.0985\n",
            "Epoch 1/1\n",
            "60000/60000 [==============================] - 13s 223us/step - loss: 14.4954 - acc: 0.0990\n",
            "Epoch 1/1\n",
            "60000/60000 [==============================] - 13s 222us/step - loss: 14.4533 - acc: 0.1017\n",
            "Epoch 1/1\n",
            "11392/60000 [====>.........................] - ETA: 18s - loss: 14.2726 - acc: 0.1064"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "60000/60000 [==============================] - 14s 226us/step - loss: 14.3993 - acc: 0.1051\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "MN-wiXxLu3Q6",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 432
        },
        "outputId": "cbb94e6b-0d16-4991-87d5-cebd7c36c717",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528032339934,
          "user_tz": -120,
          "elapsed": 946,
          "user": {
            "displayName": "Jannes Klaas",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "115843019312458301179"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "#lrs[0] = init_lr\n",
        "fig, ax = plt.subplots(figsize = (10,7))\n",
        "plt.plot(lrs,losses)\n",
        "ax.set_xscale('log')"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlYAAAGfCAYAAACdqpz+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XuQXOdd5vHn9L17emY0l5ZmdLNs\nSX5jW5biGOL4Fic4G1yBXXaDgaXIQgrYJZDsQlEUt10ghC1gN5UKkLC1yS63BWpZdgMJSwjxxiTG\nwXY58U2WbY4lWbYsjS4z0mh0erpn+nb2j77M6D7qOaf7nNPfT5Wqu0+f7n7ndVvz6H1/530t13UF\nAACA9Yv1uwEAAABRQbACAADwCMEKAADAIwQrAAAAjxCsAAAAPJLoxYfMzjo9ufRwbCyn+flSLz5q\nYNHH/qJ//Ucf+48+9hf9679CYdjq9rWRGrFKJOL9bkLk0cf+on/9Rx/7jz72F/0bbJEKVgAAAP1E\nsAIAAPAIwQoAAMAjBCsAAACPEKwAAAA8QrACAADwCMEKAADAIwQrAAAAjxCsAAAAPEKwAgAA8AjB\nCgAAwCMEKwAAAI8QrAAAADxCsAIAAPBIot8NAACg386XKnJdKWZJlmUpZlmKxVbuW5YUizXvA1dD\nsAIADLQvP31U/+vvDq35fMtSK2xdGL46oSxmdc5ZHcqs1jmxmCVLK+dbnTB34e3Fx9v3M5mkKpVa\n573b58ZWndM+lkzElErGlbr4NhlTKrFym24da5+fTsSVTMYIkl0gWAEABtrrJx1J0lt3TSoet9Ro\nuHJdqeG6arit+w1Xruuq0Truuq4aDbWOte7LvfC1rfv1hqtavdF8bet9XLd1vquVx6ve0+1zn7Ql\n4jGlk1cIZxfdZlJxpVNxZVIJZVKtx63j7WPp1vFUMh7Z0EawAgAMtGKpIkn60HfdplQy3ufWNLXD\n1cWhq+G6mpjIa3bO6QQ49+LA5rpqSHIbrqq1hpZrdVWrDVVqdVWqDS1X66rWGqpU61pu3VaqDVVr\ndS2vOu/Cx3UtVeo6X6qqUq2r3lhf9LMkpVJxZa4ayBJKp+LKrgpk6WRCN0zlNTma9aKbfUGwAgAM\nNKdcbU2FBSNUSa2pvOadS54byiZVyiR73qbVavVGJ5xVas2wtlxphq/mn5qWqxc9bt1frta1tFzT\n0qrnzxUrWq7W1/TZGzdk9Zsfutvnn7B7BCsAwEBzSlXls/0NKmGTiMeUiMeUTXsXIxquq+V28Kq0\ng1qtE77agWzrxrxnn+kHghUAYGC5rqtiuarNk0P9bsrAi1mWsumEp2GtH1jHCgAwsJq1RA0N5xix\ngjcIVgCAgeW0CteHmQqERwhWAICB5ZSrkqR8NtXnliAqCFYAgIFVbAUrpgLhlTUFK2PMHmPMYWPM\nRy46/u3GmKCsYwYAwHVpTwXmCVbwyDWDlTFmSNKnJD160fGMpF+QdMKfpgEA4K9iqTViRY0VPLKW\nEatlSe+TNHPR8V+U9LuSKl43CgCAXnA6U4HUWMEb11wswrbtmqSaMaZzzBhzs6R9tm3/sjHm49d6\nj7GxnBKJ3qxoWygM9+RzBhl97C/613/0sf/C0se1VjHL9i0bQtNmKTz9O4i6XYXrk5L+3VpPnp8v\ndfkx16dQGNbsrNOTzxpU9LG/6F//0cf+C1Mfz55t/n6qLFVC0+Yw9W9YrSe4XvdVgcaYLZLeIulP\njTFPSZo2xjzWdQsAAOiTYqkiS1K+z3vvITque8TKtu3jkna2HxtjXrdt+wFPWwUAQA845aqGsknF\nYpdudgx045rByhhzp6RPSNohqWqMeVjS+23bPutz2wAA8BUbMMNraylef0bSu67y/A4P2wMAQE80\nGq4Wl6qansj1uymIEFZeBwAMpMWlqlxXjFjBUwQrAMBAYjsb+IFgBQAYSE6JxUHhPYIVAGAgtYMV\nU4HwEsEKADCQiuXWBswEK3iIYAUAGEhF9gmEDwhWAICBtFJjxYgVvEOwAgAMJGqs4AeCFQBgILHc\nAvxAsAIADCSnVFEiHlM6Ge93UxAhBCsAwEAqlqsaziVlWWzADO8QrAAAA8kpVzVMfRU8RrACAAyc\naq2u5Updeeqr4DGCFQBg4HBFIPxCsAIADBwWB4VfCFYAgIHTWRyUESt4jGAFABg4TnufQGqs4DGC\nFQBg4BRLTAXCHwQrAMDAoXgdfiFYAQAGTqd4nWAFjxGsAAADx2GfQPiEYAUAGDjFUrN4fYgRK3iM\nYAUAGDhOuapsOqFEnF+D8BbfKADAwCmWqkwDwhcEKwDAQHFdV0U2YIZPCFYAgIFSXq6p3nBZagG+\nIFgBAAaKwz6B8BHBCgAwUDqLg1JjBR8QrAAAA6XIBszwEcEKADBQ2IAZfiJYAQAGysp2NtRYwXsE\nKwDAQKHGCn4iWAEABkqnxopgBR8QrAAAA2VlKpBgBe8RrAAAA8UpVRSPWcqmE/1uCiKIYAUAGChO\nuap8NinLsvrdFEQQwQoAMFCKpSqF6/ANwQoAMDBq9YZKyzXqq+AbghUAYGAslttLLbCGFfyxpso9\nY8weSV+Q9Enbtj9tjNkm6Q8kJSVVJX3Atu2T/jUTAID1c7giED675oiVMWZI0qckPbrq8H+U9Fnb\nth+Q9JeSftqf5gEA4J3O4qAEK/hkLVOBy5LeJ2lm1bGfkPS51v1ZSRMetwsAAM911rCieB0+ueZU\noG3bNUk1Y8zqY4uSZIyJS/qwpI/51UAAALxSLLEBM/zV9eporVD1x5L+zrbtR6927thYTolEvNuP\nui6FwnBPPmeQ0cf+on/9Rx/7L6h9XLeaEzVbp0YD28a1CHPbo249y87+gaSDtm3/6rVOnJ8vreNj\n1q5QGNbsrNOTzxpU9LG/6F//0cf+C3Ifn5orSpIa1Vpg23gtQe7fqFhPcO1quQVjzA9Iqti2/Std\nfzIAAD3WrrGieB1+ueaIlTHmTkmfkLRDUtUY87CkjZKWjDFfa532sm3bP+FXIwEA8ILTqrGieB1+\nWUvx+jOS3uV/UwAA8FexVFU6FVeyR3W/GDysvA4AGBhOucrioPAVwQoAMBBc15VTqlJfBV8RrAAA\nA2G5Wlet3tAw+wTCRwQrAMBAKLKdDXqAYAUAGAgO29mgBwhWAICB0N6AmWAFPxGsAAADoVhu7RPI\nVCB8RLACAAwEp1NjRfE6/EOwAgAMhCI1VugBghUAYCBQY4VeIFgBAAZCe59AaqzgJ4IVAGAgFMtV\nWZY0lCFYwT8EKwDAQCiWqxrKJBWLWf1uCiKMYAUAGAhOqUp9FXxHsAIARF6j4WqxXNUw9VXwGcEK\nABB5i0tVuZLybMAMnxGsAACR57ABM3qEYAUAiDwWB0WvEKwAAJHXWRyUESv4jGAFAIg8p70BMyNW\n8BnBCgAQecXOdjYUr8NfBCsAQOS1a6woXoffCFYAgMhr7xNIjRX8RrACAESeU2YqEL1BsAIARF6x\nVFUyEVMqya89+ItvGAAg8pxSVflsUpbFBszwF8EKABB5xTIbMKM3CFYAgEirVOtartYpXEdPEKwA\nAJHWWWqBwnX0AMEKABBpbGeDXiJYAQAibWXEimAF/xGsAACRxuKg6CWCFQAg0lgcFL1EsAIARFp7\nA2b2CUQvEKwAAJHmUGOFHiJYAQAirdiusWIqED1AsAIARFr7qsChTKLPLcEgIFgBACLNKVeVSyeU\niPMrD/7jWwYAiDSnxD6B6B2CFQAgslzXVbFUpXAdPbOmCWdjzB5JX5D0Sdu2P22M2SbpjyXFJZ2Q\n9K9s2172r5kAAFy/8nJNDdfVcJbCdfTGNUesjDFDkj4l6dFVhz8m6Xdt275f0iFJP+xP8wAA6F57\nn0BGrNAra5kKXJb0Pkkzq469S9Jfte7/X0nv8bZZAACsX2fVdRYHRY9ccyrQtu2apJoxZvXhoVVT\nf6clTV/tPcbGckok4l038noUCsM9+ZxBRh/7i/71H33sv6D08ZHTi5KkqUI+MG3yQpR+lqjxYlEP\n61onzM+XPPiYaysUhjU76/TkswYVfewv+td/9LH/gtTHx04sSJKshhuYNq1XkPo3qtYTXLu9KrBo\njMm27m/RhdOEAAAEQpHtbNBj3Qarr0j67tb975b0t940BwAA71BjhV675lSgMeZOSZ+QtENS1Rjz\nsKQfkPSHxpgfk/SGpD/ys5EAAHTD6ewTSLBCb6yleP0ZNa8CvNg/8bw1AAB4qNheboF1rNAjrLwO\nAIisYrmqeMxSNt2bK9MBghUAILKc1nY2lnXNC9gBTxCsAACR5ZSrFK6jpwhWAIBIqtUbKi/XlCdY\noYcIVgCASGqvYTWco3AdvUOwAgBEUpENmNEHBCsAQCSxOCj6gWAFAIik9uKg1FihlwhWAIBIosYK\n/UCwAgBEEjVW6AeCFQAgkpwSNVboPYIVACCSnHJ7A2amAtE7BCsAQCS1a6zy2USfW4JBQrACAESS\nU6oqnYormWADZvQOwQoAEElF9glEHxCsAACR47qunFJVw1wRiB4jWAEAImepUlet3lA+S+E6eotg\nBQCInJXFQRmxQm8RrAAAkbNyRSDBCr1FsAIARE57n0BGrNBrBCsAQOR0Vl1ncVD0GMEKABA5TAWi\nXwhWAIDIaY9YEazQawQrAEDkFMvUWKE/CFYAgMihxgr9QrACAESOU67KsqRchg2Y0VsEKwBA5BRL\nVeWzScUsq99NwYAhWAEAIqdYrlK4jr4gWAEAIqXRcLVYrmqYYIU+IFgBACKluFSVKwrX0R8EKwBA\npBTba1ix1AL6gGAFAIiU9j6B1FihHwhWAIBIaW9nw1Qg+oFgBQCIFKcdrBixQh8QrAAAkeJQY4U+\nIlgBACKl2NnOhmCF3iNYAQAipb0BM8Xr6AeCFQAgUjobMGcpXkfvEawAAJHilKtKJWJKp+L9bgoG\nUFfbfhtj8pL+h6QxSWlJv2rb9pe9bBgAAN0olqoUrqNvuh2x+qAk27btd0t6WNJve9YiAADWgQ2Y\n0U/dBqs5SROt+2OtxwAA9FWlWtdytc7ioOibrqYCbdv+M2PMB40xh9QMVt9xtfPHxnJKJHoz110o\nDPfkcwYZfewv+td/9LH/+tXHs/NlSdLkhlyk/ztH+WcLu25rrD4g6aht2w8ZY/ZJ+j1J33Kl8+fn\nS1027/oUCsOanXV68lmDij72F/3rP/rYf/3s4zdONj83GVNk/zvzHfbfeoJrt1OB90r6siTZtv2C\npM3GGC6/AAD0ldNaw4rFQdEv3QarQ5LukiRjzA2SirZt1z1rFQAAXSh2trOhxgr90dVUoKTPSPp9\nY8xjrff4kHdNAgCgO2zAjH7rtni9KOl7PW4LAADr4rBPIPqMldcBAJFRbI1YsY4V+oVgBQCIjGKp\ntQEzNVboE4IVACAy2lOB+Wy3JcTA+hCsAACRUSxXNZRJKB7j1xv6g28eACAyHPYJRJ8RrAAAkdBw\nXRVLVfYJRF8RrAAAkVBerqnhuoxYoa8IVgCASFhZdZ1ghf4hWAEAIoHFQREEBCsAQCR0NmDOUmOF\n/iFYAQAioTMVSI0V+ohgBQCIhM4GzEwFoo8IVgCASKB4HUFAsAIARMJKjRXBCv1DsAIARMLKVYEU\nr6N/CFYAgEgolquKxyxlUvF+NwUDjGAFAIiEYqmqfC4py7L63RQMMIIVACASnHKFNazQdwQrAEDo\n1eoNlZfrLLWAviNYAQBCr1hmcVAEA8EKABB67BOIoCBYAQBCr1hqrmHFiBX6jWAFAAi9le1sKF5H\nfxGsAAChx1QggoJgBQAIPYrXERQEKwBA6HU2YCZYoc8IVgCA0OtswEyNFfqMYAUACD2HESsEBMEK\nABB6xXJVmVRcyQS/1tBffAMBAKHnlCpcEYhAIFgBAELNdV0Vy1Xl2YAZAUCwAgCE2lKlrlrdZcQK\ngUCwAgCEWmfVdQrXEQAEKwBAqHXWsGLECgFAsAIAhFqxzAbMCA6CFQAg1Fb2CaR4Hf1HsAIAhFon\nWDFihQAgWAEAQq2zATM1VggAghUAINScEvsEIjgIVgCAUOuMWDEViABIdPtCY8wPSPpZSTVJv2zb\n9hc9axUAAGvklKuKWZZyma5/pQGe6WrEyhgzIelXJN0n6TslfZeXjQIAYK2cUlX5bEIxy+p3U4Cu\nR6zeI+krtm07khxJ/8a7JgEAsHbFUkWj+XS/mwFI6j5Y7ZCUM8b8laQxSR+1bfvRK508NpZTIhHv\n8qOuT6Ew3JPPGWT0sb/oX//Rx/7rVR/X6w2VlmvasXl0oP67DtLPGjbdBitL0oSkfyHpBklfNcbc\nYNu2e7mT5+dLXX7M9SkUhjU76/TkswYVfewv+td/9LH/etnH5xcrcl0pk4wNzH9XvsP+W09w7faq\nwFOSnrBtu2bb9mE1pwMLXbcCAIAusAEzgqbbYPWIpG8zxsRahex5SXPeNQsAgGsrttawYnFQBEVX\nwcq27eOS/o+kpyR9SdK/tW274WXDAAC4lmJnxIrFQREMXS/6Ydv2ZyR9xsO2AABwXdr7BDJihaBg\n5XUAQGhRY4WgIVgBAEKr2BqxYp9ABAXBCgAQWk65VbzOiBUCgmAFAAitIjVWCBiCFQAgtJxyValk\nTOlkb3b3AK6FYAUACK1iqULhOgKFYAUACC2nXFWeNawQIAQrAEAoLVfrqlQbGqa+CgFCsAIAhBKF\n6wgighUAIJTa29mw1AKChGAFAAil9hpWLA6KICFYAQBCqb1PIFcFIkgIVgCAUOrUWBGsECAEKwBA\nKHU2YKZ4HQFCsAIAhFKx1NonkBorBAjBCgAQSp0RK6YCESAEKwBAKBVLVVmShrKJfjcF6CBYAQBC\nySlXlcskFI/xqwzBwbcRABBKxVKF+ioEDsEKABA6DddVsVzjikAEDsEKABA6paWaGq5L4ToCh2AF\nAAgd9glEUBGsAACh0151nX0CETQEKwBA6DjtxUEZsULAEKwAAKHDdjYIKoIVACB0igQrBBTBCgAQ\nOitTgdRYIVgIVgCA0GkXr+cZsULAEKwAAKHDBswIKoIVACB0nFJVibilTCre76YAFyBYAQBCp1iu\nKJ9NyrKsfjcFuADBCgAQOsVylcVBEUgEKwBAqFRrDZWX6ywOikAiWAEAQoU1rBBkBCsAQKh0ghVr\nWCGACFYAgFDpLA7KiBUCiGAFAAiV9ogVNVYIIoIVACBUnBI1VggughUAIFTaU4Gsuo4gWlewMsZk\njTGHjTEf9Kg9AABcVWcqkHWsEEDrHbH6D5LOetEQAADWguUWEGRdBytjzFsk3Srpi941BwCAq2vX\nWFG8jiBKrOO1n5D0EUk/dK0Tx8ZySiR6s1FmoTDck88ZZPSxv+hf/9HH/vOzj8uVunKZhKanRn37\njKDjOxxcXQUrY8wPSnrStu0jxphrnj8/X+rmY65boTCs2VmnJ581qOhjf9G//qOP/ed3H59zlpTP\nJAf2vyPfYf+tJ7h2O2L1HZJuMsZ8p6StkpaNMcds2/5K1y0BAOAaXNeVU6rqhqlMv5sCXFZXwcq2\n7e9r3zfGfFTS64QqAIDflip11Rsu9VUILNaxAgCEhsMVgQi49RSvS5Js2/6oB+0AAOCaVhYHZQ0r\nBBMjVgCA0Ci2l1pgxAoBRbACAIRGZ3FQaqwQUAQrAEBoOIxYIeAIVgCA0HDK1Fgh2AhWAIDQaNdY\ncVUggopgBQAIjXaNFVOBCCqCFQAgNJxSVTHLUja97tWCAF8QrAAAoeGUq8rnkopZVr+bAlwWwQoA\nEBrFUoWlFhBoBCsAQCjUGw0tLtXYJxCBRrACAITCYrkmiSsCEWwEKwBAKDidKwJZwwrBRbACAIRC\nsbUBM1OBCDKCFQAgFBwWB0UIEKwAAKHABswIA4IVACAUnPZUICNWCDCCFQAgFJzOiBXF6wgughUA\nIBQ6U4GMWCHACFYAgFBoF69zVSCCjGAFAAiFYqmqdDKuVDLe76YAV0SwAgCEQrFcYbQKgUewAgCE\nglOqckUgAo9gBQAIvOVqXZVagzWsEHgEKwBA4BVZdR0hQbACAASeU27vE8gaVgg2ghUAIPDaI1bU\nWCHoCFYAgMBzWBwUIUGwAgAEXntxUIrXEXQEKwBA4BU7NVYEKwQbwQoAEHgrVwVSvI5gI1gBAALP\noXgdIUGwAgAEnlOuypKUzxCsEGwEKwBA4BXLVQ1lk4rFrH43BbgqghUAIPCcEhswIxwIVgCAQGu4\nroplNmBGOCT63QAvuK6r//bXL+v0uSW5jYZiMUtxy2rexixZFz2+9DammKVLj1urH8cUi1mKWVIi\nEdP4cFoTo1lNjmaUTsb73QUAEFmlpZpclzWsEA7RCFaS5haWdGJuUfV6Q/WGq3rD7dnnj+SSmhjN\nqrAho4nRjCZHsyqMtu9nlEwQvACgW06puYYVq64jDCIRrGKWpV/8wJ0qFIY1O+t0jjdcV41WyOrc\nto5dcnzV8+3HjYar+kXv0Wi4qtTqOnt+WXMLS5pbKGtuYUlHTzk6cuL8Zds3mk9pcjSjwmhWE6MZ\nFTZkO6FrYiSjRJwZWQC4kmJrOxs2YEYYRCJYXUnMshSLW+rFgFGj4epc8cKwNbewpLlzzftHZhwd\nPn5p8LIkbRhOa7I10jU5mtHkhpX74yNpxWMELwCDa2VxUEasEHxdBytjzH+WdH/rPX7Dtu2/8KxV\nIRSLWRofyWh8JKObt2245Pl6o6F5Z1lnFpY0e+7C8HVmoaxDxxd08NjCJa+LxyxNjec0PTmkLZND\n2jw5pM0TOW0azzHSBWAgOJ0RK4IVgq+rYGWMebekPbZt322MmZD0nKSBDlbXEo/FWqNQWZntlz5f\nqzd01lnujHC1R75OnS3rxJlFHZ9b1DcveD9LG8eyK2Gr9WfTWE7JBIELQHRQY4Uw6XbE6u8lPd26\nf07SkDEmbtt23ZtmDZ5EPKaNG7LauCF7yXOu62reWdbMXDNgzbT/nFnUiTMlyZ7tnBuzLG0az2rz\nxIWBa2qcwAUgnNo1VuwTiDDoKli1AtRi6+GPSPobQpV/LGtlmnHPTROd467r6lyxouNzRc3MlTqB\n6/hcM3A98+rsqveQNo7lWiNcuU7wmp7IcdUigEDr7BPIVCBCYF3F68aY71IzWL33aueNjeWU6NEv\n70JhuCefExQbN0o33zR5wbH2CNfRk+d19JSjoycdvdm6ffbVWT376sq5MUuamhjStk3D2rNzQg/c\nsVVjI5mrfuag9XGv0b/+o4/952UfV+rN5XN2bBtTjr0CJfEdDjLLdbtb78kY8+2Sfk3SQ7Ztn73a\nubOzTk8Wlbp4uQVcyHVdnV+stKYRSxdMK7aH2mOWpT03jeuePVO6Y/fkJaNZ9LG/6F//0cf+87qP\nf+2Pvqk3Txf1mZ95QJbFXoF8h/1XKAx3/UXrtnh9VNLHJb3nWqEKwWFZlkbzaY3m07plx3jnuOu6\nWlis6Bl7Vk8cOKH9h89o/+EzyqYTevstG3Xvnmnt3DLCX2gA+sIpVTScS/J3EEKh26nA75M0KenP\njTHtYz9o2/ZRT1qFnrIsSxvyaT1451Y9eOdWzcwt6okDJ/XkSyf12PMzeuz5GW0cy+qePVP6znfu\nYoNJAD1VLFdVuMyFPUAQdVu8/llJn/W4LQiIzZNDevhdO/X+d96kV96Y1xMHTugZe1aff/yIPv/4\nEZltG3TP7VP6FrNR2XSk15gF0GfVWkNLlTpLLSA0+K2IK4rFLN1247huu3FcH3hvTd+0T+ub9pxe\nPDwn+81z+tNHXtXbTEH37pnWLTeMKRZjmB6At4osDoqQIVhhTbLphO7fu1nvf9DolYOn9cRLJ/XE\ngZN66qVTeuqlUxobTusdt23SvXumtXlyqN/NBRARncVB2ScQIUGwwnWb3JDVP7v3Rv3Te3bo8PHz\n+ocDJ/T0K6f1paeO6ktPHdWN08O6Z8+07rp1E//KBLAuK4uD8ncJwoFgha5ZlqVdW0e1a+uovv/B\n3Xr+0JyeOHBSB147qyMnXtWfPXpQ+3ZN6p49U9q7c4K9DQFct87ioAQrhATBCp5IJeN6+y2b9PZb\nNmmhuKwnXzqlJw6caC1IOqt8Nqm7btmke26f0o6pYS6bBrAm1FghbAhW8NxoPq2H7tquh+7arqOn\nHP3Diyf11Msn9eizx/Tos8e0eXJI990+rXv2TGlkiLoJAFe2sgEzf1cgHAhW8NX2TcPavmlY3/Pu\nnTpw5KyeOHBSzx+c1Z9/9ZA+99hh7ds1qfv3TmvPTeOKx5gqBHAhp11jxYgVQoJghZ5IxGN6665J\nvXXXpIrlqp566aQe378yVbghn9K9t0/rvr3T2jSW63dzAQREkRorhAzBCj2Xzyb1nm/Zpgfv3Ko3\nTjl6/IUTeurlU/rik2/oi0++IbNtg+7fN607zUalk73ZvBtAMFFjhbAhWKFvLMvSjqkR7Zga0fd+\n2y49a8/q8f0z+sej55oLkP6/V3XXrVO6f+80Be/AgHJKFWXTCa4qRmgQrBAI6WRcd++Z0t17pnRq\nvqSv7z+hf3jxhL723HF97bnj2lrI6/5907r7tin+5QoMEKdcpb4KoUKwQuBsGsvpux/YqX9+/406\n8NpZPb7/hF44NKf/+ZWD+t9fPaS33VzQ/Xs365YdY4oxigVEluu6KpaqmpzK9LspwJoRrBBY8VhM\n+3ZNat+uSS0sVvTkgZN6fP+Mnn7ltJ5+5bQmRjK6b++07rt9WhOj/MULRM1Spa56w2WUGqFCsEIo\njA6l9NBd2/Xtb9+mwzPn9fgLzYD1ha8f0V99/YhuvXFc9++d1h27C0omqMUAoqC9hhVXBCJMCFYI\nFcuytGvLqHZtGdX3v2e3vvHKaT2+/4ReOnJWLx05q6FMQnffNqX7923Wto35fjcXwDp01rBicVCE\nCMEKoZVJJXT/vs26f99mzcwt6uv7T+iJAyf0lWeO6SvPHNOOqWHdt3dab7u5oA35dL+bC+A6tdew\nongdYUKwQiRsnhzS937bLr3/gZv0wqEzenz/jF587Yxef8TRnzzyqm7aPKI7dk/qrbsL2jyRY+kG\nIAQ6GzATrBAiBCtESiIe052moDtNQfPOsr7xj6f1/MFZvfrmgl6bOa/PPfaaNo5ldcfuSd2xu6Bd\nW0YVixGygCAqMhWIECJYIbIBrTNNAAAMYElEQVTGhtN677du03u/dZuK5ar2H57TcwfndOC1s/ry\n02/qy0+/qeFcUvt2TuqO3ZO69cZxVnoHAsQpU7yO8CFYYSDks0nds2da9+yZVrVW18uvz+u5g3N6\n/tCcvv7iCX39xRNKJWK6dce47tg9qX27JzXCv5KBvnKosUIIEawwcJKJeGd9rIbr6sjMeT13cE7P\nHZzV84eaYcv6krRz66jetrugO3ZPatM4G0MDvdYpXmfECiFCsMJAi1mWdm4Z1c4to3r4XTt16myp\nE7IOHVvQoWML+vOvHtL0RE53tELWjZtHWPEd6IFiuaqYZSmb5lcVwoNvK7DKpvGcHrprux66a7vO\nL1b0wqFmXdbLr5/V3zz1hv7mqTc0OpTSvl2tuqwdY0omqMsC/OCUKsrnklzFi1AhWAFXMDKU6qyT\ntVyt6+UjZzt1WX//woz+/oUZpZNx7blxXHfcPKnbbpzQ6BB1WYBXiuWqNgyzBh3ChWAFrEE6Gdcd\nNxd0x80FNRquDh1f0PMH5/TswVk982rzjySN5lPavnFY2zfltX3TsLZvzKswlmXqELhO9UZDi0s1\ndlBA6BCsgOsUi1m6edsG3bxtg77n3Ts1c6ak5w/O6vDx8zp62tGLr53Ri6+d6ZyfTsW1bWNe2ze2\nwtamvLZMDjGFCFxFsVyTxOKgCB+CFbAOlmVpy+SQtkwOdY4Vy1UdPeXo6Kmi3jzdvH3t+HkdOrbQ\nOSdmWZqezK2Mbm3Ma9umYRX68UMAAVRsbcDM4qAIG4IV4LF8Nqlbd4zr1h3jnWOVal3H5xabget0\nUW+eKurN00Udn13Uky+tvLYwltWWiaELphInRjMU72JgLFVqevn1eT310klJjFghfAhWQA+kknHd\nOD2iG6dHOscaDVenz5U7o1tHTzs6NrvYWUurLZdOaPumvLatqt3aNJZVilXiERGn50t64fAZ7T98\nRvbRedXqrqTm+lV7bhq/xquBYCFYAX0Si1maGs9pajynt9+ySZJUKAzr0JE5HT1dXBW4irKPntM/\nHj13wetH8ykVRrOa3JDR5GhWhQ2ZzuPx4Qx7ICKwavWGDh1b0P7DZ/TC4TmdOFPqPLd9Y157d01o\n785J3TQ9wvcYoUOwAgJmNJ/W7fm0br9ponNsqVLTsdOLOtqq2Zo9V9bsubJemzmvQ8cXLnmPeMzS\nxEhGkxsyKmzIanK0edu+n8+yNhB663ypohdbo1IHjpxVeblZnJ5KxPTWXZPNMHXThMZHMn1uKbA+\nBCsgBDKphHZtHdWuraMXHK83Gjp7fllz58qaXVjS7Lmy5tq358p6+fV5SfOXvF86FVehNco12Rrl\nKmzIqjCa0eSGLJtRY91c19Wbp4vNKb5Dc3pt5rzc1nMTIxndfdsm7d05qbds38C0NiKFYAWEWDwW\n64xE3XKZ55crdc0ulDV3bkmzC+VW4FrS3EJZswtlHZstXvZ9R4ZSKoxmNDGa0ehQWiNDSY0MpTQ6\nlNJwbuU2mYj5+wMiVJYrdb3yxrzsrx3W0y+d1LyzLKl5FezuraPau2tS+3ZOaPPkECOmiCyCFRBh\n6VRcWwt5bS1cusii67pyytWVoHWurNlV918/6ejwzPmrvn8undDIUKr5J5dcuT+U0miueTvcup9O\nMSoRRXPnyp3C81femFet3pAkDWUSesdtm7R354T23DjB1X0YGAQrYEBZlqWRXEojuZRu2jxyyfP1\nRkMLxYoWFis63/5Tquj8YrV12/yzsFjRqbOlzjTPlaST8c7I10gudcHt6FBK+WxS2XRC2XRcmXRC\n2VSCEbEAqjcaOnz8vF44PKf9h87o+Nxi57mthSHt2zWpd965TRO5JIXnGEgEKwCXFY/FND6SWVMx\ncb3RULFUbYawTui6MICdX6xooVTRkRlHDfdaMawpEbeUSTXDVrYVtrLphDLpuLKp5m0unVg5J5VQ\nJp1oHms9zqbjSsRjTD211OoNlZZqKi/XVFquXXK/tFxTuX17wTlVlZbrWlqudUJ0MhHT3p0T2rez\neRXfxGjzu1IoDGt21unfDwn0EcEKwLrFYzGN5tMazV97w9yG66q0VLtwJGyxomK5qqVKXeVK8xf6\n0nJN5Uq9eb9S16n5spYr9S7bZ3VGw4ZzaVlyFY/HlIhbSsRjiseat4m41ToeU6J1LH7ROe3HFz/f\neW3nvWKKxSy5rivXlVw1bxvtx6tuGw1XDV107KJz2q+9+PXtx7V6oxmEVoeii+6Xl2uq1BrX3X/Z\nVoCdGMkol45rcyGvfTsn9JYbxrjQAbgIwQpAT8UsS/lsUvls8oKtgNai0XC1VKlrqRW+2sGrHb7a\n98vLzYDWDmerQ9qJM4uqVOuqN9Y2ahZ28ZilXKY50jc2nO7cz6Vbt5mr38+kEkzpAdeBYAUgNGKt\nkJDLdP9XV3uaynVd1RuuavWGavXm/Xq90XlcqzcufP4yxy99vnVOo6F63VW97sqymvVsltUMlasf\nW5al2EW3K+dZV3ztpedKiXjssoEpmWAaFOglghWAgWRZVmcqEAC80nWwMsZ8UtI7JLmSftK27W94\n1ioAAIAQ6uqfasaYByTttm37bkk/Iul3PG0VAABACHU7Bv6gpM9Lkm3br0gaM8ZcuhAOAADAAOl2\nKnBK0jOrHs+2jl12meaxsZwSid5cklsoDPfkcwYZfewv+td/9LH/6GN/0b/B5VXx+lUvOZmfL3n0\nMVfHonT+o4/9Rf/6jz72H33sL/rXf+sJrt1OBc6oOULVtlnSia5bAQAAEAHdBqtHJD0sScaYt0ma\nsW2b+AwAAAZaV8HKtu0nJD1jjHlCzSsCP+xpqwAAAEKo6xor27Z/3suGAAAAhB1LDgMAAHiEYAUA\nAOARghUAAIBHCFYAAAAeIVgBAAB4hGAFAADgEct13X63AQAAIBIYsQIAAPAIwQoAAMAjBCsAAACP\nEKwAAAA8QrACAADwCMEKAADAIwQrAAAAjxCsAAAAPEKwAgAA8Eii3w3wmzFmStIvqvmz/lfbtvf3\nuUmRY4z5qKStks5J+hPbtp/vb4uiqfVdfk7SNtu2a/1uT5QYY+6V9CFJKUkft237m31uUuQYY+6W\n9KNq/l38O7ZtP9PnJkWOMWZa0m9LesS27f/e7/ZEiTHm7ZJ+TM0BqY/atv3Glc4NTbAyxuyR9AVJ\nn7Rt+9OtY5+U9A5JrqSftG37G5d56Y9Iel3SFkkne9PacFpHH0tSWVJS0kwv2hpW6+zjn5b0WE8a\nGlLr6N/zkv61pL2S3iWJYHUF6+jjRUkflvQWNfuYYHUF6+jjhqTPStrRo6aG3nX09Yck/biaWeJH\nJf3Sld4zFMHKGDMk6VOSHl117AFJu23bvtsYc4uk35d0tzHmpyTd1zrtJUlTkn5dUlrST0r6971s\ne1iss48/I+msmn39U2qOEOIi6+zjg5L+Qs3/uXEZ6+lf27Z/xRjzPkk/o2bAwmV40Mcjkn5C0s/3\nuOmh4UEf39LzRofU9fS1pKRt28vGmBOSNl3tfUMRrCQtS3qfpJ9bdexBSZ+XJNu2XzHGjBljRmzb\n/i1Jv9U+yRjzMUmWpKKkXO+aHDrr6eMHJX1NzanAdM9aHD7r6eNPS9ol6a2S/qWkP+lZq8NjPf17\nl6QvSXpa0kclfaRXjQ6Z9fTxqKT/JOkXbNs+28M2h03XfYzrtua+llQyxmTULHs5erU3DUWwatWT\n1Iwxqw9P6cKh5NnWsfMXvfz3JH1MUlzSb/jYzFBbZx9nJf2hpKqk3/SvleG2nj62bfsjkmSM2SHp\nz3xtaEit8zs8pubI65AIrVe0zj7+OUkjkn7JGPO4bduf87OtYbWePm79I/fHJY0aY87Ytv2XPjc3\n1K6zrz8j6b+omZuuOisTimC1RtblDrYKzH6ox22Jqiv18V9L+usetyWqLtvHbbZtf7BH7YiqK32H\n/1bS3/a4LVF1pT6mRMA7V+rjR7VqWguesCTJtu1nJf3wWl4Q5uUWZtRMkW2bJZ3oU1uiij72H33s\nL/rXf/Sx/+jj3ll3X4c5WD0i6WFJMsa8TdKMbdtOf5sUOfSx/+hjf9G//qOP/Ucf9866+9pyXdeP\nhnnKGHOnpE+oeQlpVdJxSe+X9LOS3qnmJaYftm37hX61MezoY//Rx/6if/1HH/uPPu4dv/o6FMEK\nAAAgDMI8FQgAABAoBCsAAACPEKwAAAA8QrACAADwCMEKAADAIwQrAAAAjxCsAAAAPEKwAgAA8Mj/\nB8q37ZBwUugxAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f4c61dbf198>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "g3R8-0s2vZT6",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "6509ad3c-c535-4936-feb4-003306e04668",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528014652823,
          "user_tz": -120,
          "elapsed": 16748,
          "user": {
            "displayName": "Jannes Klaas",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "115843019312458301179"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "i = 1\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Dense(512, input_shape=(784,)))\n",
        "model.add(Activation('relu')) \n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(512))\n",
        "model.add(Activation('relu'))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(10))\n",
        "model.add(Activation('softmax')) \n",
        "opt = Adam(lr=init_lr*2**i)\n",
        "\n",
        "model.compile(loss='categorical_crossentropy', optimizer=opt,metrics=['acc'])\n",
        "\n",
        "\n",
        "hist = model.fit(X_train, Y_train, batch_size = 128, epochs=1)\n",
        "\n",
        "loss = hist.history['loss'][0]"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 60000 samples, validate on 10000 samples\n",
            "Epoch 1/4\n",
            "60000/60000 [==============================] - 4s 70us/step - loss: 0.4450 - acc: 0.8750 - val_loss: 0.1921 - val_acc: 0.9418\n",
            "Epoch 2/4\n",
            "60000/60000 [==============================] - 4s 64us/step - loss: 0.1795 - acc: 0.9481 - val_loss: 0.1301 - val_acc: 0.9610\n",
            "Epoch 3/4\n",
            "60000/60000 [==============================] - 4s 66us/step - loss: 0.1293 - acc: 0.9626 - val_loss: 0.0996 - val_acc: 0.9683\n",
            "Epoch 4/4\n",
            "60000/60000 [==============================] - 4s 64us/step - loss: 0.0996 - acc: 0.9702 - val_loss: 0.0878 - val_acc: 0.9727\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "VEqJqsYKxQPU",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          },
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "991d4b7c-fb74-464e-9221-a400e4a3c406",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1528014705865,
          "user_tz": -120,
          "elapsed": 498,
          "user": {
            "displayName": "Jannes Klaas",
            "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
            "userId": "115843019312458301179"
          }
        }
      },
      "cell_type": "code",
      "source": [
        "hist.history['loss'][0]"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.4450331950267156"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "metadata": {
        "id": "XdPPPNIexXlX",
        "colab_type": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        }
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}