{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.1"},"colab":{"name":"2-Advanced-MLP.ipynb","provenance":[],"collapsed_sections":[]}},"cells":[{"cell_type":"markdown","metadata":{"id":"ZbIvihxRm80P"},"source":["# Advanced MLP\n","- Advanced techniques for training neural networks\n","    - Weight Initialization\n","    - Nonlinearity (Activation function)\n","    - Optimizers\n","    - Batch Normalization\n","    - Dropout (Regularization)\n","    - Model Ensemble"]},{"cell_type":"code","metadata":{"id":"W12bh526m80Q"},"source":["import matplotlib.pyplot as plt\n","\n","from sklearn.model_selection import train_test_split\n","from tensorflow.keras.datasets import mnist\n","from tensorflow.keras.models import Sequential\n","from tensorflow.keras.utils import to_categorical"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"1m-pC5sOm80U"},"source":["## Load Dataset\n","- MNIST dataset\n","- source: http://yann.lecun.com/exdb/mnist/"]},{"cell_type":"code","metadata":{"id":"fH78vu3rm80V"},"source":["(X_train, y_train), (X_test, y_test) = mnist.load_data()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"RroxZgNjm80Y","executionInfo":{"status":"ok","timestamp":1604684633389,"user_tz":420,"elapsed":1426,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"2f1f1c0b-1218-41c7-cfb3-69b15968d66b","colab":{"base_uri":"https://localhost:8080/","height":282}},"source":["plt.imshow(X_train[0])    # show first number in the dataset\n","plt.show()\n","print('Label: ', y_train[0])"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAOZ0lEQVR4nO3dbYxc5XnG8euKbezamMQbB9chLjjgFAg0Jl0ZEBZQobgOqgSoCsSKIkJpnSY4Ca0rQWlV3IpWbpUQUUqRTHExFS+BBIQ/0CTUQpCowWWhBgwEDMY0NmaNWYENIX5Z3/2w42iBnWeXmTMv3vv/k1Yzc+45c24NXD5nznNmHkeEAIx/H+p0AwDag7ADSRB2IAnCDiRB2IEkJrZzY4d5ckzRtHZuEkjlV3pbe2OPR6o1FXbbiyVdJ2mCpH+LiJWl50/RNJ3qc5rZJICC9bGubq3hw3jbEyTdIOnzkk6UtMT2iY2+HoDWauYz+wJJL0TE5ojYK+lOSedV0xaAqjUT9qMk/WLY4621Ze9ie6ntPtt9+7Snic0BaEbLz8ZHxKqI6I2I3kma3OrNAaijmbBvkzRn2ONP1JYB6ELNhP1RSfNsz7V9mKQvSlpbTVsAqtbw0FtE7Le9TNKPNDT0tjoinq6sMwCVamqcPSLul3R/Rb0AaCEulwWSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJpmZxRffzxPJ/4gkfm9nS7T/3F8fUrQ1OPVBc9+hjdxTrU7/uYv3Vaw+rW3u893vFdXcOvl2sn3r38mL9uD9/pFjvhKbCbnuLpN2SBiXtj4jeKpoCUL0q9uy/FxE7K3gdAC3EZ3YgiWbDHpJ+bPsx20tHeoLtpbb7bPft054mNwegUc0exi+MiG22j5T0gO2fR8TDw58QEaskrZKkI9wTTW4PQIOa2rNHxLba7Q5J90paUEVTAKrXcNhtT7M9/eB9SYskbayqMQDVauYwfpake20ffJ3bI+KHlXQ1zkw4YV6xHpMnFeuvnPWRYv2d0+qPCfd8uDxe/JPPlMebO+k/fzm9WP/Hf1lcrK8/+fa6tZf2vVNcd2X/54r1j//k0PtE2nDYI2KzpM9U2AuAFmLoDUiCsANJEHYgCcIOJEHYgST4imsFBs/+bLF+7S03FOufmlT/q5jj2b4YLNb/5vqvFOsT3y4Pf51+97K6tenb9hfXnbyzPDQ3tW99sd6N2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs1dg8nOvFOuP/WpOsf6pSf1VtlOp5dtPK9Y3v1X+Kepbjv1+3dqbB8rj5LP++b+L9VY69L7AOjr27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQhCPaN6J4hHviVJ/Ttu11i4FLTi/Wdy0u/9zzhCcPL9af+Pr1H7ing67Z+TvF+qNnlcfRB994s1iP0+v/APGWbxZX1dwlT5SfgPdZH+u0KwZGnMuaPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4exeYMPOjxfrg6wPF+ku31x8rf/rM1cV1F/zDN4r1I2/o3HfK8cE1Nc5ue7XtHbY3DlvWY/sB25tqtzOqbBhA9cZyGH+LpPfOen+lpHURMU/SutpjAF1s1LBHxMOS3nsceZ6kNbX7aySdX3FfACrW6G/QzYqI7bX7r0qaVe+JtpdKWipJUzS1wc0BaFbTZ+Nj6Axf3bN8EbEqInojoneSJje7OQANajTs/bZnS1Ltdkd1LQFohUbDvlbSxbX7F0u6r5p2ALTKqJ/Zbd8h6WxJM21vlXS1pJWS7rJ9qaSXJV3YyibHu8Gdrze1/r5djc/v/ukvPVOsv3bjhPILHCjPsY7uMWrYI2JJnRJXxwCHEC6XBZIg7EAShB1IgrADSRB2IAmmbB4HTrji+bq1S04uD5r8+9HrivWzvnBZsT79e48U6+ge7NmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2ceB0rTJr3/thOK6/7f2nWL9ymtuLdb/8sILivX43w/Xrc35+58V11Ubf+Y8A/bsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEUzYnN/BHpxfrt1397WJ97sQpDW/707cuK9bn3bS9WN+/eUvD2x6vmpqyGcD4QNiBJAg7kARhB5Ig7EAShB1IgrADSTDOjqI4Y36xfsTKrcX6HZ/8UcPbPv7BPy7Wf/tv63+PX5IGN21ueNuHqqbG2W2vtr3D9sZhy1bY3mZ7Q+3v3CobBlC9sRzG3yJp8QjLvxsR82t/91fbFoCqjRr2iHhY0kAbegHQQs2coFtm+8naYf6Mek+yvdR2n+2+fdrTxOYANKPRsN8o6VhJ8yVtl/Sdek+MiFUR0RsRvZM0ucHNAWhWQ2GPiP6IGIyIA5JukrSg2rYAVK2hsNuePezhBZI21nsugO4w6ji77TsknS1ppqR+SVfXHs+XFJK2SPpqRJS/fCzG2cejCbOOLNZfuei4urX1V1xXXPdDo+yLvvTSomL9zYWvF+vjUWmcfdRJIiJiyQiLb266KwBtxeWyQBKEHUiCsANJEHYgCcIOJMFXXNExd20tT9k81YcV67+MvcX6H3zj8vqvfe/64rqHKn5KGgBhB7Ig7EAShB1IgrADSRB2IAnCDiQx6rfekNuBheWfkn7xC+Upm0+av6VubbRx9NFcP3BKsT71vr6mXn+8Yc8OJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kwzj7OufekYv35b5bHum86Y02xfuaU8nfKm7En9hXrjwzMLb/AgVF/3TwV9uxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7IeAiXOPLtZfvOTjdWsrLrqzuO4fHr6zoZ6qcFV/b7H+0HWnFesz1pR/dx7vNuqe3fYc2w/afsb207a/VVveY/sB25tqtzNa3y6ARo3lMH6/pOURcaKk0yRdZvtESVdKWhcR8yStqz0G0KVGDXtEbI+Ix2v3d0t6VtJRks6TdPBayjWSzm9VkwCa94E+s9s+RtIpktZLmhURBy8+flXSrDrrLJW0VJKmaGqjfQJo0pjPxts+XNIPJF0eEbuG12JodsgRZ4iMiFUR0RsRvZM0ualmATRuTGG3PUlDQb8tIu6pLe63PbtWny1pR2taBFCFUQ/jbVvSzZKejYhrh5XWSrpY0sra7X0t6XAcmHjMbxXrb/7u7GL9or/7YbH+px+5p1hvpeXby8NjP/vX+sNrPbf8T3HdGQcYWqvSWD6znyHpy5Kesr2htuwqDYX8LtuXSnpZ0oWtaRFAFUYNe0T8VNKIk7tLOqfadgC0CpfLAkkQdiAJwg4kQdiBJAg7kARfcR2jibN/s25tYPW04rpfm/tQsb5ken9DPVVh2baFxfrjN5anbJ75/Y3Fes9uxsq7BXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUgizTj73t8v/2zx3j8bKNavOu7+urVFv/F2Qz1VpX/wnbq1M9cuL657/F//vFjveaM8Tn6gWEU3Yc8OJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0mkGWffcn7537XnT767Zdu+4Y1ji/XrHlpUrHuw3o/7Djn+mpfq1ub1ry+uO1isYjxhzw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSTgiyk+w50i6VdIsSSFpVURcZ3uFpD+R9FrtqVdFRP0vfUs6wj1xqpn4FWiV9bFOu2JgxAszxnJRzX5JyyPicdvTJT1m+4Fa7bsR8e2qGgXQOmOZn327pO21+7ttPyvpqFY3BqBaH+gzu+1jJJ0i6eA1mMtsP2l7te0ZddZZarvPdt8+7WmqWQCNG3PYbR8u6QeSLo+IXZJulHSspPka2vN/Z6T1ImJVRPRGRO8kTa6gZQCNGFPYbU/SUNBvi4h7JCki+iNiMCIOSLpJ0oLWtQmgWaOG3bYl3Szp2Yi4dtjy2cOedoGk8nSeADpqLGfjz5D0ZUlP2d5QW3aVpCW252toOG6LpK+2pEMAlRjL2fifShpp3K44pg6gu3AFHZAEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IIlRf0q60o3Zr0l6ediimZJ2tq2BD6Zbe+vWviR6a1SVvR0dER8bqdDWsL9v43ZfRPR2rIGCbu2tW/uS6K1R7eqNw3ggCcIOJNHpsK/q8PZLurW3bu1LordGtaW3jn5mB9A+nd6zA2gTwg4k0ZGw215s+znbL9i+shM91GN7i+2nbG+w3dfhXlbb3mF747BlPbYfsL2pdjviHHsd6m2F7W21926D7XM71Nsc2w/afsb207a/VVve0feu0Fdb3re2f2a3PUHS85I+J2mrpEclLYmIZ9raSB22t0jqjYiOX4Bh+0xJb0m6NSJOqi37J0kDEbGy9g/ljIi4okt6WyHprU5P412brWj28GnGJZ0v6Svq4HtX6OtCteF968SefYGkFyJic0TslXSnpPM60EfXi4iHJQ28Z/F5ktbU7q/R0P8sbVent64QEdsj4vHa/d2SDk4z3tH3rtBXW3Qi7EdJ+sWwx1vVXfO9h6Qf237M9tJONzOCWRGxvXb/VUmzOtnMCEadxrud3jPNeNe8d41Mf94sTtC938KI+Kykz0u6rHa42pVi6DNYN42djmka73YZYZrxX+vke9fo9OfN6kTYt0maM+zxJ2rLukJEbKvd7pB0r7pvKur+gzPo1m53dLifX+umabxHmmZcXfDedXL6806E/VFJ82zPtX2YpC9KWtuBPt7H9rTaiRPZniZpkbpvKuq1ki6u3b9Y0n0d7OVdumUa73rTjKvD713Hpz+PiLb/STpXQ2fkX5T0V53ooU5fn5T0RO3v6U73JukODR3W7dPQuY1LJX1U0jpJmyT9l6SeLurtPyQ9JelJDQVrdod6W6ihQ/QnJW2o/Z3b6feu0Fdb3jculwWS4AQdkARhB5Ig7EAShB1IgrADSRB2IAnCDiTx/65XcTNOWsh5AAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}},{"output_type":"stream","text":["Label:  5\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"bGgToRjIm80c","executionInfo":{"status":"ok","timestamp":1604684633391,"user_tz":420,"elapsed":1420,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"c0e35f90-7ffb-4a1c-97a1-4a04ff47ab59","colab":{"base_uri":"https://localhost:8080/","height":282}},"source":["plt.imshow(X_test[0])    # show first number in the dataset\n","plt.show()\n","print('Label: ', y_test[0])"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAANiklEQVR4nO3df4wc9XnH8c8n/kV8QGtDcF3j4ISQqE4aSHWBRNDKESUFImSiJBRLtVyJ5lALElRRW0QVBalVSlEIok0aySluHESgaQBhJTSNa6W1UKljg4yxgdaEmsau8QFOaxPAP/DTP24cHXD7vWNndmft5/2SVrs7z87Oo/F9PLMzO/t1RAjA8e9tbTcAoD8IO5AEYQeSIOxAEoQdSGJ6Pxc207PiBA31c5FAKq/qZzoYBzxRrVbYbV8s6XZJ0yT9bUTcXHr9CRrSeb6wziIBFGyIdR1rXe/G254m6auSLpG0WNIy24u7fT8AvVXnM/u5kp6OiGci4qCkeyQtbaYtAE2rE/YFkn4y7vnOatrr2B6xvcn2pkM6UGNxAOro+dH4iFgZEcMRMTxDs3q9OAAd1An7LkkLxz0/vZoGYADVCftGSWfZfpftmZKulLSmmbYANK3rU28Rcdj2tZL+SWOn3lZFxLbGOgPQqFrn2SPiQUkPNtQLgB7i67JAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJGoN2Wx7h6T9kl6TdDgihptoCkDzaoW98rGIeKGB9wHQQ+zGA0nUDXtI+oHtR2yPTPQC2yO2N9nedEgHai4OQLfq7sZfEBG7bJ8maa3tpyJi/fgXRMRKSSsl6WTPjZrLA9ClWlv2iNhV3Y9Kul/SuU00BaB5XYfd9pDtk44+lvRxSVubagxAs+rsxs+TdL/to+/zrYj4fiNdAWhc12GPiGcknd1gLwB6iFNvQBKEHUiCsANJEHYgCcIOJNHEhTApvPjZj3asvXP508V5nxqdV6wfPDCjWF9wd7k+e+dLHWtHNj9RnBd5sGUHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQ4zz5Ff/xH3+pY+9TQT8szn1lz4UvK5R2HX+5Yu/35j9Vc+LHrR6NndKwN3foLxXmnr3uk6XZax5YdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JwRP8GaTnZc+M8X9i35TXpZ58+r2PthQ+W/8+c82R5Hf/0V1ysz/zg/xbrt3zgvo61i97+SnHe7718YrH+idmdr5Wv65U4WKxvODBUrC854VDXy37P964u1t87srHr927ThlinfbF3wj8otuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATXs0/R0Hc2FGr13vvkerPrr39pScfan5+/qLzsfy3/5v0tS97TRUdTM/2VI8X60Jbdxfop6+8t1n91Zuff25+9o/xb/MejSbfstlfZHrW9ddy0ubbX2t5e3c/pbZsA6prKbvw3JF38hmk3SFoXEWdJWlc9BzDAJg17RKyXtPcNk5dKWl09Xi3p8ob7AtCwbj+zz4uIox+onpPUcTAz2yOSRiTpBM3ucnEA6qp9ND7GrqTpeKVHRKyMiOGIGJ6hWXUXB6BL3YZ9j+35klTdjzbXEoBe6DbsayStqB6vkPRAM+0A6JVJP7Pbvltjv1x+qu2dkr4g6WZJ37Z9laRnJV3RyyZRdvi5PR1rQ/d2rknSa5O899B3Xuyio2bs+b2PFuvvn1n+8/3S3vd1rC36u2eK8x4uVo9Nk4Y9IpZ1KB2bv0IBJMXXZYEkCDuQBGEHkiDsQBKEHUiCS1zRmulnLCzWv3LjV4r1GZ5WrP/D7b/ZsXbK7oeL8x6P2LIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKcZ0drnvrDBcX6h2eVh7LedrA8HPXcJ15+yz0dz9iyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASnGdHTx34xIc71h799G2TzF0eQej3r7uuWH/7v/1okvfPhS07kARhB5Ig7EAShB1IgrADSRB2IAnCDiTBeXb01H9f0nl7cqLL59GX/ddFxfrs7z9WrEexms+kW3bbq2yP2t46btpNtnfZ3lzdLu1tmwDqmspu/DckXTzB9Nsi4pzq9mCzbQFo2qRhj4j1kvb2oRcAPVTnAN21trdUu/lzOr3I9ojtTbY3HdKBGosDUEe3Yf+apDMlnSNpt6RbO70wIlZGxHBEDM+Y5MIGAL3TVdgjYk9EvBYRRyR9XdK5zbYFoGldhd32/HFPPylpa6fXAhgMk55nt323pCWSTrW9U9IXJC2xfY7GTmXukHR1D3vEAHvbSScV68t//aGOtX1HXi3OO/rFdxfrsw5sLNbxepOGPSKWTTD5jh70AqCH+LoskARhB5Ig7EAShB1IgrADSXCJK2rZftP7i/Xvnvo3HWtLt3+qOO+sBzm11iS27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBOfZUfR/v/ORYn3Lb/9Vsf7jw4c61l76y9OL887S7mIdbw1bdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgvPsyU1f8MvF+vWf//tifZbLf0JXPra8Y+0d/8j16v3Elh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkuA8+3HO08v/xGd/d2ex/pkTXyzW79p/WrE+7/OdtydHinOiaZNu2W0vtP1D20/Y3mb7umr6XNtrbW+v7uf0vl0A3ZrKbvxhSZ+LiMWSPiLpGtuLJd0gaV1EnCVpXfUcwICaNOwRsTsiHq0e75f0pKQFkpZKWl29bLWky3vVJID63tJndtuLJH1I0gZJ8yLi6I+EPSdpXod5RiSNSNIJmt1tnwBqmvLReNsnSrpX0vURsW98LSJCUkw0X0SsjIjhiBieoVm1mgXQvSmF3fYMjQX9roi4r5q8x/b8qj5f0mhvWgTQhEl3421b0h2SnoyIL48rrZG0QtLN1f0DPekQ9Zz9vmL5z067s9bbf/WLnynWf/Gxh2u9P5ozlc/s50taLulx25uraTdqLOTftn2VpGclXdGbFgE0YdKwR8RDktyhfGGz7QDoFb4uCyRB2IEkCDuQBGEHkiDsQBJc4nocmLb4vR1rI/fU+/rD4lXXFOuL7vz3Wu+P/mHLDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJcJ79OPDUH3T+Yd/LZu/rWJuK0//lYPkFMeEPFGEAsWUHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQ4z34MePWyc4v1dZfdWqgy5BbGsGUHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSSmMj77QknflDRPUkhaGRG3275J0mclPV+99MaIeLBXjWb2P+dPK9bfOb37c+l37T+tWJ+xr3w9O1ezHzum8qWaw5I+FxGP2j5J0iO211a12yLiS71rD0BTpjI++25Ju6vH+20/KWlBrxsD0Ky39Jnd9iJJH5K0oZp0re0ttlfZnvC3kWyP2N5ke9MhHajVLIDuTTnstk+UdK+k6yNin6SvSTpT0jka2/JP+AXtiFgZEcMRMTxDsxpoGUA3phR22zM0FvS7IuI+SYqIPRHxWkQckfR1SeWrNQC0atKw27akOyQ9GRFfHjd9/riXfVLS1ubbA9CUqRyNP1/SckmP295cTbtR0jLb52js7MsOSVf3pEPU8hcvLi7WH/6tRcV67H68wW7QpqkcjX9IkicocU4dOIbwDTogCcIOJEHYgSQIO5AEYQeSIOxAEo4+Drl7sufGeb6wb8sDstkQ67Qv9k50qpwtO5AFYQeSIOxAEoQdSIKwA0kQdiAJwg4k0dfz7Lafl/TsuEmnSnqhbw28NYPa26D2JdFbt5rs7YyIeMdEhb6G/U0LtzdFxHBrDRQMam+D2pdEb93qV2/sxgNJEHYgibbDvrLl5ZcMam+D2pdEb93qS2+tfmYH0D9tb9kB9AlhB5JoJey2L7b9H7aftn1DGz10YnuH7cdtb7a9qeVeVtketb113LS5ttfa3l7dTzjGXku93WR7V7XuNtu+tKXeFtr+oe0nbG+zfV01vdV1V+irL+ut75/ZbU+T9J+SLpK0U9JGScsi4om+NtKB7R2ShiOi9S9g2P4NSS9J+mZEfKCadoukvRFxc/Uf5ZyI+JMB6e0mSS+1PYx3NVrR/PHDjEu6XNLvqsV1V+jrCvVhvbWxZT9X0tMR8UxEHJR0j6SlLfQx8CJivaS9b5i8VNLq6vFqjf2x9F2H3gZCROyOiEerx/slHR1mvNV1V+irL9oI+wJJPxn3fKcGa7z3kPQD24/YHmm7mQnMi4jd1ePnJM1rs5kJTDqMdz+9YZjxgVl33Qx/XhcH6N7sgoj4NUmXSLqm2l0dSDH2GWyQzp1OaRjvfplgmPGfa3PddTv8eV1thH2XpIXjnp9eTRsIEbGruh+VdL8GbyjqPUdH0K3uR1vu5+cGaRjviYYZ1wCsuzaHP28j7BslnWX7XbZnSrpS0poW+ngT20PVgRPZHpL0cQ3eUNRrJK2oHq+Q9ECLvbzOoAzj3WmYcbW87lof/jwi+n6TdKnGjsj/WNKfttFDh77eLemx6rat7d4k3a2x3bpDGju2cZWkUyStk7Rd0j9LmjtAvd0p6XFJWzQWrPkt9XaBxnbRt0jaXN0ubXvdFfrqy3rj67JAEhygA5Ig7EAShB1IgrADSRB2IAnCDiRB2IEk/h9BCfQTVPflJQAAAABJRU5ErkJggg==\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}},{"output_type":"stream","text":["Label:  7\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"cKDggu2Gm80f"},"source":["# reshaping X data: (n, 28, 28) => (n, 784)\n","X_train = X_train.reshape((X_train.shape[0], -1))\n","X_test = X_test.reshape((X_test.shape[0], -1))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"MUFEC2K_m80k"},"source":["# converting y data into categorical (one-hot encoding)\n","y_train = to_categorical(y_train)\n","y_test = to_categorical(y_test)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Y1D9nxJ_m80m","executionInfo":{"status":"ok","timestamp":1604684633546,"user_tz":420,"elapsed":1561,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"cd5049e9-e452-4751-faa9-7d723eca60ee","colab":{"base_uri":"https://localhost:8080/"}},"source":["print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["(60000, 784) (10000, 784) (60000, 10) (10000, 10)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"X7X7pN-Sm80p"},"source":["## Basic MLP model\n","- Naive MLP model without any alterations"]},{"cell_type":"code","metadata":{"id":"OPUXkSYmm80p"},"source":["from tensorflow.keras.models import Sequential\n","from tensorflow.keras.layers import Activation, Dense\n","from tensorflow.keras import optimizers"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"b32x9pvIm80r"},"source":["model = Sequential()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XPnz8u9ym80u"},"source":["model.add(Dense(50, input_shape = (784, )))\n","model.add(Activation('sigmoid'))\n","model.add(Dense(50))\n","model.add(Activation('sigmoid'))\n","model.add(Dense(50))\n","model.add(Activation('sigmoid'))\n","model.add(Dense(50))\n","model.add(Activation('sigmoid'))\n","model.add(Dense(10))\n","model.add(Activation('softmax'))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ua5rjdGYm80y"},"source":["sgd = optimizers.SGD(lr = 0.001)\n","model.compile(optimizer = sgd, loss = 'categorical_crossentropy', metrics = ['accuracy'])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Lj23wJ2lm800","executionInfo":{"status":"ok","timestamp":1604684711031,"user_tz":420,"elapsed":79023,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"ac601aeb-351d-4689-ae21-fb2bfbe6a27b","colab":{"base_uri":"https://localhost:8080/"}},"source":["history = model.fit(X_train, y_train, batch_size = 256, validation_split = 0.3, epochs = 100, verbose = 1)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Epoch 1/100\n","165/165 [==============================] - 1s 7ms/step - loss: 2.4838 - accuracy: 0.0995 - val_loss: 2.4486 - val_accuracy: 0.0966\n","Epoch 2/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.4176 - accuracy: 0.0995 - val_loss: 2.3965 - val_accuracy: 0.0966\n","Epoch 3/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3761 - accuracy: 0.0995 - val_loss: 2.3634 - val_accuracy: 0.0966\n","Epoch 4/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3496 - accuracy: 0.0995 - val_loss: 2.3422 - val_accuracy: 0.0966\n","Epoch 5/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3327 - accuracy: 0.0995 - val_loss: 2.3283 - val_accuracy: 0.0966\n","Epoch 6/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3217 - accuracy: 0.0997 - val_loss: 2.3193 - val_accuracy: 0.0970\n","Epoch 7/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3145 - accuracy: 0.1019 - val_loss: 2.3135 - val_accuracy: 0.0989\n","Epoch 8/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3099 - accuracy: 0.0827 - val_loss: 2.3096 - val_accuracy: 0.1059\n","Epoch 9/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3070 - accuracy: 0.1140 - val_loss: 2.3072 - val_accuracy: 0.1079\n","Epoch 10/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.3050 - accuracy: 0.1143 - val_loss: 2.3055 - val_accuracy: 0.1079\n","Epoch 11/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3038 - accuracy: 0.1143 - val_loss: 2.3044 - val_accuracy: 0.1079\n","Epoch 12/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3029 - accuracy: 0.1143 - val_loss: 2.3036 - val_accuracy: 0.1079\n","Epoch 13/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3023 - accuracy: 0.1143 - val_loss: 2.3031 - val_accuracy: 0.1079\n","Epoch 14/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3019 - accuracy: 0.1143 - val_loss: 2.3028 - val_accuracy: 0.1079\n","Epoch 15/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.3016 - accuracy: 0.1143 - val_loss: 2.3025 - val_accuracy: 0.1079\n","Epoch 16/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3014 - accuracy: 0.1143 - val_loss: 2.3023 - val_accuracy: 0.1079\n","Epoch 17/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3013 - accuracy: 0.1143 - val_loss: 2.3021 - val_accuracy: 0.1079\n","Epoch 18/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3011 - accuracy: 0.1143 - val_loss: 2.3020 - val_accuracy: 0.1079\n","Epoch 19/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3010 - accuracy: 0.1143 - val_loss: 2.3018 - val_accuracy: 0.1079\n","Epoch 20/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.3009 - accuracy: 0.1143 - val_loss: 2.3017 - val_accuracy: 0.1079\n","Epoch 21/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3008 - accuracy: 0.1143 - val_loss: 2.3017 - val_accuracy: 0.1079\n","Epoch 22/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3007 - accuracy: 0.1143 - val_loss: 2.3016 - val_accuracy: 0.1079\n","Epoch 23/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3007 - accuracy: 0.1143 - val_loss: 2.3015 - val_accuracy: 0.1079\n","Epoch 24/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3006 - accuracy: 0.1143 - val_loss: 2.3014 - val_accuracy: 0.1079\n","Epoch 25/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3005 - accuracy: 0.1143 - val_loss: 2.3013 - val_accuracy: 0.1079\n","Epoch 26/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3004 - accuracy: 0.1143 - val_loss: 2.3012 - val_accuracy: 0.1079\n","Epoch 27/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.3003 - accuracy: 0.1143 - val_loss: 2.3011 - val_accuracy: 0.1079\n","Epoch 28/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3002 - accuracy: 0.1143 - val_loss: 2.3010 - val_accuracy: 0.1079\n","Epoch 29/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3001 - accuracy: 0.1143 - val_loss: 2.3009 - val_accuracy: 0.1079\n","Epoch 30/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3001 - accuracy: 0.1143 - val_loss: 2.3008 - val_accuracy: 0.1079\n","Epoch 31/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.3000 - accuracy: 0.1143 - val_loss: 2.3007 - val_accuracy: 0.1079\n","Epoch 32/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.2999 - accuracy: 0.1143 - val_loss: 2.3007 - val_accuracy: 0.1079\n","Epoch 33/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2998 - accuracy: 0.1143 - val_loss: 2.3006 - val_accuracy: 0.1079\n","Epoch 34/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2997 - accuracy: 0.1143 - val_loss: 2.3005 - val_accuracy: 0.1079\n","Epoch 35/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2996 - accuracy: 0.1143 - val_loss: 2.3004 - val_accuracy: 0.1079\n","Epoch 36/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2995 - accuracy: 0.1143 - val_loss: 2.3003 - val_accuracy: 0.1079\n","Epoch 37/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2995 - accuracy: 0.1143 - val_loss: 2.3002 - val_accuracy: 0.1079\n","Epoch 38/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.2994 - accuracy: 0.1143 - val_loss: 2.3002 - val_accuracy: 0.1079\n","Epoch 39/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.2993 - accuracy: 0.1143 - val_loss: 2.3001 - val_accuracy: 0.1079\n","Epoch 40/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.2992 - accuracy: 0.1143 - val_loss: 2.3000 - val_accuracy: 0.1079\n","Epoch 41/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2991 - accuracy: 0.1143 - val_loss: 2.2999 - val_accuracy: 0.1079\n","Epoch 42/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.2991 - accuracy: 0.1143 - val_loss: 2.2998 - val_accuracy: 0.1079\n","Epoch 43/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2990 - accuracy: 0.1143 - val_loss: 2.2998 - val_accuracy: 0.1079\n","Epoch 44/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2989 - accuracy: 0.1143 - val_loss: 2.2997 - val_accuracy: 0.1079\n","Epoch 45/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2988 - accuracy: 0.1143 - val_loss: 2.2996 - val_accuracy: 0.1079\n","Epoch 46/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2987 - accuracy: 0.1143 - val_loss: 2.2995 - val_accuracy: 0.1079\n","Epoch 47/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2987 - accuracy: 0.1143 - val_loss: 2.2994 - val_accuracy: 0.1079\n","Epoch 48/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2986 - accuracy: 0.1143 - val_loss: 2.2994 - val_accuracy: 0.1079\n","Epoch 49/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2985 - accuracy: 0.1143 - val_loss: 2.2993 - val_accuracy: 0.1079\n","Epoch 50/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2984 - accuracy: 0.1143 - val_loss: 2.2992 - val_accuracy: 0.1079\n","Epoch 51/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.2983 - accuracy: 0.1143 - val_loss: 2.2991 - val_accuracy: 0.1079\n","Epoch 52/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2983 - accuracy: 0.1143 - val_loss: 2.2991 - val_accuracy: 0.1079\n","Epoch 53/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2982 - accuracy: 0.1143 - val_loss: 2.2990 - val_accuracy: 0.1079\n","Epoch 54/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2981 - accuracy: 0.1143 - val_loss: 2.2989 - val_accuracy: 0.1079\n","Epoch 55/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2980 - accuracy: 0.1143 - val_loss: 2.2988 - val_accuracy: 0.1079\n","Epoch 56/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.2979 - accuracy: 0.1143 - val_loss: 2.2988 - val_accuracy: 0.1079\n","Epoch 57/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2979 - accuracy: 0.1143 - val_loss: 2.2987 - val_accuracy: 0.1079\n","Epoch 58/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2978 - accuracy: 0.1143 - val_loss: 2.2986 - val_accuracy: 0.1079\n","Epoch 59/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.2977 - accuracy: 0.1143 - val_loss: 2.2985 - val_accuracy: 0.1079\n","Epoch 60/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2976 - accuracy: 0.1143 - val_loss: 2.2984 - val_accuracy: 0.1079\n","Epoch 61/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2975 - accuracy: 0.1143 - val_loss: 2.2983 - val_accuracy: 0.1079\n","Epoch 62/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2975 - accuracy: 0.1143 - val_loss: 2.2982 - val_accuracy: 0.1079\n","Epoch 63/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2974 - accuracy: 0.1143 - val_loss: 2.2981 - val_accuracy: 0.1079\n","Epoch 64/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2973 - accuracy: 0.1143 - val_loss: 2.2981 - val_accuracy: 0.1079\n","Epoch 65/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2972 - accuracy: 0.1143 - val_loss: 2.2980 - val_accuracy: 0.1079\n","Epoch 66/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2971 - accuracy: 0.1143 - val_loss: 2.2979 - val_accuracy: 0.1079\n","Epoch 67/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.2971 - accuracy: 0.1143 - val_loss: 2.2978 - val_accuracy: 0.1079\n","Epoch 68/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2970 - accuracy: 0.1143 - val_loss: 2.2977 - val_accuracy: 0.1079\n","Epoch 69/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2969 - accuracy: 0.1143 - val_loss: 2.2976 - val_accuracy: 0.1079\n","Epoch 70/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2968 - accuracy: 0.1143 - val_loss: 2.2976 - val_accuracy: 0.1079\n","Epoch 71/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2967 - accuracy: 0.1143 - val_loss: 2.2975 - val_accuracy: 0.1079\n","Epoch 72/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2967 - accuracy: 0.1143 - val_loss: 2.2974 - val_accuracy: 0.1079\n","Epoch 73/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2966 - accuracy: 0.1143 - val_loss: 2.2973 - val_accuracy: 0.1079\n","Epoch 74/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2965 - accuracy: 0.1143 - val_loss: 2.2972 - val_accuracy: 0.1079\n","Epoch 75/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2964 - accuracy: 0.1143 - val_loss: 2.2972 - val_accuracy: 0.1079\n","Epoch 76/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2963 - accuracy: 0.1143 - val_loss: 2.2971 - val_accuracy: 0.1079\n","Epoch 77/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2963 - accuracy: 0.1143 - val_loss: 2.2970 - val_accuracy: 0.1079\n","Epoch 78/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2962 - accuracy: 0.1143 - val_loss: 2.2969 - val_accuracy: 0.1079\n","Epoch 79/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2961 - accuracy: 0.1143 - val_loss: 2.2969 - val_accuracy: 0.1079\n","Epoch 80/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2960 - accuracy: 0.1143 - val_loss: 2.2968 - val_accuracy: 0.1079\n","Epoch 81/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2959 - accuracy: 0.1143 - val_loss: 2.2967 - val_accuracy: 0.1079\n","Epoch 82/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.2958 - accuracy: 0.1143 - val_loss: 2.2966 - val_accuracy: 0.1079\n","Epoch 83/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2958 - accuracy: 0.1143 - val_loss: 2.2965 - val_accuracy: 0.1079\n","Epoch 84/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2957 - accuracy: 0.1143 - val_loss: 2.2964 - val_accuracy: 0.1079\n","Epoch 85/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2956 - accuracy: 0.1143 - val_loss: 2.2963 - val_accuracy: 0.1079\n","Epoch 86/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2955 - accuracy: 0.1143 - val_loss: 2.2963 - val_accuracy: 0.1079\n","Epoch 87/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2954 - accuracy: 0.1143 - val_loss: 2.2962 - val_accuracy: 0.1079\n","Epoch 88/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2953 - accuracy: 0.1143 - val_loss: 2.2961 - val_accuracy: 0.1079\n","Epoch 89/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2952 - accuracy: 0.1143 - val_loss: 2.2960 - val_accuracy: 0.1079\n","Epoch 90/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.2952 - accuracy: 0.1143 - val_loss: 2.2959 - val_accuracy: 0.1079\n","Epoch 91/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2951 - accuracy: 0.1143 - val_loss: 2.2958 - val_accuracy: 0.1079\n","Epoch 92/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2950 - accuracy: 0.1143 - val_loss: 2.2958 - val_accuracy: 0.1079\n","Epoch 93/100\n","165/165 [==============================] - 1s 4ms/step - loss: 2.2949 - accuracy: 0.1143 - val_loss: 2.2957 - val_accuracy: 0.1079\n","Epoch 94/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2948 - accuracy: 0.1143 - val_loss: 2.2956 - val_accuracy: 0.1079\n","Epoch 95/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2947 - accuracy: 0.1143 - val_loss: 2.2955 - val_accuracy: 0.1079\n","Epoch 96/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2946 - accuracy: 0.1143 - val_loss: 2.2954 - val_accuracy: 0.1079\n","Epoch 97/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2945 - accuracy: 0.1143 - val_loss: 2.2953 - val_accuracy: 0.1079\n","Epoch 98/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2945 - accuracy: 0.1143 - val_loss: 2.2952 - val_accuracy: 0.1079\n","Epoch 99/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2944 - accuracy: 0.1143 - val_loss: 2.2951 - val_accuracy: 0.1079\n","Epoch 100/100\n","165/165 [==============================] - 1s 5ms/step - loss: 2.2943 - accuracy: 0.1143 - val_loss: 2.2950 - val_accuracy: 0.1079\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"w4egvBDNm803","executionInfo":{"status":"ok","timestamp":1604684711175,"user_tz":420,"elapsed":79159,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"7cf5d450-074c-4d5f-8530-bcade9e6deaf","colab":{"base_uri":"https://localhost:8080/","height":265}},"source":["plt.plot(history.history['accuracy'])\n","plt.plot(history.history['val_accuracy'])\n","plt.plot(history.history['loss'])\n","plt.plot(history.history['val_loss'])\n","plt.legend(['train acc', 'valid acc', 'train loss', 'valid loss'], loc = 'upper left')\n","plt.show()"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAeuElEQVR4nO3de3hU9b3v8fd3kpA0XEOIipA24PHREBJuAVOpgvVS8AJSRejBWnn2xkfrZXvsdpvj6bbU2hY9tra0XooWq1VxW6gXWpSntGC6zyPWwAFMhX0AxRKuCZAYJFyS+Z4/ZhiGkBtkQsjK5/U8kZm1fuu3vmut8ZOVNWt+Y+6OiIh0fqGOLkBERBJDgS4iEhAKdBGRgFCgi4gEhAJdRCQgkjtqxf369fOcnJyOWr2ISKe0atWqSnfPamxehwV6Tk4OpaWlHbV6EZFOycw+bWqeLrmIiASEAl1EJCAU6CIiAdFh19Abc+TIEcrLyzl48GBHl9KppaWlMXDgQFJSUjq6FBE5jVoMdDPLBl4EzgYcmOfuP2/QZjzwJvBJdNLv3f3hky2mvLycnj17kpOTg5md7OICuDt79uyhvLycQYMGdXQ5InIateYMvQ74jruvNrOewCoz+5O7f9Sg3V/d/dq2FHPw4EGFeRuZGZmZmVRUVHR0KSJymrV4Dd3dd7j76ujjGmA9MKC9ClKYt532oUjXdFJvippZDjACeL+R2V82s7Vm9raZ5TWx/G1mVmpmpad6Bnm4/jA7Pt9B2MOntLyISFC1OtDNrAewCLjX3T9rMHs18CV3Hwb8AnijsT7cfZ67F7p7YVZWox90atGh+kPsrd3LvoP7Tmn55lRVVfHUU0+d0rJXX301VVVVCa5IRKT1WhXoZpZCJMxfdvffN5zv7p+5+/7o4yVAipn1S2ilUT1SetA9pTsVtRXUhesS2ndzgV5X1/y6lixZQp8+fRJaj4jIyWgx0C1yQfbXwHp3/2kTbc6JtsPMxkT73ZPIQo/yQ4c4uwrC9fVU1lYmtO/i4mI2b97M8OHDuf/++1mxYgWXXHIJkyZNYsiQIQBcf/31jBo1iry8PObNmxdbNicnh8rKSrZs2UJubi6zZs0iLy+Pq666itra2hPWtXjxYi666CJGjBjBFVdcwa5duwDYv38/M2fOJD8/n4KCAhYtWgTAO++8w8iRIxk2bBiXX355QrdbRIKhNXe5jAW+CXxoZmui0x4Evgjg7s8ANwJ3mFkdUAtM9zZ+t933F/+dj7Y3vLID1NcTPniQcHKIw0k7+ULyZqyVV46GnNuL713X6OV9AObMmUNZWRlr1kQ2c8WKFaxevZqysrLYLYDz58+nb9++1NbWMnr0aG644QYyMzOP62fjxo0sWLCAZ599lptuuolFixZx8803H9fmK1/5CitXrsTMeO6553jsscf4yU9+wg9+8AN69+7Nhx9+CMC+ffuoqKhg1qxZlJSUMGjQIPbu3duq7RWRrqXFQHf3/wSavW3C3X8J/DJRRTUrKQlLSSF05AhJZhyuP0JqUmq7rW7MmDHH3c89d+5cXn/9dQC2bt3Kxo0bTwj0QYMGMXz4cABGjRrFli1bTui3vLycadOmsWPHDg4fPhxbx7Jly3j11Vdj7TIyMli8eDGXXnpprE3fvn0Tuo0iEgxn1CdF4zV3Ju3hMIc//oT6w4f4Rz+nf++B9E7t3S51dO/ePfZ4xYoVLFu2jPfee4/09HTGjx/f6KdaU1OP/YJJSkpq9JLL3XffzX333cekSZNYsWIFs2fPbpf6RaTr6JRjuVgoREr2QAzoXx1iW802Pj/8eZv77dmzJzU1NU3Or66uJiMjg/T0dDZs2MDKlStPeV3V1dUMGBC5nf+FF16ITb/yyit58sknY8/37dtHUVERJSUlfPJJ5IO4uuQiIo3plIEOEEpNJaV/f7odquecatj62T84WNe2MWAyMzMZO3YsQ4cO5f777z9h/oQJE6irqyM3N5fi4mKKiopOeV2zZ89m6tSpjBo1in79jt0Q9N3vfpd9+/YxdOhQhg0bxvLly8nKymLevHl8/etfZ9iwYUybNu2U1ysiwWVtfO/ylBUWFnrDL7hYv349ubm5J9VPXWUlR3bu5FA3o6JvEtl9ckhLTktkqZ3SqexLETnzmdkqdy9sbF6nPUM/KrlfP7plZ5N6BM6urKe8cjOVtZV01C8qEZGO0ukDHSCpd2+6fSmHFA8xoNIJb9/J1n1bOFR3qKNLExE5bc7Yu1xOVlKP7qSdfz51FRX03LOH7gc/p6pqI/W9utO791mkJ6dr0CoRCbTABDqAJSeT0r8/SX37cmT3Lnp9VgMHPudI5SdUpCUR6p5OWs8MvtCtO0mhpI4uV0QkoQIV6EeFUlNJzf4iXl9PXVUV4X176Ln/MOyvgV01fJ4E9SkhvFsKoZRuhFK6kdStG0kpqSQndyMpOUVn8yLS6QQy0I+ypCRSMjNJyczEw2HqD3zOoZpq7GAtyYePkFRzCPNj19kdOBL9CRu4GR4CzHCL/Bt7jEU+P3s0+Bv+2/AxkXHKnUY+dms0NvXYOprdyMYbHKzaw19+8O1oH3Zs/bEyDSwU97iR+o+uImTH6guFsPiiQtG+G6znxP3CsfXFbZdZKG4brYn+4t7qOTotriY7ro7Q8euOrf74vo/rM3S0gGP1W2zasXYWsrhf9I1vb2Q9cfs1Wkf8CUJs3aHQcf3F1mkWt48Na6S/+P0Xaxuy2DAYx52QhI5tr8UdP4vfv3HHKVZfXB0WatAmbj0W9zo6Wutx2xpfa6yOY7XG75/j9mV8+7jjfmzasf1ncfuJE15/Fvewhf9fY/01aNNE+zPtxC/QgR7PQiGSe/QkuUfP2DR3J1x3hLrDB6k7fJBw3RHCdfV4fR2Ew9EfB4/8mHvsuQE4nDtqNDtK/8aO3bv5tx/N4aUnThy/bOK3ZvKjf/1XRg499unXE14GjdyU05aXStKBQ/R/eXkbepDTwZt4LJ1TGGL/43r8Y4s7vgY7Jo3haz96oeHibdZlAr0xZkZSSjeSUrqR2r3XKXYSomfuUHrmwhvjvtpok6Tu3UkbPJj0IccC3Zv437c1t1uesKzH/hN7GjLo97e/RH4ZRTrGwx59GMajXxDi0V9WEBlS4Vgfx+Yfne4ejr5iIzXE2oTDsbrdw3HrJDYttm7Csfo9HD5Wtvux7Qo3qO/YzjmxJo5f7thu8LiavMH2xn05SlyNDdsSt24/+ss8bt803AYnbv3x9YXjt6GxfXn8PvO4+o7VErd8rOa4bT+u7mOri013jt/H8a+L445XOO5hXNv49vGvtdhrxo+rq+G0E+prrL+Gxzru+DU+zU/sL64fb1A38XXFyoufH7/80ddUg9dAY+uM7zf2GnEs/jiEI9tp0UWzRo6kPXTpQG+ouLiY7Oxs7rzzTiDyac4ePXpw++23M3nyZPbt28eRI0d45JFHmDx58nHLbtmyhWuvvZaysjJqa2uZOXMma9eu5cILL6S2tjb6p2ncn90YDz/8MIsXL6a2tpaLL76YX/3qV4RCITZt2sTtt99ORUUFSUlJ/O53v+O8887j0Ucf5aWXXiIUCjFx4kTmzJnT5LYkhZLI6tW/fXaUiJyRztxAf7sYdn6Y2D7PyYeJTYfgtGnTuPfee2OB/tprr7F06VLS0tJ4/fXX6dWrF5WVlRQVFTFp0qQmr589/fTTpKens379etatW8fIJn4b33XXXTz00EMAfPOb3+QPf/gD1113HTNmzKC4uJgpU6Zw8OBBwuEwb7/9Nm+++Sbvv/8+6enpGs9FRE5w5gZ6BxgxYgS7d+9m+/btVFRUkJGRQXZ2NkeOHOHBBx+kpKSEUCjEtm3b2LVrF+ecc06j/ZSUlHDPPfcAUFBQQEFBQaPtli9fzmOPPcaBAwfYu3cveXl5jB8/nm3btjFlyhQA0tIiwxgsW7aMmTNnkp6eDmgIXRE50Zkb6M2cSbenqVOnsnDhQnbu3BkbBOvll1+moqKCVatWkZKSQk5OTqPD5p6MgwcP8u1vf5vS0lKys7OZPXt2m/sUka4tEB/9T6Rp06bx6quvsnDhQqZOnQpEhro966yzSElJYfny5Xz66afN9nHppZfyyiuvAFBWVsa6detOaHM0vPv168f+/ftZuHAhEBnCd+DAgbzxRuR7tg8dOsSBAwe48soref755zlw4ACgIXRF5EQK9Aby8vKoqalhwIAB9O8feVNxxowZlJaWkp+fz4svvsiFF17YbB933HEH+/fvJzc3l4ceeohRo0ad0KZPnz7MmjWLoUOH8rWvfY3Ro0fH5v32t79l7ty5FBQUcPHFF7Nz504mTJjApEmTKCwsZPjw4Tz++OOJ3XAR6fQ6/fC50jjtS5FgCvTwuSIiEqFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgx6mqquKpp546pWWvvvpqqqqqWt1+9uzZupdcRBJKgR6nuUCvq6trdtklS5bQp0+f9ihLRKRVFOhxiouL2bx5M8OHD+f+++9nxYoVXHLJJUyaNIkhQ4YAcP311zNq1Cjy8vKYN29ebNmcnBwqKyvZsmULubm5zJo1i7y8PK666ipqa2ubXe+aNWsoKiqioKCAKVOmsG/fPgDmzp3LkCFDKCgoYPr06QC8++67DB8+nOHDhzNixAhqamraaW+ISGdzxg7O9ejfHmXD3g0J7fPCvhfywJgHmpw/Z84cysrKWLNmDQArVqxg9erVlJWVMWjQIADmz59P3759qa2tZfTo0dxwww1kZmYe18/GjRtZsGABzz77LDfddBOLFi3i5ptvbnK9t9xyC7/4xS8YN24cDz30EN///vf52c9+xpw5c/jkk09ITU2NXc55/PHHefLJJxk7diz79++PjcYoIqIz9BaMGTMmFuYQOWseNmwYRUVFbN26lY0bN56wzKBBgxg+fDgAo0aNYsuWLU32X11dTVVVFePGjQPgW9/6FiUlJUBk6N0ZM2bw0ksvkZwc+d07duxY7rvvPubOnUtVVVVsuojIGZsGzZ1Jn07du3ePPV6xYgXLli3jvffeIz09nfHjxzc65G1qamrscVJSUouXXJryxz/+kZKSEhYvXswPf/hDPvzwQ4qLi7nmmmtYsmQJY8eOZenSpS0OFiYiXYPO0OP07Nmz2WvS1dXVZGRkkJ6ezoYNG1i5cmWb19m7d28yMjL461//CkRGWhw3bhzhcJitW7dy2WWX8eijj1JdXc3+/fvZvHkz+fn5PPDAA4wePZoNGxJ7WUpEOq8Wz9DNLBt4ETibyPebznP3nzdoY8DPgauBA8Ct7r468eW2r8zMTMaOHcvQoUOZOHEi11xzzXHzJ0yYwDPPPENubi4XXHABRUVFCVnvCy+8wO23386BAwcYPHgwzz//PPX19dx8881UV1fj7txzzz306dOHf//3f2f58uWEQiHy8vKYOHFiQmoQkc6vxeFzzaw/0N/dV5tZT2AVcL27fxTX5mrgbiKBfhHwc3e/qLl+NXxu+9K+FAmmNg2f6+47jp5tu3sNsB4Y0KDZZOBFj1gJ9In+IhARkdPkpK6hm1kOMAJ4v8GsAcDWuOflnBj6mNltZlZqZqUVFRUnV6mIiDSr1YFuZj2ARcC97v7ZqazM3ee5e6G7F2ZlZZ1KFyIi0oRWBbqZpRAJ85fd/feNNNkGZMc9HxidJiIip0mLgR69g+XXwHp3/2kTzd4CbrGIIqDa3XcksE4REWlBaz5YNBb4JvChma2JTnsQ+CKAuz8DLCFyh8smIrctzkx8qSIi0pzW3OXyn+5u7l7g7sOjP0vc/ZlomBO9u+VOdz/P3fPdvbSlfoOiR48eAGzfvp0bb7yx0Tbjx4+n4S2azU0XETkV+qRogpx77rksXLiwo8sQkS5MgR6nuLiYJ598Mvb86JdQ7N+/n8svv5yRI0eSn5/Pm2++ecKyW7ZsYejQoQDU1tYyffp0cnNzmTJlSqvGclmwYAH5+fkMHTqUBx6IjGNTX1/PrbfeytChQ8nPz+eJJ54AGh9WV0TkjB2ca+ePfsSh9YkdpyQ190LOefDBJudPmzaNe++9lzvvvBOA1157jaVLl5KWlsbrr79Or169qKyspKioiEmTJhF5v/hETz/9NOnp6axfv55169YxcuTIZuvavn07DzzwAKtWrSIjI4OrrrqKN954g+zsbLZt20ZZWRlAbAjdxobVFRHRGXqcESNGsHv3brZv387atWvJyMggOzsbd+fBBx+koKCAK664gm3btrFr164m+ykpKYmNf15QUEBBQUGz6/3ggw8YP348WVlZJCcnM2PGDEpKShg8eDAff/wxd999N++88w69evWK9dlwWF0RkTM2DZo7k25PU6dOZeHChezcuZNp06YB8PLLL1NRUcGqVatISUkhJyen0WFzEy0jI4O1a9eydOlSnnnmGV577TXmz5/f6LC6CnYR0Rl6A9OmTePVV19l4cKFTJ06FYgMm3vWWWeRkpLC8uXL+fTTT5vt49JLL+WVV14BoKysjHXr1jXbfsyYMbz77rtUVlZSX1/PggULGDduHJWVlYTDYW644QYeeeQRVq9e3eSwuiIiOq1rIC8vj5qaGgYMGED//pHxxWbMmMF1111Hfn4+hYWFLX6hxB133MHMmTPJzc0lNzeXUaNGNdu+f//+zJkzh8suuwx355prrmHy5MmsXbuWmTNnEg6HAfjxj3/c5LC6IiItDp/bXjR8bvvSvhQJpjYNnysiIp2DAl1EJCDOuEDvqEtAQaJ9KNI1nVGBnpaWxp49exRIbeDu7Nmzh7S0tI4uRUROszPqLpeBAwdSXl6Ovs2obdLS0hg4cGBHlyEip9kZFegpKSkMGjSoo8sQEemUzqhLLiIicuoU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIB0WKgm9l8M9ttZmVNzB9vZtVmtib681DiyxQRkZa05jtFfwP8EnixmTZ/dfdrE1KRiIickhbP0N29BNh7GmoREZE2SNQ19C+b2Voze9vM8ppqZGa3mVmpmZVWVFQkaNUiIgKJCfTVwJfcfRjwC+CNphq6+zx3L3T3wqysrASsWkREjmpzoLv7Z+6+P/p4CZBiZv3aXJmIiJyUNge6mZ1jZhZ9PCba55629isiIienxbtczGwBMB7oZ2blwPeAFAB3fwa4EbjDzOqAWmC6u3u7VSwiIo1qMdDd/RstzP8lkdsaRUSkA+mToiIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgWgx0M5tvZrvNrKyJ+WZmc81sk5mtM7ORiS9TRERa0poz9N8AE5qZPxE4P/pzG/B028sSEZGT1WKgu3sJsLeZJpOBFz1iJdDHzPonqkAREWmdRFxDHwBsjXteHp12AjO7zcxKzay0oqIiAasWEZGjTuubou4+z90L3b0wKyvrdK5aRCTwEhHo24DsuOcDo9NEROQ0SkSgvwXcEr3bpQiodvcdCehXREROQnJLDcxsATAe6Gdm5cD3gBQAd38GWAJcDWwCDgAz26tYERFpWouB7u7faGG+A3cmrCIRETkl+qSoiEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCYhWBbqZTTCz/zKzTWZW3Mj8W82swszWRH/+OfGliohIc5JbamBmScCTwJVAOfCBmb3l7h81aPof7n5XO9QoIiKt0Joz9DHAJnf/2N0PA68Ck9u3LBEROVmtCfQBwNa45+XRaQ3dYGbrzGyhmWUnpDoREWm1RL0puhjIcfcC4E/AC401MrPbzKzUzEorKioStGoREYHWBfo2IP6Me2B0Woy773H3Q9GnzwGjGuvI3ee5e6G7F2ZlZZ1KvSIi0oTWBPoHwPlmNsjMugHTgbfiG5hZ/7ink4D1iStRRERao8W7XNy9zszuApYCScB8d/+7mT0MlLr7W8A9ZjYJqAP2Are2Y80iItIIc/cOWXFhYaGXlpZ2yLpFRDorM1vl7oWNzdMnRUVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAiK5ows4E/19ezWrP91H2CHsjntHVyQiQVIwsDeFOX0T3m+nC/R9nx/m48rPT3KpSCi7Ox4Oc6j2Mw7u/4zDn1eR/4VKvnh4M+z8EGp28Fn1PjJrqphIXbvULyLyUfZ0+Of/nfB+O12gf1zyMvkrv3NSyxgQIkySNX6qHcY40mcw27wf66v70b3n+RSedw7JSSEsATWLiMS76LxL2qXfThfoORcMY2fNrJNezi0JQiGwJEKp3emW3ouktF78aXs3frouhYqdkV1xc9EX+d51eaQk6e0FEelcOl2gZw4eSebgkQnr7xvAhAmHef7/fMK5fb7AtNHZmOm8XEQ6n04X6O0ho3s37rvqgo4uQ0SkTXRdQUQkIBToIiIB0apAN7MJZvZfZrbJzIobmZ9qZv8Rnf++meUkulAREWlei4FuZknAk8BEYAjwDTMb0qDZPwH73P2/AU8Ajya6UBERaV5rztDHAJvc/WN3Pwy8Ckxu0GYy8EL08ULgctOtIiIip1VrAn0AsDXueXl0WqNt3L0OqAYyG3ZkZreZWamZlVZUVJxaxSIi0qjT+qaou89z90J3L8zKyjqdqxYRCbzWBPo2IDvu+cDotEbbmFky0BvYk4gCRUSkdVrzwaIPgPPNbBCR4J4O/PcGbd4CvgW8B9wI/MW9+TEKV61aVWlmn558yQD0AypPcdnOrCtud1fcZuia290VtxlOfru/1NSMFgPd3evM7C5gKZAEzHf3v5vZw0Cpu78F/Br4rZltAvYSCf2W+j3lay5mVuruhae6fGfVFbe7K24zdM3t7orbDInd7lZ99N/dlwBLGkx7KO7xQWBqIgoSEZFTo0+KiogERGcN9HkdXUAH6Yrb3RW3GbrmdnfFbYYEbre18N6liIh0Ep31DF1ERBpQoIuIBESnC/SWRn4MAjPLNrPlZvaRmf3dzP4lOr2vmf3JzDZG/83o6Frbg5klmdn/NbM/RJ8Pio7iuSk6qme3jq4xkcysj5ktNLMNZrbezL7cFY61mf2P6Ou7zMwWmFlaEI+1mc03s91mVhY3rdHjaxFzo9u/zsxO6uvZOlWgt3LkxyCoA77j7kOAIuDO6HYWA3929/OBP0efB9G/AOvjnj8KPBEdzXMfkdE9g+TnwDvufiEwjMi2B/pYm9kA4B6g0N2HEvmMy3SCeax/A0xoMK2p4zsROD/6cxvw9MmsqFMFOq0b+bHTc/cd7r46+riGyP/gAzh+VMsXgOs7psL2Y2YDgWuA56LPDfgqkVE8IWDbbWa9gUuJfDgPdz/s7lV0gWNN5HMwX4gOF5IO7CCAx9rdS4h84DJeU8d3MvCiR6wE+phZ/9auq7MFemtGfgyU6JeFjADeB8529x3RWTuBszuorPb0M+DfgHD0eSZQFR3FE4J3zAcBFcDz0ctMz5lZdwJ+rN19G/A48A8iQV4NrCLYxzpeU8e3TRnX2QK9SzGzHsAi4F53/yx+XnSsnEDdc2pm1wK73X1VR9dyGiUDI4Gn3X0E8DkNLq8E9FhnEDkbHQScC3TnxMsSXUIij29nC/TWjPwYCGaWQiTMX3b330cn7zr651f0390dVV87GQtMMrMtRC6nfZXI9eU+0T/LIXjHvBwod/f3o88XEgn4oB/rK4BP3L3C3Y8Avydy/IN8rOM1dXzblHGdLdBjIz9G3/2eTmSkx0CJXjf+NbDe3X8aN+voqJZE/33zdNfWntz9f7r7QHfPIXJs/+LuM4DlREbxhIBtt7vvBLaa2QXRSZcDHxHwY03kUkuRmaVHX+9Htzuwx7qBpo7vW8At0btdioDquEszLXP3TvUDXA38P2Az8L86up522savEPkTbB2wJvpzNZHryX8GNgLLgL4dXWs77oPxwB+ijwcDfwM2Ab8DUju6vgRv63CgNHq83wAyusKxBr4PbADKgN8CqUE81sACIu8THCHyF9k/NXV8ASNyJ99m4EMidwG1el366L+ISEB0tksuIiLSBAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQg/j+4PmDWFJaXmwAAAABJRU5ErkJggg==\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"8g6__3Onm805"},"source":["Training and validation accuracy seems to improve after around 60 epochs"]},{"cell_type":"code","metadata":{"id":"TwlF_-OSm805","executionInfo":{"status":"ok","timestamp":1604684711638,"user_tz":420,"elapsed":79615,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"9386ef36-2632-4ab1-d50d-70896eed329e","colab":{"base_uri":"https://localhost:8080/"}},"source":["results = model.evaluate(X_test, y_test)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["313/313 [==============================] - 0s 1ms/step - loss: 2.2944 - accuracy: 0.1135\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"DSeFp3jfm808","executionInfo":{"status":"ok","timestamp":1604684711640,"user_tz":420,"elapsed":79611,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"4648dd6d-4283-4d19-812c-a28e24876e78","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Test accuracy: ', results[1])"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Test accuracy:  0.11349999904632568\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"aQLGUyPsm80-"},"source":["## 1. Weight Initialization\n","- Changing weight initialization scheme can sometimes improve training of the model by preventing vanishing gradient problem up to some degree\n","- He normal or Xavier normal initialization schemes are SOTA at the moment\n","- Doc: https://keras.io/initializers/"]},{"cell_type":"code","metadata":{"id":"0kOFoXgtm80-"},"source":["# from now on, create a function to generate (return) models\n","def mlp_model():\n","    model = Sequential()\n","    \n","    model.add(Dense(50, input_shape = (784, ), kernel_initializer='he_normal'))     # use he_normal initializer\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(50, kernel_initializer='he_normal'))                            # use he_normal initializer\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(50, kernel_initializer='he_normal'))                            # use he_normal initializer\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(50, kernel_initializer='he_normal'))                            # use he_normal initializer\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(10, kernel_initializer='he_normal'))                            # use he_normal initializer\n","    model.add(Activation('softmax'))\n","    \n","    sgd = optimizers.SGD(lr = 0.001)\n","    model.compile(optimizer = sgd, loss = 'categorical_crossentropy', metrics = ['accuracy'])\n","    \n","    return model"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"scrolled":true,"id":"T_4QYzNGm81A","executionInfo":{"status":"ok","timestamp":1604684947977,"user_tz":420,"elapsed":315938,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"e05cac1e-6c6e-431e-b9a1-86090ed6eb74","colab":{"base_uri":"https://localhost:8080/"}},"source":["model = mlp_model()\n","history = model.fit(X_train, y_train, validation_split = 0.3, epochs = 100, verbose = 1)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Epoch 1/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.3656 - accuracy: 0.1143 - val_loss: 2.3063 - val_accuracy: 0.1079\n","Epoch 2/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.3001 - accuracy: 0.1143 - val_loss: 2.2984 - val_accuracy: 0.1079\n","Epoch 3/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2958 - accuracy: 0.1143 - val_loss: 2.2955 - val_accuracy: 0.1079\n","Epoch 4/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2929 - accuracy: 0.1143 - val_loss: 2.2927 - val_accuracy: 0.1079\n","Epoch 5/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2902 - accuracy: 0.1143 - val_loss: 2.2900 - val_accuracy: 0.1079\n","Epoch 6/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2876 - accuracy: 0.1143 - val_loss: 2.2872 - val_accuracy: 0.1079\n","Epoch 7/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2849 - accuracy: 0.1143 - val_loss: 2.2844 - val_accuracy: 0.1079\n","Epoch 8/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2821 - accuracy: 0.1143 - val_loss: 2.2815 - val_accuracy: 0.1079\n","Epoch 9/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2791 - accuracy: 0.1143 - val_loss: 2.2785 - val_accuracy: 0.1080\n","Epoch 10/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2760 - accuracy: 0.1176 - val_loss: 2.2755 - val_accuracy: 0.1084\n","Epoch 11/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2731 - accuracy: 0.1170 - val_loss: 2.2724 - val_accuracy: 0.1083\n","Epoch 12/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2700 - accuracy: 0.1160 - val_loss: 2.2692 - val_accuracy: 0.1257\n","Epoch 13/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2668 - accuracy: 0.1228 - val_loss: 2.2659 - val_accuracy: 0.1266\n","Epoch 14/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2634 - accuracy: 0.1285 - val_loss: 2.2626 - val_accuracy: 0.1390\n","Epoch 15/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2599 - accuracy: 0.1586 - val_loss: 2.2588 - val_accuracy: 0.1309\n","Epoch 16/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2561 - accuracy: 0.1680 - val_loss: 2.2549 - val_accuracy: 0.1638\n","Epoch 17/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2520 - accuracy: 0.1872 - val_loss: 2.2507 - val_accuracy: 0.2122\n","Epoch 18/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2476 - accuracy: 0.2439 - val_loss: 2.2460 - val_accuracy: 0.1938\n","Epoch 19/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2429 - accuracy: 0.2310 - val_loss: 2.2410 - val_accuracy: 0.2599\n","Epoch 20/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2377 - accuracy: 0.2752 - val_loss: 2.2356 - val_accuracy: 0.2886\n","Epoch 21/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2321 - accuracy: 0.2968 - val_loss: 2.2296 - val_accuracy: 0.3260\n","Epoch 22/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2260 - accuracy: 0.3393 - val_loss: 2.2234 - val_accuracy: 0.3254\n","Epoch 23/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2194 - accuracy: 0.3507 - val_loss: 2.2164 - val_accuracy: 0.3616\n","Epoch 24/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2121 - accuracy: 0.3837 - val_loss: 2.2088 - val_accuracy: 0.3750\n","Epoch 25/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.2041 - accuracy: 0.3929 - val_loss: 2.2003 - val_accuracy: 0.4189\n","Epoch 26/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.1953 - accuracy: 0.4090 - val_loss: 2.1911 - val_accuracy: 0.4568\n","Epoch 27/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.1858 - accuracy: 0.4468 - val_loss: 2.1812 - val_accuracy: 0.4503\n","Epoch 28/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.1753 - accuracy: 0.4504 - val_loss: 2.1702 - val_accuracy: 0.4673\n","Epoch 29/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.1637 - accuracy: 0.4713 - val_loss: 2.1582 - val_accuracy: 0.4663\n","Epoch 30/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.1507 - accuracy: 0.4746 - val_loss: 2.1445 - val_accuracy: 0.4826\n","Epoch 31/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.1362 - accuracy: 0.4836 - val_loss: 2.1293 - val_accuracy: 0.5059\n","Epoch 32/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.1204 - accuracy: 0.4987 - val_loss: 2.1129 - val_accuracy: 0.5090\n","Epoch 33/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.1032 - accuracy: 0.5108 - val_loss: 2.0953 - val_accuracy: 0.5013\n","Epoch 34/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.0844 - accuracy: 0.5070 - val_loss: 2.0756 - val_accuracy: 0.5133\n","Epoch 35/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.0637 - accuracy: 0.5155 - val_loss: 2.0542 - val_accuracy: 0.5234\n","Epoch 36/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.0411 - accuracy: 0.5282 - val_loss: 2.0305 - val_accuracy: 0.5218\n","Epoch 37/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 2.0164 - accuracy: 0.5228 - val_loss: 2.0052 - val_accuracy: 0.5376\n","Epoch 38/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.9896 - accuracy: 0.5318 - val_loss: 1.9774 - val_accuracy: 0.5453\n","Epoch 39/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.9607 - accuracy: 0.5416 - val_loss: 1.9474 - val_accuracy: 0.5428\n","Epoch 40/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.9295 - accuracy: 0.5428 - val_loss: 1.9155 - val_accuracy: 0.5483\n","Epoch 41/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.8963 - accuracy: 0.5511 - val_loss: 1.8813 - val_accuracy: 0.5532\n","Epoch 42/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.8610 - accuracy: 0.5515 - val_loss: 1.8453 - val_accuracy: 0.5653\n","Epoch 43/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.8240 - accuracy: 0.5595 - val_loss: 1.8076 - val_accuracy: 0.5727\n","Epoch 44/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.7854 - accuracy: 0.5665 - val_loss: 1.7686 - val_accuracy: 0.5749\n","Epoch 45/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.7452 - accuracy: 0.5699 - val_loss: 1.7278 - val_accuracy: 0.5812\n","Epoch 46/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.7038 - accuracy: 0.5786 - val_loss: 1.6858 - val_accuracy: 0.5847\n","Epoch 47/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.6616 - accuracy: 0.5838 - val_loss: 1.6430 - val_accuracy: 0.5901\n","Epoch 48/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.6185 - accuracy: 0.5881 - val_loss: 1.5999 - val_accuracy: 0.5997\n","Epoch 49/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.5749 - accuracy: 0.5949 - val_loss: 1.5558 - val_accuracy: 0.6107\n","Epoch 50/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.5313 - accuracy: 0.6037 - val_loss: 1.5124 - val_accuracy: 0.6128\n","Epoch 51/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.4881 - accuracy: 0.6118 - val_loss: 1.4686 - val_accuracy: 0.6141\n","Epoch 52/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.4447 - accuracy: 0.6153 - val_loss: 1.4259 - val_accuracy: 0.6274\n","Epoch 53/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.4026 - accuracy: 0.6263 - val_loss: 1.3840 - val_accuracy: 0.6333\n","Epoch 54/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.3612 - accuracy: 0.6324 - val_loss: 1.3434 - val_accuracy: 0.6470\n","Epoch 55/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.3206 - accuracy: 0.6457 - val_loss: 1.3039 - val_accuracy: 0.6506\n","Epoch 56/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.2821 - accuracy: 0.6517 - val_loss: 1.2672 - val_accuracy: 0.6568\n","Epoch 57/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.2454 - accuracy: 0.6589 - val_loss: 1.2306 - val_accuracy: 0.6644\n","Epoch 58/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.2104 - accuracy: 0.6664 - val_loss: 1.1962 - val_accuracy: 0.6756\n","Epoch 59/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.1771 - accuracy: 0.6752 - val_loss: 1.1638 - val_accuracy: 0.6823\n","Epoch 60/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.1453 - accuracy: 0.6855 - val_loss: 1.1338 - val_accuracy: 0.6892\n","Epoch 61/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.1154 - accuracy: 0.6917 - val_loss: 1.1042 - val_accuracy: 0.7019\n","Epoch 62/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.0874 - accuracy: 0.7005 - val_loss: 1.0775 - val_accuracy: 0.7031\n","Epoch 63/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.0615 - accuracy: 0.7063 - val_loss: 1.0526 - val_accuracy: 0.7127\n","Epoch 64/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.0366 - accuracy: 0.7143 - val_loss: 1.0286 - val_accuracy: 0.7214\n","Epoch 65/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.0128 - accuracy: 0.7223 - val_loss: 1.0067 - val_accuracy: 0.7251\n","Epoch 66/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.9912 - accuracy: 0.7271 - val_loss: 0.9850 - val_accuracy: 0.7341\n","Epoch 67/100\n","1313/1313 [==============================] - 3s 2ms/step - loss: 0.9702 - accuracy: 0.7352 - val_loss: 0.9644 - val_accuracy: 0.7395\n","Epoch 68/100\n","1313/1313 [==============================] - 3s 2ms/step - loss: 0.9500 - accuracy: 0.7442 - val_loss: 0.9455 - val_accuracy: 0.7449\n","Epoch 69/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.9313 - accuracy: 0.7496 - val_loss: 0.9282 - val_accuracy: 0.7540\n","Epoch 70/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.9141 - accuracy: 0.7563 - val_loss: 0.9115 - val_accuracy: 0.7555\n","Epoch 71/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.8965 - accuracy: 0.7618 - val_loss: 0.8964 - val_accuracy: 0.7647\n","Epoch 72/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.8794 - accuracy: 0.7692 - val_loss: 0.8789 - val_accuracy: 0.7761\n","Epoch 73/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.8636 - accuracy: 0.7744 - val_loss: 0.8655 - val_accuracy: 0.7739\n","Epoch 74/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.8482 - accuracy: 0.7787 - val_loss: 0.8504 - val_accuracy: 0.7805\n","Epoch 75/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.8342 - accuracy: 0.7845 - val_loss: 0.8375 - val_accuracy: 0.7871\n","Epoch 76/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.8214 - accuracy: 0.7887 - val_loss: 0.8230 - val_accuracy: 0.7866\n","Epoch 77/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.8069 - accuracy: 0.7934 - val_loss: 0.8115 - val_accuracy: 0.7932\n","Epoch 78/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.7945 - accuracy: 0.7967 - val_loss: 0.8010 - val_accuracy: 0.7973\n","Epoch 79/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.7816 - accuracy: 0.8010 - val_loss: 0.7875 - val_accuracy: 0.8012\n","Epoch 80/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.7692 - accuracy: 0.8051 - val_loss: 0.7758 - val_accuracy: 0.8077\n","Epoch 81/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.7577 - accuracy: 0.8087 - val_loss: 0.7662 - val_accuracy: 0.8058\n","Epoch 82/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.7457 - accuracy: 0.8120 - val_loss: 0.7550 - val_accuracy: 0.8089\n","Epoch 83/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.7345 - accuracy: 0.8165 - val_loss: 0.7434 - val_accuracy: 0.8163\n","Epoch 84/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.7244 - accuracy: 0.8196 - val_loss: 0.7346 - val_accuracy: 0.8154\n","Epoch 85/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.7130 - accuracy: 0.8231 - val_loss: 0.7243 - val_accuracy: 0.8217\n","Epoch 86/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.7017 - accuracy: 0.8275 - val_loss: 0.7152 - val_accuracy: 0.8234\n","Epoch 87/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.6929 - accuracy: 0.8316 - val_loss: 0.7069 - val_accuracy: 0.8229\n","Epoch 88/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.6835 - accuracy: 0.8341 - val_loss: 0.6979 - val_accuracy: 0.8276\n","Epoch 89/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.6726 - accuracy: 0.8356 - val_loss: 0.6890 - val_accuracy: 0.8304\n","Epoch 90/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.6645 - accuracy: 0.8394 - val_loss: 0.6823 - val_accuracy: 0.8307\n","Epoch 91/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.6549 - accuracy: 0.8445 - val_loss: 0.6707 - val_accuracy: 0.8353\n","Epoch 92/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.6459 - accuracy: 0.8457 - val_loss: 0.6651 - val_accuracy: 0.8388\n","Epoch 93/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.6375 - accuracy: 0.8502 - val_loss: 0.6558 - val_accuracy: 0.8389\n","Epoch 94/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.6291 - accuracy: 0.8533 - val_loss: 0.6479 - val_accuracy: 0.8426\n","Epoch 95/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.6208 - accuracy: 0.8541 - val_loss: 0.6406 - val_accuracy: 0.8453\n","Epoch 96/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.6119 - accuracy: 0.8574 - val_loss: 0.6317 - val_accuracy: 0.8479\n","Epoch 97/100\n","1313/1313 [==============================] - 3s 2ms/step - loss: 0.6037 - accuracy: 0.8609 - val_loss: 0.6270 - val_accuracy: 0.8495\n","Epoch 98/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.5967 - accuracy: 0.8618 - val_loss: 0.6185 - val_accuracy: 0.8507\n","Epoch 99/100\n","1313/1313 [==============================] - 3s 2ms/step - loss: 0.5882 - accuracy: 0.8646 - val_loss: 0.6135 - val_accuracy: 0.8517\n","Epoch 100/100\n","1313/1313 [==============================] - 3s 2ms/step - loss: 0.5817 - accuracy: 0.8655 - val_loss: 0.6051 - val_accuracy: 0.8533\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"k-ttTMOwm81C","executionInfo":{"status":"ok","timestamp":1604684947982,"user_tz":420,"elapsed":315935,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"2d83304a-8ada-4d26-9c66-e830a4a27f47","colab":{"base_uri":"https://localhost:8080/","height":265}},"source":["plt.plot(history.history['accuracy'])\n","plt.plot(history.history['val_accuracy'])\n","plt.plot(history.history['loss'])\n","plt.plot(history.history['val_loss'])\n","plt.legend(['train acc', 'valid acc', 'train loss', 'valid loss'], loc = 'upper left')\n","plt.show()"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUVf7H8feZnkkhHVJJgpQACaHHRRHFgqKgIuKKjVVcUVHUVVh3VXTXVVzbsqIuKhZWsYAiKIqiFF2aEClBek0IpEFC6mTK+f0xkR9igAhJJpl8X88zD5m5d+793lz9zMmZc89VWmuEEEK0fAZfFyCEEKJhSKALIYSfkEAXQgg/IYEuhBB+QgJdCCH8hMlXO46MjNRJSUm+2r0QQrRIa9euLdJaR9W1zGeBnpSUxJo1a3y1eyGEaJGUUntPtEy6XIQQwk9IoAshhJ+QQBdCCD/hsz70ujidTnJzc6murvZ1KS2azWYjPj4es9ns61KEEE2oWQV6bm4uwcHBJCUloZTydTktktaa4uJicnNzSU5O9nU5Qogm1Ky6XKqrq4mIiJAwPwNKKSIiIuSvHCFaoWYV6ICEeQOQ36EQrVOzC/RTqXHXcKDiAB7t8XUpQgjRrLS4QHe4HRyqOkSJo6TBt11SUsLLL798Wu+97LLLKClp+JqEEKK+WlygB5mDsJvtFFYW4va4G3TbJwt0l8t10vcuWLCA0NDQBq1HCCF+ixYX6Eop2trb4vK4OFR9qEG3PWnSJHbu3ElGRgYPPvggS5Ys4dxzz2XYsGF07doVgCuvvJLevXvTrVs3pk+ffvS9SUlJFBUVsWfPHlJTUxk7dizdunXj4osvpqqq6lf7mj9/Pv3796dnz55ceOGF5OfnA1BeXs6YMWNIS0sjPT2dOXPmAPDll1/Sq1cvevToweDBgxv0uIUQ/qFZDVs81uPzN/FT3pETLne4Hbj1QQJMASjq9yVg19gQHrui2wmXP/3002RnZ7Nu3ToAlixZQlZWFtnZ2UeHAM6YMYPw8HCqqqro27cvI0aMICIi4hfb2b59O7NmzeK1117j2muvZc6cOdxwww2/WOecc85h5cqVKKV4/fXXeeaZZ3juuef429/+Rps2bdi4cSMAhw8fprCwkLFjx7Js2TKSk5M5dKhhP8iEEP6h2Qb6CWkNHg9mZcLtceH0OLEYLI22u379+v1iPPfUqVP55JNPAMjJyWH79u2/CvTk5GQyMjIA6N27N3v27PnVdnNzcxk1ahQHDhygpqbm6D4WLVrE+++/f3S9sLAw5s+fz8CBA4+uEx4e3qDHKITwD8020E/UknYdPoxz/34APCYDlSaN0WJBmUwYTGYMJjNGsxmDyYLJZMFoMp/RML7AwMCjPy9ZsoRFixaxYsUK7HY7gwYNqnO8t9VqPfqz0Wiss8tl/Pjx3H///QwbNowlS5YwefLk065RCCGgGQf6iRjbtEFZrOjKSlyV5dgrK1EOB0o7frGeBpxADeAxgMeg0EaFNhrAaDz6MJhMqNoPAYvZQFlZGVrrOj8ESktLCQsLw263s2XLFlauXHnax1FaWkpcXBwAb7/99tHXL7roIqZNm8aLL74IeLtcMjMzufPOO9m9e/fRLhdppQshjtfiAl0ZDBgD7RBox0Qk4L3cXXs8uJwO3M4aPK4aPC4X2uVCu13gdoPLg/J4MNS4MXhcGOoYxh4GZHbrRrdOnbhw4DlcfP55OCvLObx3O8poJDOjKy9VVtC5cyc6depE//79Tns8/OTJkxk5ciRhYWFccMEF7N69G4C//vWv3HXXXXTv3h2j0chjjz3G1VdfzfTp07n66qvxeDxER0fz9ddfn+6vUAjhp5TW2ic77tOnjz7+BhebN28mNTW1SfavtcbjcuJ21Xg/BJxOPC6n90PA5f0QUG4Pyu3B4NF1fgAAaAVuA3iMBjxGA5iMYDKizGYMZgtGsxWT1YbZZMGgmm5QUVP+LoUQTUcptVZr3aeuZS2uhd5QlFIYzRaMZgsEnHp97fHgcTn//6+A2g8Aaj8AlMuDyenBUO3CcNxnpAeoUuAyKjwmA9pkBLMJg8WCwWrFbLVjNlsxGVrt6RBCNABJkHpSBgNGixWjxXrKdbXHg7vGgavGgbumGo/TiXbWoJwuTC43hpoaDJ4aoPLoexwKyk0Kj9mANptRVgtGawCWADsWkw2jwdiIRyeE8AcS6I1AGQyYbAGYbCdu+mu3G7ejGqejCrejGl1Tg7HGicnhwlhZDVQD3nH4FUZwmg1oiwlls2IMsGO1BWI12Zq0G0cI0bxJoPuIMhox2QMx2QN/tUx7PHgcDpzVFbirqlCOaqwOJ8bqGjhSA5ShFZSawGU1gs2KyR6ENSAIm4S8EK2WBHozpAwGjAEBGAMCvENvammPB091Nc6qCjxVlZirqrFWOFHllUAlHlXAYQu4bWbKKw7zzc6F9InPpI21jc+ORQjRdCTQWxBlMGC02zHa7Udf01qjHQ6cFWV4KsqxVVVjKHViLa0ifMwEPk9Q5KdGY8vsR7fMofRp1xe72X6SvQghWir52/wMBQUFAZCXl8c111xT5zqDBg3i+CGaDUUphcFmwxoRRWBiMoGdU7F16YIxLAzT5RfR2RHGkM/zGfTIfExX3sE7o/vyzFPDmLnqP+wu3Y2vhq0KIRqetNAbSGxsLLNnz/Z1GQDeaRACAuj21FQAnPkFlH6/lJpvPufs1euwbNiOa+aLrGz/L95NiyB8yKUM6nElqeGpcrcjIVowaaEfY9KkSUybNu3o88mTJ/Pss89SXl7O4MGD6dWrF2lpaXz66ae/eu+ePXvo3r07AFVVVVx33XWkpqZy1VVX1TmXC8ATTzxB37596d69O7fffvvR1vKOHTu48MIL6dGjB7169WLnzp0ATJkyhbS0NHr06MGkSZPqfVzmttFEjhhJj5ffIn3VWtq/9y6Bo0eR6ghjxLwizrtrJhtHX8PjfxnIq8tfIOdITr23LYRoPppvC/2LSXBwY8Nus10aXPr0CRePGjWKCRMmcNdddwHw4YcfsnDhQmw2G5988gkhISEUFRWRmZnJsGHDTtiafeWVV7Db7WzevJkNGzbQq1evOte7++67efTRRwG48cYb+eyzz7jiiisYPXo0kyZN4qqrrqK6uhqPx8MXX3zBp59+yqpVq7Db7ac9ha4yGrH36kVKr17ohx/DsW07hfM/ocvn80j/uIjq+dOZn/oauYO6kHnxzVySPASr8dRj74UQvict9GP07NmTgoIC8vLyWL9+PWFhYSQkJKC15uGHHyY9PZ0LL7yQ/fv3H70hRV2WLVt2dP7z9PR00tPT61xv8eLF9O/fn7S0NL799ls2bdpEWVkZ+/fv56qrrgLAZrNht9tZtGgRY8aMwV77hWhDTM6llMLWuRMJf5pI+rffk/TB+4RcPpSB20zcOHUzplsn8ejD5/DC8insL99/xvsTQjSu5ttCP0lLujGNHDmS2bNnc/DgQUaNGgXAu+++S2FhIWvXrsVsNpOUlFTntLm/RXV1NXfeeSdr1qwhISGByZMnn/E2z4RSioAePejQoweev1ZQ+tnnuN96jVs+zaX067d4o/dMaq4czHX9x9I9srvP6hRCnJi00I8zatQo3n//fWbPns3IkSMB71S30dHRmM1mFi9ezN69e0+6jYEDB/Lee+8BkJ2dzYYNG361zs/hHRkZSXl5+dEvVIODg4mPj2fu3LkAOBwOKisrueiii3jzzTeprPROF9CYdy0yBAYSNupaui34isS33iKyz+8Y+Z2bayZ9xcIJ1zLhgxv54eAPMkJGiGZGAv043bp1o6ysjLi4OGJiYgAYPXo0a9asIS0tjXfeeYcuXbqcdBvjxo2jvLyc1NRUHn30UXr37v2rdUJDQxk7dizdu3fnkksuoW/fvkeXzZw5k6lTp5Kens7vfvc7Dh48yJAhQxg2bBh9+vQhIyODZ599tmEPvA5KKQIz+3PW9DdImT+P8EuHMuRHxR+eWMP399/M+PevZ/WB1Y1ehxCiflrt9Ln+rrF+l879+8l/9RWOfPwJLjws7KXIuSaTcec8SGqEnDshGtvJps+VFrr4TcxxccT/7e+ctfArwodfydA1mhsnr+TNx67h4cUTKaws9HWJQrRaEujitFji44j/x1OkfPIJEel9GLPIw/mPzOeBF4fwZvabON1OX5coRKsjgS7OiK1LF5Lfepv4l18mwRzNn9+ppOaxf3LLe1eyrmCdr8sTolWRQBdnTClF8AXn02nBF0SMu4Nztxq47/k9vPrcDfxj5ZNUOCt8XaIQrYIEumgwhoAAou+9lw7z5hHWsSv3fuom5ql3ufm9K8nKz/J1eUL4vVMGulIqQSm1WCn1k1Jqk1Lq3jrWUUqpqUqpHUqpDUqpuq91F62CNSWFlPdmEf2nB+i/y8hDU/N44ZWbmZo1FadH+taFaCz1aaG7gAe01l2BTOAupVTX49a5FOhY+7gdeKVBq2wiJSUlvPzyy6f13ssuu4ySkpJ6r//zxF/+SplMRNx2Gylz5hAek8zD77uofvFVxsy7QaYREKKRnDLQtdYHtNZZtT+XAZuBuONWGw68o71WAqFKqZgGr7aRnSzQXS7XSd+7YMECQkNDG6OsFs3WqRMps2cTdv31XLFaM/pfm7hj5giW5iz1dWlC+J3f1IeulEoCegKrjlsUBxw752ouvw59lFK3K6XWKKXWFBY2v/HKkyZNYufOnWRkZPDggw+yZMkSzj33XIYNG0bXrt4/Sq688kp69+5Nt27dmD59+tH3JiUlUVRUxJ49e0hNTWXs2LF069aNiy+++ITT5/5s3bp1ZGZmkp6ezlVXXcXhw4cBmDp1Kl27diU9PZ3rrrsOgKVLl5KRkUFGRgY9e/akrKyskX4bDcdgs9Hu0UeIn/YSSWU2HnutnP+8cSdTs6bi9rh9XZ4QfqPek3MppYKAOcAErfWR09mZ1no6MB28V4qebN0pq6ew5dCW09nNCXUJ78LEfhNPuPzpp58mOzubdeu8w+2WLFlCVlYW2dnZJCcnAzBjxgzCw8Opqqqib9++jBgxgoiIiF9sZ/v27cyaNYvXXnuNa6+9ljlz5hydfbEuN910E//+978577zzePTRR3n88cd58cUXefrpp9m9ezdWq/Vod86zzz7LtGnTGDBgAOXl5dhstjP9tTSZ4MGD6TB7Nvvuvpu/fLCL9/L/w70jtvL0wCkEWYJ8XZ4QLV69WuhKKTPeMH9Xa/1xHavsBxKOeR5f+1qL169fv6NhDt5Wc48ePcjMzCQnJ4ft27f/6j3JyclkZGQA0Lt3b/bs2XPC7ZeWllJSUsJ5550HwM0338yyZcsA79S7o0eP5r///S8mk/ezd8CAAdx///1MnTqVkpKSo6+3FJakJFI++IA2l1zCDYs9dJu+hJvnjZabagjRAE6ZBsp7F4c3gM1a6+dPsNo84G6l1PtAf6BUa33gTAo7WUu6KQUGBh79ecmSJSxatIgVK1Zgt9sZNGhQnVPeWq3/f0MIo9F4yi6XE/n8889ZtmwZ8+fP58knn2Tjxo1MmjSJoUOHsmDBAgYMGMDChQtPOVlYc2MIDCTuhecpSklh0Msv0+7ILm4tG8UzQ18mIzrD1+UJ0WLVp4U+ALgRuEApta72cZlS6g6l1B216ywAdgE7gNeAOxun3MYVHBx80j7p0tJSwsLCsNvtbNmyhZUrV57xPtu0aUNYWBjfffcd4J1p8bzzzsPj8ZCTk8P555/PlClTKC0tpby8nJ07d5KWlsbEiRPp27cvW7Y0bLdUU1FKEXXPeGKfmUKX/Yq/zChn4od/4Jt93/i6NCFarFO20LXW3wMnvXOw9k7ZeFdDFeUrERERDBgwgO7du3PppZcydOjQXywfMmQIr776KqmpqXTu3JnMzMwG2e/bb7/NHXfcQWVlJSkpKbz55pu43W5uuOEGSktL0Vpzzz33EBoayiOPPMLixYsxGAx069aNSy+9tEFq8JU2w4ZhjolB3XkXT7xTzRPVEyi8/C9c1+U6X5cmRIsj0+f6qZb2u6zeuo19Y2+jouwwT47QXHTFeG5Pv/2E920VorWS6XNFs2fr3Imk92YR3Daexz7QLPt4Ks+vfV7uiiTEbyCBLpoNS3wcSe+9S2CHjvx5Dqz/dAZPrHwCj/b4ujQhWgQJdNGsmMLDaf/WWwR2TmXix7B3/odMXj5ZQl2IepBAF82OMTSUxLfexN49jQfmanI/n8MTK6SlLsSpSKCLZskYHEziG28QmN6DBz6F3Qs+klAX4hQk0EWzZQwKJOG16dhTu/LQXNix8COmrJ4iX5QKcQIS6GcoKMg7B0leXh7XXHNNnesMGjSI44donux18f+MwcEkvv4aAR07MeljxY9fvcvUH6f6uiwhmiUJ9AYSGxvL7NmzfV2GXzKGhpI4YwYBie356xzFN1+9xusbX/d1WUI0OxLox5g0aRLTpk07+vznm1CUl5czePBgevXqRVpaGp9++umv3rtnzx66d+8OQFVVFddddx2pqalcddVV9ZrLZdasWaSlpdG9e3cmTvTOY+N2u7nlllvo3r07aWlpvPDCC0Dd0+r6O1NYGIlvvEFAeDSPzzby0dcv8sGWD3xdlhDNSrOdqu/gP/6BY3PDzlNiTe1Cu4cfPuHyUaNGMWHCBO66yzuLwYcffsjChQux2Wx88sknhISEUFRURGZmJsOGDTvhVYyvvPIKdrudzZs3s2HDBnr1Ovkd+fLy8pg4cSJr164lLCyMiy++mLlz55KQkMD+/fvJzs4GODqFbl3T6rYG5nbtaD/jDfaMHs3fPqxgounvhNnCuDjpYl+XJkSzIC30Y/Ts2ZOCggLy8vJYv349YWFhJCQkoLXm4YcfJj09nQsvvJD9+/eTn59/wu0sW7bs6Pzn6enppKenn3S/P/zwA4MGDSIqKgqTycTo0aNZtmwZKSkp7Nq1i/Hjx/Pll18SEhJydJvHT6vbWliSkkh84w1C3GaemGPib19NZPWB1b4uS4hmodmmwcla0o1p5MiRzJ49m4MHDzJq1CgA3n33XQoLC1m7di1ms5mkpKQ6p81taGFhYaxfv56FCxfy6quv8uGHHzJjxow6p9VtTcFu69KFhGnT0Lfexl9mG/mTdTzTr3ibLuEtaxphIRqatNCPM2rUKN5//31mz57NyJEjAe+0udHR0ZjNZhYvXszevXtPuo2BAwfy3nvvAZCdnc2GDRtOun6/fv1YunQpRUVFuN1uZs2axXnnnUdRUREej4cRI0bw97//naysrBNOq9vaBPbrR9w//0n7HAd3f1zD+IV3crDioK/LEsKnWk+zrp66detGWVkZcXFxxMR473M9evRorrjiCtLS0ujTp88pbygxbtw4xowZQ2pqKqmpqfTu3fuk68fExPD0009z/vnno7Vm6NChDB8+nPXr1zNmzBg8Hu/FNE899dQJp9VtjUKGXILr0F/hib9x1WfFjAu4g3cum0mwJdjXpQnhEzJ9rp9qTb/Lgueeo/i11/nvYBMFwzN5efDLmI1mX5clRKOQ6XOFX4u67z6Chwxh9LduPEuW88TKJ+RqUtEqSaCLFk8ZDMQ+/RT29HTu+8zAxmWfMCN7hq/LEqLJNbtAl5bVmWuNv0ODzUb8y9OwRbfjkU9MvLPkBRbtXeTrsoRoUs0q0G02G8XFxa0ykBqK1pri4mJsNpuvS2lypogIEl55mUC3icfmWnjsm0lsKtrk67KEaDLNapRLfHw8ubm5FBYW+rqUFs1msxEfH+/rMnzC2rEj8S88j75jHOM/s3KvfTzvXjGLtoFtfV2aEI2uWQW62WwmOTnZ12WIFi5o4EDaTnyIHk89zcVfu7k36F7eHPImAaYAX5cmRKNqVl0uQjSUsJtuInTkNVzxvxrafJfNo/97VLryhN+TQBd+SSlFu0ceIaB3b8YvUGxd8QX/2fAfX5clRKOSQBd+S1ksxE/9F9bIKB6da+a/37/EN/u+8XVZQjQaCXTh10wRESS8PI0gBzw2z8qji//M9sPbfV2WEI1CAl34PVtqKrFPPUX83kr+sNDF+G/upqS69cwjL1oPCXTRKoQMGULEuDv4XVYVGcsO8Kelf8Llcfm6LCEalAS6aDWixo8n6PzzuWmRm/KVK3h+7fO+LkmIBiWBLloNZTAQ+89nsCYnM2m+mS9WvMP8nfN9XZYQDUYCXbQqxqAg4l/6NwGYeWyejSeXPibTAwi/IYEuWh1rcjKx/3yGtrkV3LlQce+391BUVeTrsoQ4YxLoolUKPv98IsffTd/1lWR+V8QDSx7A6Xb6uiwhzogEumi1IseNI2jwYK7/xoVj9Rqm/DDF1yUJcUYk0EWrpQwGYqc8jTU5mT/PN/PtqveZs22Or8sS4rSdMtCVUjOUUgVKqewTLB+klCpVSq2rfTza8GUK0TiMQUEkTHsJm8HK5E9t/PO7v7GuYJ2vyxLitNSnhf4WMOQU63yntc6ofTxx5mUJ0XQsSUnEPfcsUQeruPcLExO+vZeDFQd9XZYQv9kpA11rvQw41AS1COEzQeeeS/T995GRXcH5y0qZsHgC1a5qX5clxG/SUH3oZyul1iulvlBKdTvRSkqp25VSa5RSa+SuRKK5Cb/1VkIuu5SRi2swrd7IEyuekDnURYvSEIGeBbTXWvcA/g3MPdGKWuvpWus+Wus+UVFRDbBrIRqOUoqYv/8dW6dOPPS5mdVr5jHzp5m+LkuIejvjQNdaH9Fal9f+vAAwK6Uiz7gyIXzAYLcTP+0lrCYbj88LYNryZ1met9zXZQlRL2cc6EqpdkopVftzv9ptFp/pdoXwFUt8PPEvvkB4QRUPfWHloSV/IudIjq/LEuKU6jNscRawAuislMpVSt2qlLpDKXVH7SrXANlKqfXAVOA6LR2PooULPPts2k6aRNefyrlycTX3LL6H8ppyX5clxEmZTrWC1vr3p1j+EvBSg1UkRDMRdsNoHNu2MfSjj9gVvoOJQROZev5UjAajr0sTok5ypagQJ+C90fRfCejTm7u+gP2rl/Ji1ou+LkuIE5JAF+IkvDeanoo1qi2PzjUzb8WbzN1xwoFcQviUBLoQp2AKDyfh1Vewu4w8/qmNp5dOZs3BNb4uS4hfkUAXoh6sHTsS/8LztM2r4v4FJu779l72Hdnn67KE+AUJdCHqKWjgQNpOmkjapgpGLKrirm/uotRR6uuyhDhKAl2I3yDsxhsJu/56LlleRZele+TGGKJZkUAX4jdQStH24T8TNGgQY75yU/P9Cp5YKXO+iOZBAl2I30iZTMQ99yy2Ll14cJ6R9d9/wvQN031dlhAS6EKcDkNgIAmvvIotPJLJc0x8sOTffLbrM1+XJVo5CXQhTpO5bTSJr79GIFb+NsfMM18/wqoDq3xdlmjFJNCFOAPWDh1IePUVwks1D89R/OnL8Wwq3uTrskQrJYEuxBmy9+pF3LP/pH2ug/tnO7nni3HsPbLX12WJVkgCXYgGEHLRRcQ8+SRddlYz9sNSxn15O/kV+b4uS7QyEuhCNJDQq66k7aOP0GNbDde+n8ftC2/jULXcjlc0HQl0IRpQ+PXXE/3gg/T/ycVl7+/hj1+OlatJRZORQBeigUXc+gcix9/NwA0uzv1wK3d+PY4KZ4WvyxKtgAS6EI0g8s47ibjtVi7KcpPx0Qbu+OqPcscj0egk0IVoBEopoh54gLDRo7l8lZvus9dxx9cS6qJxSaAL0UiUUrT9y8OEXjeKYSvcZHywXkJdNCoJdCEakTIYaPfYY4TdcANDV7vpPWs9ty28lZLqEl+XJvyQBLoQjeznlnr4LbcwZI2b8/67iVsX3EJhZaGvSxN+RgJdiCaglCJ64kNE3jmOQevdDH9nJ7fNv4m88jxflyb8iAS6EE1EKUXUPffQ9uE/03eLi5vfyuW2T0az/fB2X5cm/IQEuhBNLPymm4id8jSp+zzc90YREz68kaz8LF+XJfyABLoQPtBm+HASXv0P8WUWHnmjnCffuZVv9n3j67JECyeBLoSPBJ17DsnvzSIsMJJHZ9bw3+n38vamt+V2duK0SaAL4UO2zp3o8OFHBHVK5cE5brZOe4bHl0/G6ZEbT4vfTgJdCB8zRUWRPHMmIZdcwo3feoic+hHjvhgrMzWK30wCXYhmwBAQQNzzzxN55zgGr9dc/sIP/PHdEXL3I/GbSKAL0Uwog4Goe+4h7vnn6FRs4aFpBfxz2g3M3THX16WJFkICXYhmJuSyy+gwZw6hsUk8NKua7H88zF8WT6LSWenr0kQzJ4EuRDNkTUmhw0ezaXPllYxYrun/t3mMe/sqNhdv9nVpohmTQBeimTIEBBD31FPEvfgCZx0JYMLUHKb/YxRvZ7+FR3t8XZ5ohiTQhWjmQoYMoeP8zwhJ78ntnzsxTHqGCXNukZtQi1+RQBeiBTDHxJDyzkyiJ02i1z4jNz75A0/9Yyif7/xMLkQSR50y0JVSM5RSBUqp7BMsV0qpqUqpHUqpDUqpXg1fphBCGQxE3HIzZ839lJCUzvxxTgUl4x/isdl3UFxV7OvyRDNQnxb6W8CQkyy/FOhY+7gdeOXMyxJCnIg1JYVOH8wm6s8T6ZFn4urHl/HKgxczd8scaa23cqcMdK31MuBkl6wNB97RXiuBUKVUTEMVKIT4NWUyEXnzLXT+YiG232UyYlElAWP/yl9fuYadJTt9XZ7wkYboQ48Dco55nlv72q8opW5XSq1RSq0pLJS7tQhxpswxMXSZ/iZx0/5NjG7DDVN/YuEfh/Pvb/9OhbPC1+WJJtakX4pqradrrftorftERUU15a6F8Gshgy+k+8JvsN/0e87L9pB5/7s8N/ECFmyfL90wrUhDBPp+IOGY5/G1rwkhmpAhMJD2Dz/KWZ/Ow949jZELjmAc8xBPPDecTYV1jmkQfqYhAn0ecFPtaJdMoFRrfaABtiuEOA3Ws86i68wPiJ36L9qZI7j29e1svf5aXnx7HAcrDvq6PNGI1Kn+HFNKzQIGAZFAPvAYYAbQWr+qlFLAS3hHwlQCY7TWa0614z59+ug1a065mhDiDGink/z33+XgS1OxllaxPsVAxY1DGXHVw4TaQn1dnnIP/8oAABoOSURBVDgNSqm1Wus+dS7zVf+aBLoQTcdTUcGeN1+h9K2Z2MpryE4x4Rh9OZePmCjB3sJIoAshAG+wb39zGuVvv4u9rIbt8UbKRl3EpaP/Srg9wtfliXqQQBdC/IKnupptM1/hyFszCS6uIi/CQMnlZzPo9slERMT7ujxxEicLdJnLRYhWyGCz0WXsffRdsgrT5D9hDg6h69v/Y88FF/HZQ7+n8MAuX5coToMEuhCtmDKb6XjdrQz8cgWm158jv3sMyfPWkXPJUD59cBR5e+QWeC2JdLkIIX5h549L2Pb8kyT+kItWkJfWloRRN9Nl2A0os9nX5bU4Ho/mSLWTaqeHGpeHGreHNgFmooKtp7U96UMXQvxmOZtW8eObzxG5NJuwMk1ZGzMMv4Qet/0Ja3RbX5fXpBwuN6VVTmpcHpxuTVWNm+IKB4Vl3kdBmYP8I9UUlTuoqg1uh8tNaaWTw5U1eDSAxoqTIKq48ZyOTLi872nVIoEuhDhtRyoP8+2Hz+Ge8zldt1fjMkJJZiodf38bbQddjDKZfF3iadFaU1jmYHdRBYXlDqqdHqqd3uDOOVRJzuFK8kqqKSqvxlBdSrQqIZwywlQZ4aqMUMoIU+WEUk6I0UGoyUmwoYYA5cCia7DqGsw4MePEpJ2Y3NUY8N5pqijjLiKv/Mdp1S2BLoQ4Yy6Pi2X/m8X+d16n8w8FBFdDZYgVw2UX0GXMPQS0T/J1iVQ4XBworaKovIaSyhpKKp2UO1xUOVy4qo9QUV5G4ZEqisurqSgpJMp1gARVQJwqJlqVEKVKaEMFZgOYjQq7wUkbVzFmXVPn/jxmO9hCMdhCwBIIZjuYA8Bkq31YvQ+jxbvcEgiWIIjtCfF1ZvIpSaALIRrUtoKfWD7nJSxffk/6NicKKEpPIPa6G+lw6UgMNluD7k9rTbnDRVF5DQWllZTn78Kdv4XK0kIqy0qoriyjrLqGihqNBwOR6ggJKp9EVUC0KiGMMizKfcLte0x23EHt0EHtMNrDMBoNgPKGcXA7CI6F4LZgjwR7BNjDISAczA17nPUhgS6EaBTVrmq+WzeXvPfeoeOy3YSVQ43FgCuzB8nDf0+bgYMwBgfX/WaPGyoP4TQFUFRtoKCshuLDJZQfysNRvBdbwQYiSrOJcezC7dE4tLdrp73KJ0hVn7Quj8GCMzgBHZqIoU0sxqBIjIGR3hayMngf1mAIS/I+AsJAqYb95TQSCXQhRKMrLDvIkvnTKP3yS7pllxNaCR6jwpWaQkDPdKo6tcPhKMZYuofwI1uJdezCigMAlzZQgxm7cvxim/mGtuTbO2I2W7AbXVgM4AlLwti2G/b4NIIjY1GWILDYvSHtcXk/KKzBYDD64tfQ6E4W6C3z2wwhhO85yqjJ20hxUSFFlS6KKtwEesIJ692DrWkbyTlyBEsOZOzYSeLMnZgMGnu8g+AkNwfj4tkaPJSaoATCLG7CTA6CjS4sIVHYwtoRGBmPJa4nbYOiaV3jac6MBLoQ4tecVbD3f6A1TnMwB6tNHM7biefABmzFPxF2ZCttXfuxADG1j585sJBrTuZwRDql7duSdbGT+YW7SP4hl3Ozbbj2gTGoii6DI4jsMxh7v34YAgJ8dKD+RbpchBBeFcWUb1lExfq5hOUuxuKpqnO1fTqaPeYOlIR0wRPdnZDIWKKDTEQHGgmLbIc5ujMYf91WPFhxkM+3zmXHl7NJWptHnx0auwM8JiO23j0JHTiI4MGDsSQlNfKBtmzShy6E8NIaCjbj2LuagoJ8ig4fprKkkNjSLJJd3vlbCnUIX3n6kh18LuERUaQEuYgPcBLSNpGw5J5ERUZhNJz+F4haa7Yd3sYXW+exa8k84jcX02M3JBZ6s8h0VgdCL76E4EsuwdqpI6qFfFnZVCTQhWiNXA44tIsjuZvI27ER14GNJJauJcRT8ovVqrCw09KFvLB+VMWfQ1SX35GeGEGQtfF7ZLXW/FT8E1/t/YrV6xaQ8GMe/bdpuuRoDBqM7RMJvWQIgQMGEJDRA4P19C6X9ycS6EL4O1cN5GWhdy2les9q3IVbsVfsP3plIkCejmSLNY0D4f1wxWeSGJ9Il/ZtaRca1CxawT+H+xe7v2D5pgUk/ZjP2Vs0Xfd5wx2LGXtGTwLPOYegQedh7dg6W+8S6EL4C48HCrfAnu9xHtzEkYIcXCX7aVO5B5uuxqMV23Uc23U8OYY4iDiLiKQ0OnfLoFtSHCZjy5hg9edwX7RvEcu3fk3Apt103afpmWMi/oATAFNMDIEDfkdg/0wCM/tjiorycdVNQwJdiJbs8F48OxdTuflrzDn/w1pzGIBDOoiDOoJ8HcphWzx5oX2oiMkkJiaWXolhdGkX3GIC/FTyyvP4fv/3LM1dyuaty0nbXkPmHjNd92osld7L8i1JSQT06Y29V2/s/fpiiffPG3VIoAvRElSXwqa5kD0b16G9OKsr0c4q7J5yAA7ocJZ7upGlulMRezaJHVLp0z6MjMRQQmytZ1rb8ppyluYu5eu9X7M853ti8qrpm2ulb34wsbuPYCr3js4xx8Ziz8wksH8/7P37Y27XzseVNwwJdCGao/ICOLCeqn1rKd2+isj87zHpGvYSS5Y7mWptAZMNT1gS5fEDCU/sTmpsG79qeZ+pKlcV/9v/P77d9y2rDqyisDKf+CLofyCIzLxA4rYfxvhzwLdPxN6zF7auqdhSU7F17YohMNDHR/DbSaAL4WtaQ14WbP0ST14W7v3rMVcVHl28y9OO73UGa0MvxhjXi/SEUM7uEEnH6CAMZzBEsDXRWrOvbB+rD65m1YFVrDqwitKqwyQXKi4qjqFXroWw3cVwyDvKR1mtBJ57DiGXDCFo4LkY27Tx8RHUjwS6EE1NayjNgdwfqN75PWz9AlvlAdwY2KHj2ehJ4idPe8rDupHQtR8DuqeQHtdGWt4NyKM9bC7ezNLcpXy771u2Ht4KQHxNEIMqE+mzx0TMD3tQRYcAMEVFYUlJwdalM/bMTOx9+2EMan4teAl0IRqLowzKDkLZAXTJPipyNlKTl43t0GbsNcUAVGkL33nSWOjpy96IgaSmtCczJYJ+yeGnfRsy8dvtL9/PDwd/YF3BOn4s+JFdpbtQWtO7IJjzD7fjrCMBRORXobfvRjscYDIRkJZGQM+eBPTMwJ6R0SxG0kigC9GQnFXw06eQ9Y53vpNjOLSZbTqOrTqRvbYu1LTrRXhyT9LbR5Ee34bAJrhYR9RPYWUhKw+sZHnecm//e20XWJw5mkvK29Nrn4m2Wwph6060s3aoZLt2BKR1x5aWjr1vHwK6d2/y+6xKoAtxJkpzYdcSKNyCO38zet8qTM4yiixxfKbPYV1FBEfMkXQ8qzMJHVLp2C6UTm2DCQu0+LpyUU9aa/Yc2cPqA6tZfXA1WQVZFFUVARBpbMNF1R3oXRRMfG411m05OPftA8BgtxPQsyeWlBQsCQlY2idiS0/HFBbWaLVKoAtRH24XOCvAWQ2uKjy5a3GtnYl5zxIUGqcys8MTy0Z3Eh97zmVnQA+6xYcyrEcsQ7q3w26R1re/0FqTU5bD2vy1rM1fS1ZBFjllOQAEmALob0tlUFEUnXY7sG/OwZmTg66sPPp+y1kdsPfug7VjRyztE7EkJmJOSEAZzvw7Egl0IY5XmgtbFsDWz6Foh3cMeE3Zr1bL1ZHMcQ9kvjsTR0gKg7vFckGXaLrFhhARJP3frUlBZQFZBVlk5WexNn8t2w5vA8BmtJEW2Z3+1i5kVEcTt7sMz7psqn5ch6fs//+bMrZpQ0Dv3tj79CHo3HOwdux4WnVIoAvhcsC+lbBrMez4Bg5uAKA0KIVtxk7sKjeTV22hHBuYAmgbEUpw2xScCWcTHRJAUmQgndsGt8q5Q0TdSqpLWJu/lh/yf+DHgh/ZcmgLHu2dOycuKI6u4an0tXSihyOadsVuqtdvoOqHNdTs3UvE2LFEP3D/ae1XAl20DpWHoGAzHNkPZQdxHcmj+uAOVPF2AipyMGg3LoxsMnZhoSOdL9292aVjCbObyUyJODrypFPb4DOaHla0ThXOCjYWbeSn4p/YVLSJTcWb2F++H4AgcxBdwrvQJbwLqbod6VFpJKX0Oq39yC3ohH/Q2hvW+9fCwY3eAK8uRVcWowu2YCg/8IvVa7SVXB3NTh3DLp3BDnMn8sL6EBEWSUpUIH+Ka0O32BASwuxy8Y44Y4HmQDJjMsmMyTz62sGKg0e7aLYc2sKc7XOoclXxB9MfuI/TC/STkRa6aN48Hti3ArJne/u8yw96X1ZGqozBlGo7ha4Adnhi2OpJYKtOZD+RRLRLIj0ljm5xoSRHBpIUGUibgNYz34lontweNzllOViNVmKCYk79hjpIC100P85qqCyGgDDvHdt/bn0XbcNduI1DOVupPLiD0NKfaOMqwqFsrA/oxzfqMlZWJ7FZt8diDaBrbAjdYkOID7OTFmzlwhAbqTHBBLeiyapEy2E0GElqk9Ro25dAF01Da9jzHWz9AnJWw4H14PFerOFUFtxaYcMBgBEI1FaKdFu2mzqx3Hozq8z98JgDSUsJYXT7cHq1DyMlMlC6SoQ4Rr0CXSk1BPgX3v/XXtdaP33c8luAfwL7a196SWv9egPWKVqS0lyoLCav0sjCbUdok7+CQcUfEl62BbfRyn57Kqstw8gqC6UNFbQzV9AuxMLhgEQOByThCEmhY4ez6JscTmqIjQt9fTxCtBCnDHSllBGYBlwE5AI/KKXmaa1/Om7VD7TWdzdCjaKZqnC4MChFgEnBvhV4tn5JzZYvsdWOz40FxtSuu90TxxT3WOZWD0BX2+iZGMrv+kRyXuco0uLayKgSIRpAfVro/YAdWutdAEqp94HhwPGBLvxRyT5YMgUO7YKKAnRFMSXWWLI8Z7HwcAyd2csVppVEcxg3Rta4u7DYMxp3SCID2tvpE2shNLYjQZGZDCuqYjjQMzGMAIvR10cmhN+pT6DHATnHPM8F+tex3gil1EBgG3Cf1jqnjnVES6E1rH0TvnoEtKYyKp0drkR+qkwmoSKPTONCBpuqcSszW0My+cRyHnnRA+l5Vjx/SA4nLjTgF5uLAWJCm99UpEL4k4b6UnQ+MEtr7VBK/RF4G7jg+JWUUrcDtwMkJiY20K5Fg8tdg170OGrPMnLD+jHFfCfzd1mwmAwM7xFLx34JBMSFwKEdGIPb0TUglK6+rlkIUa9A3w8kHPM8nv//8hMArXXxMU9fB56pa0Na6+nAdPCOQ/9NlYqG56yGfcsBxe4SJ0t+3Er/gg/oWrORIzqQKa5bee/ABXSICuLBS+L5fb9Ewo+dQTC6i89KF0L8Wn0C/Qego1IqGW+QXwdcf+wKSqkYrfXPl+kNAzY3aJXizFSVwJ7vIaIDRHQEV7W3O2X5v6E8H4Dk2keRIZLZkePYFjeC/smxTEiJIDrE5tPyhRD1c8pA11q7lFJ3AwvxDlucobXepJR6AlijtZ4H3KOUGga4gEPALY1Ys/gtqkrgrcshfyMA2milBhNWdwXZ1p78R40h32Hh8q4RXN07gciO53KNSebxFqIlkkv//VlNJfz3ashdA8NfotzhZsGir1BVh/naNoSC0B4khtsZN6gDqTEhvq5WCFEPcul/a+R2wke3eKeMveYNyjsOZ/RrK9lceR3v/KEf01MifF2hEKKByS3G/ZHW8Pn9sH0hXP481Z2v5PZ31pCdd4SXr+9FpoS5EH5JWuj+aM0MyHoHx9n38Xblecz45xIOHqnmhVE9uLBrW19XJ4RoJBLofqb4p2WELXiIrUH9uW55f0odW/hdhwieu7YHA86K9HV5QohGJIHuJ2pcHv4x62vu2jGWvTqC2yvGcX5qO249J4W0+Da+Lk8I0QQk0Fs6jwfH9sVkffpv/lzxPRjNlF49m6Vd+8rUskK0MhLoLVlNBa7/Xot13/ek6kD2JF1D56ETOEuu4BSiVZJAb4EcLjf/+ymX+IVj6FCRxWTXGHpfdQ9X9E7xdWlCCB+SQG9hKhwufv/KMu4rnsxZxg28HzuJyy++nT5J4b4uTQjhYxLoLYGjHGaPQe9fi9OhectVQ7ixHNdlL3J9vzGnfr8QolWQQG/uXA74YDTs/o7syMvYeKSCPgnBhA8Yjqn7CF9XJ4RoRiTQmzO3C2b/AXYtYW3Ppxixoj2j+iTw+xFpoGQEixDil+TS/+bKVQNzx8GWz/i47T2MWNGefknh/O3K7igJcyFEHaSF3hyVF8KHN8K+FfyL3/NS7tmMv6ADd51/FhaTfAYLIeomgd7cHFgPs67HXVHEvTXjKWg/lC+uSuOs6CBfVyaEaOYk0JuT8kJ463Kc5iB+73wMZ1wPPvhDP2xmo68rE0K0ABLoPlbtdPPXudkkhtsZc+QVgmoquM30FHssbZl/Qy8JcyFEvUmg+9jLS3Yye20u8aqQOyxvMdd4Af87HM67t/Uipk2Ar8sTQrQg8g2bD+0sLOfVJTsZnhHLwozvMRhNfBw8mieGd6e/3IRCCPEbSQvdFwq3od8byW5HVxLMw3isfwyBb8+GAfcw86KrfF2dEKKFkkBvas5qmP0HXGWFDHQu5HzjIozz48EaAgMm+Lo6IUQLJl0uTW3RY5C/kT957uHuyNcxpI2Aw3tg4ANglwm2hBCnT1roTWjv8jm0X/Uqb7ouISukHzOu7YtqezUMeRpsclchIcSZkUBvRB6PZvHWApZuK2TX1o38q+JBtqr2qIueYNGAjlhNtUMSA0J9W6gQwi9IoDeSqho3Ez74kYWb8kk372em+R9YLUYsN7/PLfFyRyEhRMOTQG9IbifsWkLFzhVM3WQlqyiBfw2wM2zTkyhzANw4F5vcHk4I0Ugk0E/Xod2weym4anA5HZTlbcW+4zOsNYcJBP4M/NkKZBkhNBFumgthSb6tWQjh11pcoOccqmT5zqJG3YfWUFHjpqCsmsIyB5UON0ajwmxQRLvyuLj4v/Q6vBADbsD7SwzQZr729OZTzwCKIvvx7EALHRyb4EgeDLgXgts1as1CCNHiAj1v7Wf0/G5yg29XHf1XH/3XrNyYlcZQu9CDIkoX4cLEu1zCx4YhtG0bQ8fYcDrGRXJWu3D+HRV4zPwrFzR4nUIIcSItLtAzOiTgKejZSFv//1g3Gg2YzRaUwQjK4G22oyEkFlO/P3JjcFtubKQqhBDidLS4QLemnA0pZ/u6DCGEaHbkSlEhhPATEuhCCOEnJNCFEMJPSKALIYSfqFegK6WGKKW2KqV2KKUm1bHcqpT6oHb5KqVUUkMXKoQQ4uROGehKKSMwDbgU6Ar8XinV9bjVbgUOa63PAl4ApjR0oUIIIU6uPi30fsAOrfUurXUN8D4w/Lh1hgNv1/48GxislFIIIYRoMvUJ9Dgg55jnubWv1bmO1toFlAK/uimmUup2pdQapdSawsLC06tYCCFEnZr0wiKt9XRgOoBSqlAptfc0NxUJNO6ELs1Tazzu1njM0DqPuzUeM/z2425/ogX1CfT9QMIxz+NrX6trnVyllAloAxSfbKNa66h67LtOSqk1Wus+p/v+lqo1HndrPGZoncfdGo8ZGva469Pl8gPQUSmVrJSyANcB845bZx5wc+3P1wDfaq11QxQohBCifk7ZQtdau5RSdwMLASMwQ2u9SSn1BLBGaz0PeAOYqZTaARzCG/pCCCGaUL360LXWC4AFx7326DE/VwMjG7a0k5rehPtqTlrjcbfGY4bWedyt8ZihAY9bSc+IEEL4B7n0Xwgh/IQEuhBC+IkWF+inmlfGHyilEpRSi5VSPymlNiml7q19PVwp9bVSanvtv2G+rrUxKKWMSqkflVKf1T5Prp0jaEftnEEWX9fYkJRSoUqp2UqpLUqpzUqps1vDuVZK3Vf733e2UmqWUsrmj+daKTVDKVWglMo+5rU6z6/ymlp7/BuUUr1+y75aVKDXc14Zf+ACHtBadwUygbtqj3MS8I3WuiPwTe1zf3QvsPmY51OAF2rnCjqMd+4gf/Iv4EutdRegB95j9+tzrZSKA+4B+mitu+MdQXcd/nmu3wKGHPfaic7vpUDH2sftwCu/ZUctKtCp37wyLZ7W+oDWOqv25zK8/4PH8cs5c94GrvRNhY1HKRUPDAVer32u8N5te3btKn513EqpNsBAvEN/0VrXaK1LaAXnGu8ou4DaixHtwAH88FxrrZfhHc59rBOd3+HAO9prJRCqlIqp775aWqDXZ14Zv1I7FXFPYBXQVmt9oHbRQaCtj8pqTC8CDwGe2ucRQEntHEHgf+c8GSgE3qztZnpdKRWIn59rrfV+4FlgH94gLwXW4t/n+lgnOr9nlHEtLdBbFaVUEDAHmKC1PnLsstorcf1qzKlS6nKgQGu91te1NCET0At4RWvdE6jguO4VPz3XYXhbo8lALBDIr7slWoWGPL8tLdDrM6+MX1BKmfGG+bta649rX87/+c+v2n8LfFVfIxkADFNK7cHbnXYB3v7l0No/y8H/znkukKu1XlX7fDbegPf3c30hsFtrXai1dgIf4z3//nyuj3Wi83tGGdfSAr0+88q0eLX9xm8Am7XWzx+z6Ng5c24GPm3q2hqT1vrPWut4rXUS3nP7rdZ6NLAY7xxB4GfHrbU+COQopTrXvjQY+Ak/P9d4u1oylVL22v/efz5uvz3XxznR+Z0H3FQ72iUTKD2ma+bUtNYt6gFcBmwDdgJ/8XU9jXSM5+D9E2wDsK72cRne/uRvgO3AIiDc17U24u9gEPBZ7c8pwGpgB/ARYPV1fQ18rBnAmtrzPRcIaw3nGngc2AJkAzMBqz+ea2AW3u8JnHj/Irv1ROcXUHhH8u0ENuIdBVTvfcml/0II4SdaWpeLEEKIE5BAF0IIPyGBLoQQfkICXQgh/IQEuhBC+AkJdCGE8BMS6EII4Sf+D3tEkIUr0dMeAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"2hwzerhum81E"},"source":["Training and validation accuracy seems to improve after around 60 epochs"]},{"cell_type":"code","metadata":{"id":"mzo9TrI5m81E","executionInfo":{"status":"ok","timestamp":1604684948276,"user_tz":420,"elapsed":316223,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"fb2d1917-506d-417d-ee2d-3e98540bf119","colab":{"base_uri":"https://localhost:8080/"}},"source":["results = model.evaluate(X_test, y_test)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["313/313 [==============================] - 0s 1ms/step - loss: 0.5939 - accuracy: 0.8625\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"_JDbnxfRm81G","executionInfo":{"status":"ok","timestamp":1604684948277,"user_tz":420,"elapsed":316219,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"3d9794e1-46d3-4574-8adb-1249943d0555","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Test accuracy: ', results[1])"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Test accuracy:  0.862500011920929\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"OMfsNCzcm81I"},"source":["## 2. Nonlinearity (Activation function)\n","- Sigmoid functions suffer from gradient vanishing problem, making training slower\n","- There are many choices apart from sigmoid and tanh; try many of them!\n","    - **'relu'** (rectified linear unit) is one of the most popular ones\n","    - **'selu'** (scaled exponential linear unit) is one of the most recent ones\n","- Doc: https://keras.io/activations/"]},{"cell_type":"markdown","metadata":{"id":"fO6HhMaLm81I"},"source":["<img src=\"http://cs231n.github.io/assets/nn1/sigmoid.jpeg\" style=\"width: 400px\"/>\n","<center> **Sigmoid Activation Function** </center>\n","<img src=\"http://cs231n.github.io/assets/nn1/relu.jpeg\" style=\"width: 400px\"/>\n","<center> **Relu Activation Function** </center>"]},{"cell_type":"code","metadata":{"id":"phc_0Rgdm81J"},"source":["def mlp_model():\n","    model = Sequential()\n","    \n","    model.add(Dense(50, input_shape = (784, )))\n","    model.add(Activation('relu'))    # use relu\n","    model.add(Dense(50))\n","    model.add(Activation('relu'))    # use relu\n","    model.add(Dense(50))\n","    model.add(Activation('relu'))    # use relu\n","    model.add(Dense(50))\n","    model.add(Activation('relu'))    # use relu\n","    model.add(Dense(10))\n","    model.add(Activation('softmax'))\n","    \n","    sgd = optimizers.SGD(lr = 0.001)\n","    model.compile(optimizer = sgd, loss = 'categorical_crossentropy', metrics = ['accuracy'])\n","    \n","    return model"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"yeEYkRCwm81L","executionInfo":{"status":"ok","timestamp":1604685184126,"user_tz":420,"elapsed":552058,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"37cbe657-7e64-42d7-be4e-7422a71f5ab0","colab":{"base_uri":"https://localhost:8080/"}},"source":["model = mlp_model()\n","history = model.fit(X_train, y_train, validation_split = 0.3, epochs = 100, verbose = 1)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Epoch 1/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 1.1128 - accuracy: 0.7314 - val_loss: 0.5504 - val_accuracy: 0.8466\n","Epoch 2/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.4489 - accuracy: 0.8700 - val_loss: 0.4168 - val_accuracy: 0.8827\n","Epoch 3/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.3434 - accuracy: 0.8988 - val_loss: 0.3719 - val_accuracy: 0.8918\n","Epoch 4/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.2895 - accuracy: 0.9141 - val_loss: 0.3231 - val_accuracy: 0.9103\n","Epoch 5/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.2528 - accuracy: 0.9248 - val_loss: 0.2951 - val_accuracy: 0.9153\n","Epoch 6/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.2266 - accuracy: 0.9321 - val_loss: 0.2940 - val_accuracy: 0.9170\n","Epoch 7/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.2094 - accuracy: 0.9368 - val_loss: 0.2848 - val_accuracy: 0.9194\n","Epoch 8/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1935 - accuracy: 0.9423 - val_loss: 0.2735 - val_accuracy: 0.9215\n","Epoch 9/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1820 - accuracy: 0.9458 - val_loss: 0.2632 - val_accuracy: 0.9277\n","Epoch 10/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1702 - accuracy: 0.9486 - val_loss: 0.2506 - val_accuracy: 0.9304\n","Epoch 11/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1605 - accuracy: 0.9518 - val_loss: 0.2467 - val_accuracy: 0.9319\n","Epoch 12/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1527 - accuracy: 0.9535 - val_loss: 0.2390 - val_accuracy: 0.9327\n","Epoch 13/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1447 - accuracy: 0.9558 - val_loss: 0.2948 - val_accuracy: 0.9189\n","Epoch 14/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1380 - accuracy: 0.9576 - val_loss: 0.2315 - val_accuracy: 0.9362\n","Epoch 15/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1324 - accuracy: 0.9604 - val_loss: 0.2351 - val_accuracy: 0.9369\n","Epoch 16/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1266 - accuracy: 0.9624 - val_loss: 0.2313 - val_accuracy: 0.9373\n","Epoch 17/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1217 - accuracy: 0.9626 - val_loss: 0.2308 - val_accuracy: 0.9385\n","Epoch 18/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1172 - accuracy: 0.9646 - val_loss: 0.2286 - val_accuracy: 0.9398\n","Epoch 19/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1130 - accuracy: 0.9659 - val_loss: 0.2267 - val_accuracy: 0.9398\n","Epoch 20/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1095 - accuracy: 0.9664 - val_loss: 0.2253 - val_accuracy: 0.9408\n","Epoch 21/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1049 - accuracy: 0.9676 - val_loss: 0.2281 - val_accuracy: 0.9411\n","Epoch 22/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.1017 - accuracy: 0.9695 - val_loss: 0.2252 - val_accuracy: 0.9417\n","Epoch 23/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0984 - accuracy: 0.9697 - val_loss: 0.2267 - val_accuracy: 0.9412\n","Epoch 24/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0946 - accuracy: 0.9718 - val_loss: 0.2310 - val_accuracy: 0.9422\n","Epoch 25/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0921 - accuracy: 0.9720 - val_loss: 0.2272 - val_accuracy: 0.9419\n","Epoch 26/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0891 - accuracy: 0.9733 - val_loss: 0.2260 - val_accuracy: 0.9438\n","Epoch 27/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0860 - accuracy: 0.9749 - val_loss: 0.2251 - val_accuracy: 0.9429\n","Epoch 28/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0823 - accuracy: 0.9757 - val_loss: 0.2363 - val_accuracy: 0.9436\n","Epoch 29/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0800 - accuracy: 0.9759 - val_loss: 0.2431 - val_accuracy: 0.9423\n","Epoch 30/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0776 - accuracy: 0.9765 - val_loss: 0.2289 - val_accuracy: 0.9447\n","Epoch 31/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0759 - accuracy: 0.9773 - val_loss: 0.2300 - val_accuracy: 0.9434\n","Epoch 32/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0735 - accuracy: 0.9784 - val_loss: 0.2301 - val_accuracy: 0.9450\n","Epoch 33/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0715 - accuracy: 0.9792 - val_loss: 0.2318 - val_accuracy: 0.9441\n","Epoch 34/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0692 - accuracy: 0.9794 - val_loss: 0.2324 - val_accuracy: 0.9446\n","Epoch 35/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0676 - accuracy: 0.9804 - val_loss: 0.2306 - val_accuracy: 0.9457\n","Epoch 36/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0651 - accuracy: 0.9807 - val_loss: 0.2379 - val_accuracy: 0.9436\n","Epoch 37/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0639 - accuracy: 0.9815 - val_loss: 0.2304 - val_accuracy: 0.9456\n","Epoch 38/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0613 - accuracy: 0.9825 - val_loss: 0.2417 - val_accuracy: 0.9455\n","Epoch 39/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0600 - accuracy: 0.9825 - val_loss: 0.2418 - val_accuracy: 0.9459\n","Epoch 40/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0588 - accuracy: 0.9832 - val_loss: 0.2460 - val_accuracy: 0.9445\n","Epoch 41/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0572 - accuracy: 0.9828 - val_loss: 0.2414 - val_accuracy: 0.9452\n","Epoch 42/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0552 - accuracy: 0.9837 - val_loss: 0.2456 - val_accuracy: 0.9441\n","Epoch 43/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0544 - accuracy: 0.9844 - val_loss: 0.2437 - val_accuracy: 0.9461\n","Epoch 44/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0532 - accuracy: 0.9843 - val_loss: 0.2438 - val_accuracy: 0.9466\n","Epoch 45/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0511 - accuracy: 0.9855 - val_loss: 0.2429 - val_accuracy: 0.9460\n","Epoch 46/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0492 - accuracy: 0.9862 - val_loss: 0.2568 - val_accuracy: 0.9448\n","Epoch 47/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0485 - accuracy: 0.9865 - val_loss: 0.2469 - val_accuracy: 0.9459\n","Epoch 48/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0469 - accuracy: 0.9863 - val_loss: 0.2509 - val_accuracy: 0.9459\n","Epoch 49/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0467 - accuracy: 0.9863 - val_loss: 0.2481 - val_accuracy: 0.9472\n","Epoch 50/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0447 - accuracy: 0.9870 - val_loss: 0.2511 - val_accuracy: 0.9464\n","Epoch 51/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0442 - accuracy: 0.9868 - val_loss: 0.2573 - val_accuracy: 0.9456\n","Epoch 52/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0425 - accuracy: 0.9875 - val_loss: 0.2563 - val_accuracy: 0.9455\n","Epoch 53/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0419 - accuracy: 0.9881 - val_loss: 0.2541 - val_accuracy: 0.9458\n","Epoch 54/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0404 - accuracy: 0.9886 - val_loss: 0.2624 - val_accuracy: 0.9472\n","Epoch 55/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0396 - accuracy: 0.9885 - val_loss: 0.2640 - val_accuracy: 0.9445\n","Epoch 56/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0388 - accuracy: 0.9888 - val_loss: 0.2718 - val_accuracy: 0.9448\n","Epoch 57/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0375 - accuracy: 0.9895 - val_loss: 0.2643 - val_accuracy: 0.9468\n","Epoch 58/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0361 - accuracy: 0.9897 - val_loss: 0.2728 - val_accuracy: 0.9439\n","Epoch 59/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0360 - accuracy: 0.9897 - val_loss: 0.2633 - val_accuracy: 0.9461\n","Epoch 60/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0347 - accuracy: 0.9905 - val_loss: 0.2640 - val_accuracy: 0.9468\n","Epoch 61/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0336 - accuracy: 0.9906 - val_loss: 0.2716 - val_accuracy: 0.9468\n","Epoch 62/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0330 - accuracy: 0.9908 - val_loss: 0.2772 - val_accuracy: 0.9449\n","Epoch 63/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0325 - accuracy: 0.9907 - val_loss: 0.2772 - val_accuracy: 0.9453\n","Epoch 64/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0320 - accuracy: 0.9911 - val_loss: 0.2770 - val_accuracy: 0.9471\n","Epoch 65/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0309 - accuracy: 0.9917 - val_loss: 0.2720 - val_accuracy: 0.9482\n","Epoch 66/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0302 - accuracy: 0.9918 - val_loss: 0.2828 - val_accuracy: 0.9473\n","Epoch 67/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0288 - accuracy: 0.9926 - val_loss: 0.2858 - val_accuracy: 0.9460\n","Epoch 68/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0287 - accuracy: 0.9923 - val_loss: 0.2855 - val_accuracy: 0.9459\n","Epoch 69/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0277 - accuracy: 0.9925 - val_loss: 0.2984 - val_accuracy: 0.9451\n","Epoch 70/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0269 - accuracy: 0.9929 - val_loss: 0.2891 - val_accuracy: 0.9447\n","Epoch 71/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0264 - accuracy: 0.9929 - val_loss: 0.2865 - val_accuracy: 0.9466\n","Epoch 72/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0251 - accuracy: 0.9933 - val_loss: 0.2896 - val_accuracy: 0.9472\n","Epoch 73/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0245 - accuracy: 0.9934 - val_loss: 0.2961 - val_accuracy: 0.9467\n","Epoch 74/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0248 - accuracy: 0.9934 - val_loss: 0.2961 - val_accuracy: 0.9462\n","Epoch 75/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0243 - accuracy: 0.9937 - val_loss: 0.2974 - val_accuracy: 0.9469\n","Epoch 76/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0226 - accuracy: 0.9943 - val_loss: 0.2977 - val_accuracy: 0.9474\n","Epoch 77/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0223 - accuracy: 0.9946 - val_loss: 0.3038 - val_accuracy: 0.9480\n","Epoch 78/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0220 - accuracy: 0.9940 - val_loss: 0.3051 - val_accuracy: 0.9472\n","Epoch 79/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0216 - accuracy: 0.9944 - val_loss: 0.3116 - val_accuracy: 0.9462\n","Epoch 80/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0207 - accuracy: 0.9950 - val_loss: 0.3021 - val_accuracy: 0.9489\n","Epoch 81/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0203 - accuracy: 0.9945 - val_loss: 0.3098 - val_accuracy: 0.9484\n","Epoch 82/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0196 - accuracy: 0.9952 - val_loss: 0.3139 - val_accuracy: 0.9468\n","Epoch 83/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0193 - accuracy: 0.9953 - val_loss: 0.3157 - val_accuracy: 0.9466\n","Epoch 84/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0188 - accuracy: 0.9954 - val_loss: 0.3180 - val_accuracy: 0.9458\n","Epoch 85/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0191 - accuracy: 0.9953 - val_loss: 0.3196 - val_accuracy: 0.9485\n","Epoch 86/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0182 - accuracy: 0.9953 - val_loss: 0.3267 - val_accuracy: 0.9452\n","Epoch 87/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0176 - accuracy: 0.9960 - val_loss: 0.3224 - val_accuracy: 0.9472\n","Epoch 88/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0177 - accuracy: 0.9957 - val_loss: 0.3195 - val_accuracy: 0.9476\n","Epoch 89/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0167 - accuracy: 0.9961 - val_loss: 0.3250 - val_accuracy: 0.9476\n","Epoch 90/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0163 - accuracy: 0.9964 - val_loss: 0.3281 - val_accuracy: 0.9472\n","Epoch 91/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0155 - accuracy: 0.9966 - val_loss: 0.3259 - val_accuracy: 0.9470\n","Epoch 92/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0154 - accuracy: 0.9964 - val_loss: 0.3387 - val_accuracy: 0.9447\n","Epoch 93/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0152 - accuracy: 0.9968 - val_loss: 0.3323 - val_accuracy: 0.9466\n","Epoch 94/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0147 - accuracy: 0.9968 - val_loss: 0.3295 - val_accuracy: 0.9470\n","Epoch 95/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0147 - accuracy: 0.9970 - val_loss: 0.3338 - val_accuracy: 0.9472\n","Epoch 96/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0143 - accuracy: 0.9970 - val_loss: 0.3401 - val_accuracy: 0.9477\n","Epoch 97/100\n","1313/1313 [==============================] - 3s 2ms/step - loss: 0.0143 - accuracy: 0.9968 - val_loss: 0.3434 - val_accuracy: 0.9459\n","Epoch 98/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0136 - accuracy: 0.9971 - val_loss: 0.3385 - val_accuracy: 0.9469\n","Epoch 99/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0130 - accuracy: 0.9975 - val_loss: 0.3472 - val_accuracy: 0.9472\n","Epoch 100/100\n","1313/1313 [==============================] - 2s 2ms/step - loss: 0.0127 - accuracy: 0.9977 - val_loss: 0.3448 - val_accuracy: 0.9450\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Fig-BD1Qm81O","executionInfo":{"status":"ok","timestamp":1604685184396,"user_tz":420,"elapsed":552321,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"93c42cd3-cfb0-4708-d293-145d66ba0e2f","colab":{"base_uri":"https://localhost:8080/","height":265}},"source":["plt.plot(history.history['accuracy'])\n","plt.plot(history.history['val_accuracy'])\n","plt.plot(history.history['loss'])\n","plt.plot(history.history['val_loss'])\n","plt.legend(['train acc', 'valid acc', 'train loss', 'valid loss'], loc = 'upper left')\n","plt.show()"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZwU9Z3/8de3qqvPuQ+uGWQAEYf7EklQQFGDF96iIYlxXd2YaGLM5hc2uyHGXBrNJrprYoyrUVdljQZPlETDoYkaUQFRVJBDZjjmvvus+v7+qJ5mBoZhkBlmeubzfDz6MdPd1VXf6up+17e+9e1vKa01Qggh0p/R2wUQQgjRPSTQhRCin5BAF0KIfkICXQgh+gkJdCGE6Cc8vbXggoICXVJS0luLF0KItPT2229Xaa0LO3qu1wK9pKSEdevW9dbihRAiLSmldh7qOWlyEUKIfkICXQgh+gkJdCGE6Cd6rQ29I/F4nLKyMiKRSG8XJa35/X6Ki4uxLKu3iyKEOIb6VKCXlZWRmZlJSUkJSqneLk5a0lpTXV1NWVkZI0eO7O3iCCGOoT7V5BKJRMjPz5cwPwpKKfLz8+UoR4gBqE8FOiBh3g3kPRRiYOpzgX44zfFm9jXvQ4b9FUKI9tIu0MOJMFXhKhztdPu86+rq+M1vfvOZXnvOOedQV1fXzSUSQoiuS7tAV7jNCZrur6F3FuiJRKLT165YsYKcnJxuL5MQQnRV+gV6sn24J5pclixZwieffMKUKVP47ne/y+rVqzn11FNZuHAh48aNA+DCCy9k+vTpjB8/nvvuuy/12pKSEqqqqtixYwelpaVce+21jB8/nrPOOotwOHzQsp577jlOPvlkpk6dyhlnnMG+ffsAaGpq4uqrr2bixIlMmjSJp556CoCXXnqJadOmMXnyZObPn9/t6y6ESH99qttiWz967n0+2N1w0OMJJ0HMjhKw6lO19a4aNyyLH54//pDP33bbbWzatIn169cDsHr1at555x02bdqU6gL4wAMPkJeXRzgc5qSTTuKSSy4hPz+/3Xy2bNnC448/zu9//3suv/xynnrqKb70pS+1m+aUU07hjTfeQCnF/fffzy9+8Qt++ctf8uMf/5js7Gzee+89AGpra6msrOTaa69l7dq1jBw5kpqamiNabyHEwNBnA/2QWjNcazgGvTlmzpzZrj/33XffzfLlywHYtWsXW7ZsOSjQR44cyZQpUwCYPn06O3bsOGi+ZWVlLFq0iD179hCLxVLLePnll1m2bFlqutzcXJ577jnmzJmTmiYvL69b11EI0T/02UA/VE26IdrArsZdjM4Zjd/j7/FyhEKh1P+rV6/m5Zdf5vXXXycYDDJv3rwO+3v7fL7U/6ZpdtjkcuONN3LzzTezcOFCVq9ezS233NIj5RdCDBzSht5GZmYmjY2Nh3y+vr6e3NxcgsEgH374IW+88cZnXlZ9fT1FRUUAPPTQQ6nHzzzzTO65557U/draWmbNmsXatWvZvn07gDS5CCE6lH6B3oO9XPLz85k9ezYTJkzgu9/97kHPL1iwgEQiQWlpKUuWLGHWrFmfeVm33HILl112GdOnT6egoCD1+H/8x39QW1vLhAkTmDx5MqtWraKwsJD77ruPiy++mMmTJ7No0aLPvFwhRP+leusHOjNmzNAHXuBi8+bNlJaWdvq65ngzO+p3UJJVQsgb6nTagawr76UQIv0opd7WWs/o6Lm0raE7dP8Pi4QQIp2lX6D3YBu6EEKks/QL9B5sQxdCiHSWfoEuNXQhhOhQ+gW61NCFEKJD6RfoUkMXQogOpV+g97EaekZGBgC7d+/m0ksv7XCaefPmcWAXTSGE6G4S6N1k2LBhPPnkk71dDCHEAHbYQFdKPaCUqlBKbTrE80opdbdSaqtSaqNSalr3F3M/Q7lF7okLXCxZsqTdz+5vueUW7rzzTpqampg/fz7Tpk1j4sSJPPPMMwe9dseOHUyYMAGAcDjMFVdcQWlpKRdddFGHY7kA3HrrrZx00klMmDCB6667LtWMtHXrVs444wwmT57MtGnT+OSTTwC4/fbbmThxIpMnT2bJkiXdvfpCiDTXlcG5/gD8N/DwIZ4/GxiTvJ0M/Db59+i8uAT2vtfBE5qSeAuWaYHhPbJ5DpkIZ992yKcXLVrETTfdxDe+8Q0AnnjiCVauXInf72f58uVkZWVRVVXFrFmzWLhw4SGv3fnb3/6WYDDI5s2b2bhxI9OmdbyPu+GGG1i6dCkAX/7yl3n++ec5//zzWbx4MUuWLOGiiy4iEongOA4vvvgizzzzDG+++SbBYFDGcxFCHOSwNXSt9Vqgs/S4AHhYu94AcpRSQ7urgAdSKI5wGPQumzp1KhUVFezevZsNGzaQm5vL8OHD0Vrz/e9/n0mTJnHGGWdQXl6euiBFR9auXZsa/3zSpElMmjSpw+lWrVrFySefzMSJE/nrX//K+++/T2NjI+Xl5Vx00UUA+P1+gsEgL7/8MldffTXBYBCQIXSFEAfrjuFzi4Bdbe6XJR/bc1Rz7aQm/Wn1ZnL8OQwNdf9+47LLLuPJJ59k7969qUGwHn30USorK3n77bexLIuSkpIOh809EpFIhK9//eusW7eO4cOHc8sttxz1PIXoDlprbEeTcDSO1jj64F5ltqOJ25qE4xCNO0QSNpG4Q9x2b7ajMZXCZ5n4LQNDqdRz0UTyFrexHfB6DLweA0NBJO4QjttE4zZau+fKHE27cijc3m5KtT+nFm19bcJBAR5DYRgKx9Hucm0H23bXK+G4ZXRv7qUVTOVODxBL7F+PVk7yfWktC5Asi1se9/WQsN3p4o4mkVznmK2xHQfbAdtx+Ob8MZw3aVi3b7tjOh66Uuo64DqA44477mjm02PdFhctWsS1115LVVUVa9asAdyhbgcNGoRlWaxatYqdO3d2Oo85c+bw2GOPcfrpp7Np0yY2btx40DSt4V1QUEBTUxNPPvkkl156KZmZmRQXF/P0009z4YUXEo1GsW2bM888k1tvvZXFixenmlykln7s2I4mkgyLaMImGndDKW477YLF0W5gxG2HlphNOJYgHLdJ2BqdDKZWGojEbVpiNi2xRCoI9geYTk2vcAPMTs47YWuiCYdI3E7N39FuV4GE7dAUTdASs4klHDymwjINTEO583c0tnYDOZ5wiDtOKqDQEHcc+mOvYK9pYJkK01B4ku+Hqdz7WrvviZ08Nefz7J+2tWlVgfsaQ2EoleqY4Thtt5e7I/GYCtMw8Cbf+4DXwJN8rakUmX6rR9axOwK9HBje5n5x8rGDaK3vA+4Dd7TFz7pAheqxXi7jx4+nsbGRoqIihg51jwAWL17M+eefz8SJE5kxYwYnnnhip/O4/vrrufrqqyktLaW0tJTp06cfNE1OTg7XXnstEyZMYMiQIZx00kmp5x555BH+5V/+haVLl2JZFn/84x9ZsGAB69evZ8aMGXi9Xs455xx+9rOfde/KHyMJ26ExkiBm7z+xnXA04ZhNOGaTcBws0621KaAxmqApkqA56oajW4NzX6sUOBoawnHqw3EawnGaYwnCcYdIzCaerIklbO3WwpJfUKNNs13cdgjH3Bqm7WgMBUYy/FqXGbd7NuFMQ6W+8EayfEbyf3Brphq3FmmZBh5T4fMY+C2TgGXiMRUeZaAUeHweinODBL0mXo9Bwtap96G1FmoqheVReE33tW13Sl7TDTyP6U7X+lzrKSOt3fJaydDyW245/JaB1zSTgana7QQdR2OZBlYyKP2Wid9jYhgQT2iiCRtHQ8AyCXgNfB4TpcBILt8tx/4yOFoftNNpfT98HrclufUoo/V9PdQ5r/6kS8PnKqVKgOe11hM6eO5c4AbgHNyToXdrrWcebp6fdfhcgI9rPyboCVKcWXzYaQeqoxk+V2u39tcSs2lO1vQaIm5YNkTiNEfdGmU45uw/dNWaluj+6aIJtxaZcJzUYXRzNJGqOfaELL+HrIBFhs+TChjL3F8zaq0h2wc0IXhNA7/XDRiPobCTtS1TKUI+DwGvG5p+yw0ar8dI/W8a7X/oZhr7lxfymQQs9/UeQ7ULqFZugJl4TWNABI44ep0Nn3vYGrpS6nFgHlCglCoDfghYAFrre4EVuGG+FWgBru6eYh+agdHn+qH3Np08hHbb/DSxhMM/ttdQ0xylujlGTVOMyqYolY1RappjbljHEkRibs3ITrYPth7Cd/WQO9X2qNwAywpYZPo9bjiaipDlIS9kEvS6t5DPQ5bfncZnGalaoWlAwOtxa5uGImY7xBIOGsj0ecj0ewj5PAS9ZrtaGLi1xwy/JxWuQgxUhw10rfWVh3leA9/othJ1QU+2ofem1rbN1sPJ1pMwrbe4nTzJkmwHdd+BZJttQrfbyVU0Rrn20dfbzT87YFGY6SMv6KUgw8sIX5CAZaYOR00jWWO0THyWSchrEvR6CPpMsvwW2QGLrIBFyOc+3vpaIUTf0GcvEt2ZnmxD706Odk86xWwnFcYaUmfv4wmdfM5JnT3vjIL2bZuAUgZej8IbcP+a7il3EjVeHrlmJnkhL/khH7khC5/HPBarLYToJekZ6H2kht7avau1eSBu61TXrEi8tcmg43Iq5Z5UskyDDJ9n/xnw5Imw1vZWo83JMs8RnNgJWCbTxhR25+oKIfq49Ax01DG7BF1r23Rr23Jr/9REMrztA3YsRrIXgt8yyA548HlMLI/bfcljGKk+q0IMeI4D8RYwvWBaIN+Lo5aega4U2un+GnrrScGWmPvDhkiyz3HbHxd4TQOPaeCzDDL8HrzJ7nWWx8Dqi92jtIbmKog1gS8LfJnulycRdb9Mjg3+bPAkh1GINkHtdmjYA/4sCOa7rwvXQtNeaKkGj9+djzcDtAOJiDu/aAOE6yBS7z6fcxxkDwc7Bg3lUF8G0UZQhvvl1Q7Ycfe1hgdCBe7NmwHxsHtzEm45/NlgWNC4251P4z5IhN152wm3/B6/e/NnQyAHfNnuNK1l0q29a5RbjpZq96aUu0xvBnhD+2+md39ZwS2LY0OsGVqqoLnafSz/eCgY466vY4MTd8veUA715RCugdwSKCyFvJHu9qj7FBrKINLgbodYs7utkk1mBHIhqwgyh4LH565nIuquS+MeaExui0hy3RwbfMl18Ge7rw/mudtBme56GB7wBsEKuX9bgxTc97OhHJoqIN4M8QjYye3i8bvTevzu+2wmyxNvgVjL/u1vR93XJZLbTpnucrwhtxz+bPcWD0PlR1C91Z1HKysIGYMgY7D7uTNMdx6mldwmGe5jNdugagvU7XLfz6GTYdA49zNev8t9z+Otn424u91bK16Gx52fx+f+rwz3Fsh1t1HeSHc968vc96Ol2p1H23k5jrvd7WjyMe2WO2uY+57HWtzPV7TRfZ0Td1+TOcRdRu4IGDEb8kd3+9c9PQO9m9rQW7vntfZfrqiqYcXTf2TRVf+Mx3BDOyfgbdPP9tAnAc855xwee+wxcnJyOlugG2JOglt+dCsZwQD/+q2vu19GNKkxDbTtPpb6IOrka5Ovx3GnVSr5Gu1+YFqnV8r9wt/9RWjY7X7hDsebDPpwL4wRo4zkeh0Bb0YyYHzul9yOu+sZD3e8vspww6H1vfRluqERzAMUNFVCrNEN1liLG0qHYljujidY4L79O//WPpja8meDPwfef7rNDiXJ9LrPW8nQU8b+bdxSDc0VHc8zkOeGQzDf3ZH4c9z3INrkhlqk3g3McI0bKq2fITvu/j0U0wsZQ9yyWH43uJ0WsCvdwG692VH3OSvgTtu6I/VlQqjQ/d8KuOsRa3Lf02iT+5mM1LvvX+EJUHKKux6tgRltcte5aZ8b1k7Cfc/seHK7NLvLzi2BgrEwah5UfwJb/gzrH3W3Y+YQd0foy9hfeWm3U3bceSSiye9YMqRrd8DmZ91lprZdjrudTZ87H9NyP0OGx92x+TLdz5923DJv+9jdblbQrYT4Mve/FgVlb8H7y93lnfdrCfRWR9OG3vpjkaZogsZIgmjC/ZIFvR68doTljz3ILUtuxjLbD3OTSCQwFcngTIan4XG/SMCKFSvcCcN17sbVTptbm1Bu/UK1VIEKQt2hfnWqkjUUY//91g9m62NOmwBXhvvhaQ14wwNDp8CJ50JWsfvhijW5tehEzP3CWUH3dZE6aKlxgzDnOMgb5dY2oo3u45E6twaTMdj9gNux/TUQZbofao8vWQvLcUMqUu/Wcup3uR/o7GK3tu7L3B9aymgTyAk3gJor3UC1Au5NGe5yIvXuFzGryL35sw69kVtrspF6dx7+7GRN9QiOnBzbXc/UThR3Pdps8/3TOu6RQ8Nu9znDckMta6i7XHDf85pP3OAIFbrvRagQjE6GU0rE3KMiJ+GGrWEldwD+rq9HW1q727g1HJ1Ech0dN8iDeenZ7KG1ewTpzdh/pPlZ2An382rHIbvI3Vl1NzvhHpn5Ovn8HoW0DHQD44gDPRyzqW6OUtcSx9E62W/aQ0GGl6yAhWUaXPHNH7J92zZOmjaVM0+bw7lnzuUHP7md3OxMPtyyjY9ffZoL/+nb7Nq9l0g0xrf+eTHXffO74MukpKSEda++TNPujzj7y9/klFkn8fd/vEPRsCE889j/EAgE3S+LkdzD+3MhmAmFpazf+B5f+/oNtIRbGD1qNA/8z/+Qm5/P3Xffzb333ovH42HcuHEsW7aMNWvW8K1vfQtwd2xr164lMzPz4BXeF4XLHuyOt/uz8WW4X4quDrxpepKH24OOftkeH2QOdm+flWGCEejitEZyh9XJD908XhhU6t66yuN1d7DdRan9O8pQQffNt7cplTzSOkqmx21y6Ummxz3C6CF9NtBv/8ftfFjzYYfPxewYCZ0g6Akedj5OslnFcTQjMsfwran/Sk7AIuj1pAbicWsuUW77wb+yacM7rH/xDwCs/vs63tnwPptee4GRI0eB4eGB3/2GvPx8wpEoJ51yOpecfRr5JePdWk79p2AF2LJtJ48/8RS/nzKFyy+/nKdefjM1+mKKlWwusPx85epr+K//+i/mzp3L0qVL+dGPf8yvf/1rbrvtNrZv347P56Ourg6AO++8k3vuuYfZs2fT1NSE3/8Za2tCiH6nzwb60dLgDjyUHDzJayry/IpifwwSjRCJuoe0rSc20G5TCTrZBpcJ+bXMPPlkRk47LTXfu3/1e5YvXw7Arj0VbCmvIT+v3D189QQhWMjIkSOZMmUKANOnT2fHjh2HLGd9fT11dXXMnTsXgKuuuorLLrsMcIfeXbx4MRdeeCEXXnghALNnz+bmm29m8eLFXHzxxRQXy/AHQghXnw3078383iGf29u8l9pILaX5HR++tsQS7KmuJ+A0kWdG8DlhVGvbde12929r268VhIDXbaNsDrjt0K2H/cogFNrfjrZ69WpefvllXn/9dYLBIPPmzSPizXN7IijDPVxrCePz+VKvMU3zkFcsOpwXXniBtWvX8txzz/HTn/6U9957jyVLlnDuueeyYsUKZs+ezcqVKw87WJgQYmDos4Hemc56uTRG4jRU72W0qnLPDxp+CBS64W1ayRNWXrcd+wCZeQaNjY2HXG59fT25ubkEg0E+/PBD3njjDbf9LnNIxyfLuiA7O5vc3FxeffVVTj31VB555BHmzp2L4zjs2rWL0047jVNOOYVly5bR1NREdXU1EydOZOLEibz11lt8+OGHEuhCCCBdAz3Zy0Vr3a7Pd21zjMa6SoarKhxvJkbOcDfIuyg/P5/Zs2czYcIEzj77bM4999x2zy9YsIB7772X0tJSxo4dy6xZs7plfR566CG+9rWv0dLSwqhRo3jwwQexbZsvfelL1NfXo7Xmm9/8Jjk5OfzgBz9g1apVGIbB+PHjOfvss7ulDEKI9Nel4XN7wtEMn1vZUklFSwWl+aWpi0Y3hONUVVdRYuxDWUFU/ujPVGPuL45m+FwhRN91VMPn9kWttXKtNShwHIfm2n2UGFUojw+VP2pAh7kQYmBKz0Bvcx1BwrU4dbsZSgzbE8TIH9Vh+7gQQvR3aZl8qRp6SzU07CahvdR6iygsKEzPX7oJIUQ36OR3x32XkSy2bq4iovx8QjE5eQUS5kKIAS0tAz1VQ3fi7LGzGZTlO2jsFSGEGGjSMgVb6+Fxw0sjQfJCXe+aKIQQ/VV6Bnrc/eVlk5mVuppPb8nIyABg9+7dXHrppR1OM2/ePA7sotnZ40II8VmkX6BrjQrXAtCs/X2mqWXYsGE8+eSTvV0MIcQA1jfS8EhEG1GJKABxR+P1dN8qLFmyhHvuuSd1/5ZbbuHOO++kqamJ+fPnM23aNCZOnMgzzzxz0Gt37NjBhAkTAAiHw1xxxRWUlpZy0UUXdWksl8cff5yJEycyYcIEvvc9dxwb27b56le/yoQJE5g4cSK/+tWvALj77rsZN24ckyZN4oorruiOVRdC9AN9ttvi3p/9jOjmDobPdeLYdhyFxtEWUcPDzi6Guq/0RIZ8//uHfH7RokXcdNNNfOMb3wDgiSeeYOXKlfj9fpYvX05WVhZVVVXMmjWLhQsXHvJSc7/97W8JBoNs3ryZjRs3Mm3atE7LtXv3br73ve/x9ttvk5uby1lnncXTTz/N8OHDKS8vZ9OmTQCpIXQ7GlZXCCHSr4beejWY1rvd2Hw+depUKioq2L17Nxs2bCA3N5fhw4ejteb73/8+kyZN4owzzqC8vJx9+/Ydcj5r165NjX8+adIkJk2a1Oly33rrLebNm0dhYSEej4fFixezdu1aRo0axbZt27jxxht56aWXyMrKSs1z8eLF/O///i8eT5/dJwshjrE+mwad1aQjiQif1H2CE89jUE4BWQGr25Z72WWX8eSTT7J3714WLVoEwKOPPkplZSVvv/02lmVRUlJCJNKF63QepdzcXDZs2MDKlSu59957eeKJJ3jggQc6HFZXgl0IkX41dGjT1KG7/aTookWLWLZsGU8++WTqQhP19fUMGjQIy7JYtWoVO3ce6jqgrjlz5vDYY48BsGnTJjZu3Njp9DNnzmTNmjVUVVVh2zaPP/44c+fOpaqqCsdxuOSSS/jJT37CO++8025Y3dtvv536+nqampq6Z+WFEGktLat1Rmo/pPF6urfL4vjx42lsbKSoqIihQ4cCsHjxYs4//3wmTpzIjBkzDjv++PXXX8/VV19NaWkppaWlTJ8+vdPphw4dym233cZpp52G1ppzzz2XCy64gA0bNnD11VfjOO5Fin/+858fclhdIYTo0vC5SqkFwF2ACdyvtb7tgOePAx4CcpLTLNFar+hsnkczfG7cifNxzceQyGH8kKLDTj8QyfC5QvRPnQ2fe9j2CqWUCdwDnA2MA65USo07YLL/AJ7QWk8FrgB+c3RF7lxrDb03f1AkhBB9TVcaoGcCW7XW27TWMWAZcMEB02ggK/l/NrC7+4p4sNY29D7ymyIhhOgTuhKJRcCuNvfLko+1dQvwJaVUGbACuLGjGSmlrlNKrVNKrausrOxwYV1qAkqO5mJIoHeot65CJYToXd0ViVcCf9BaFwPnAI8opQ6at9b6Pq31DK31jMLCwoNm4vf7qa6uPmwg2Y77vNTQD6a1prq6Gr/ff/iJhRD9Sld6uZQDw9vcL04+1tY1wAIArfXrSik/UABUHElhiouLKSsr41C191axhENVZB9+s4FwoP5IFjEg+P1+iouLe7sYQohjrCuB/hYwRik1EjfIrwC+eMA0nwLzgT8opUoBP9B5KnfAsixGjhx52Ole2rSHf33vqywoOY87p/3wSBcjhBD90mEbLbTWCeAGYCWwGbc3y/tKqVuVUguTk30HuFYptQF4HPiq7sGG3LLaMFp78FlOTy1CCCHSTpd+WJTsU77igMeWtvn/A2B29xbt0Mpqwyg8KGUfq0UKIUSfl5anFctqw3iURcyJ9XZRhBCiz0jLQC+vC+M1LBJOoreLIoQQfUZaBnpZbQtej5eYLTV0IYRolXaBXh+O0xhJEPD4JNCFEKKNtAv08lr3cm4By0vcifdyaYQQou9Iu0Avq20BIOT1y0lRIYRoI+0CvbzOraFneH3EbamhCyFEq7QL9HFDs7j21JGEvD5pchFCiDbSLtBPHpXPv587Dq8pvVyEEKKttAv0VpZhSQ1dCCHaSNtAlxq6EEK0l7aBLjV0IYRoL20D3WtIP3QhhGgrfQNdmlyEEKKdtA10y7CwtY3tyBC6QggB6RzopgUgzS5CCJGUvoFuSKALIURbaRvoXtMLIO3oQgiRlL6BbriBLjV0IYRwpW2gp9rQZYAuIYQA0jjQW2voMoSuEEK40jbQ5aSoEEK0l76BnmxykZOiQgjhSttAl14uQgjRXtoGujS5CCFEe2kb6NJtUQgh2kvbQJdui0II0V6XAl0ptUAp9ZFSaqtSaskhprlcKfWBUup9pdRj3VvMg0m3RSGEaM9zuAmUUiZwD3AmUAa8pZR6Vmv9QZtpxgD/BszWWtcqpQb1VIFbSS8XIYRorys19JnAVq31Nq11DFgGXHDANNcC92itawG01hXdW8yDyUlRIYRoryuBXgTsanO/LPlYWycAJyil/qaUekMptaCjGSmlrlNKrVNKrausrPxsJU6SbotCCNFed50U9QBjgHnAlcDvlVI5B06ktb5Paz1Daz2jsLDwqBYoNXQhhGivK4FeDgxvc784+VhbZcCzWuu41no78DFuwPeY1hq6BLoQQri6EuhvAWOUUiOVUl7gCuDZA6Z5Grd2jlKqALcJZls3lvMgrTV0aXIRQgjXYQNda50AbgBWApuBJ7TW7yulblVKLUxOthKoVkp9AKwCvqu1ru6pQgMYysCjPFJDF0KIpMN2WwTQWq8AVhzw2NI2/2vg5uTtmLFMS2roQgiRlLa/FAW32UVq6EII4UrrQPeaXqmhCyFEUtoFenjjRqruvRettdTQhRCijbQL9JZ33qHy13dh19XhNb0yOJcQQiSlXaB7i4sBiJeVYxmWDM4lhBBJaRfoVmugl5dJk4sQQrSRfoFe5A4jEy8rk5OiQgjRRtoFupmZiZGdTaw82eQigS6EEEAaBjqAt6iIeFk5XtNLwkn0dnGEEKJPSMtAt4qL3SYXwysnRYUQIn48/NkAACAASURBVCl9A728HEt5pNuiEEIkpWmgF6FjMbKaHKmhCyFEUloGujfZ0yWnJiYnRYUQIiktA721L3pOTVT6oQshRFJ6Bnqyhp5VHZY2dCGESErLQDf8fszCAjKqWqSGLoQQSWkZ6ADeomJClU1yUlQIIZLSNtCtoiICVU042pEfFwkhBOkc6MXF+KsaMRwtzS5CCEFaB3oRytHkNyBdF4UQgjQO9NZx0QfVSw1dCCEgjQO9tS/6oDoIx8O9XBohhOh96RvoQ4agDUVhvWZHw47eLo4QQvS6tA10ZVl4Bg9mUB1sq9/W28URQohel7aBDuArHs6wRo8EuhBCkOaBbhUXM7gettVJoAshRJoHehGZ9XF2Vm9Fa93bxRFCiF7VpUBXSi1QSn2klNqqlFrSyXSXKKW0UmpG9xXx0Hyjjwdg6PYGqiPVx2KRQgjRZx020JVSJnAPcDYwDrhSKTWug+kygW8Bb3Z3IQ8lY+4cdCjAaRs1n9R9cqwWK4QQfVJXaugzga1a621a6xiwDLigg+l+DNwORLqxfJ0yAgH8XziDkz/S7Ni7+VgtVggh+qSuBHoRsKvN/bLkYylKqWnAcK31C53NSCl1nVJqnVJqXWVl5REXtiNDLvsi/jjE/7y6W+YnhBDp6qhPiiqlDOA/ge8cblqt9X1a6xla6xmFhYVHu2gAAlMmUznYz5DVH3TL/IQQIl11JdDLgeFt7hcnH2uVCUwAViuldgCzgGeP1YlRpRRlc06gaHsj0W3bj8UihRCiT+pKoL8FjFFKjVRKeYErgGdbn9Ra12utC7TWJVrrEuANYKHWel2PlLgD+qxTsRVU/PHxY7VIIYTocw4b6FrrBHADsBLYDDyhtX5fKXWrUmphTxewK4aPnMS7oxWNzz6HTsjFLoQQA1OX2tC11iu01idorUdrrX+afGyp1vrZDqaddyxr5wCjskexapJCVdfR/Prrx3LRQgjRZ6T1L0VbDcsYxuYTAsSCXhqe77SjjRBC9Fv9ItANZTA8fxQfT86j8S9/wYkcs67wQgjRZ/SLQAcYlTOKNSc6OC0tNK1e3dvFEUKIY67/BHr2KNYOqsYsLKD++ed7uzhCiAFEa028ooLwhg040Wjqcbupicrf/IYtp53OtvPPZ9c3bmDf7b8gvHFjj5TD0yNz7QXH5xyPNhSReTNwnn4Fu74eMzu7t4slhEgj0S1biO/dS2jWLJRldTqtXVdHw4sv0vjXVUQ++AC72h0gUAUChGbOxDtyJHXLl+PU1xM69VSUZRHbuYPm117DN2YMgUmTur38/SbQZw6ZiWVYvDnByyl/jNP4l7+Qc+mlvV0sIUQa0FpT+/DD7LvzlxCPY+blkX3++WR+4Sx8Y8ZgZmYCEN+zh5Z//IPGV/5K06pV6Hgc78iRZMyZg7/0RDyDBtPy1ls0vfoqTWvWkDFvHgU33EBgwvj9y3IcsO0eWY9+E+gZ3gxOHnoyf6rbwGkjRlD//AsS6EIMANpxiH70EdbQoZg5OYeeTmsSlZVEP/qY6McfgTKwiouwhgyh6nf30fTKK2ScfjrZF15Aw/MvUPPYY9Q89BAAnkGDUJZFvNz9kbyZl0fOlVeQc+GF+EpLUUqllpO14AsAOM3NGKHQQeVQhgFGz7R295tAB5h/3Hx+VP4j4vMvIf7gE8T37cMaPLi3iyWE6AF2fT11y5dT9/gyYjt3glL4TjiB4Ekn4R8/Ht/xx+MdWULk/Q9o/MtfaHzlFRJ79nQ8M8ti8L8tIfcrX0EpRdZZZ5GorSX87rtEP/mE2CfbcMJh8q76CsGZM/GdcIIbzJ3oKMx7muqtK/3MmDFDr1vXvb8/qg5Xc9oTp3HT4Cv53L8+Rvb55zPs5z/r1mUIIXqedhzsmhrie/eR2LuHWFkZ8V1lxMvKSFRVubfqakgkCEydSvZFF5KoqqLlrbcIv7sefUDXZeXzETr1FEIzZ+IbeyL+sSeAUu58y8rxjR6F7/jje2ltj4xS6m2tdYdjZfWrGnp+IJ+pg6byQmwd5139Vap/fz85l1xMcMYxGSdMCNEJu6mZxpUr0Y5NzgUXoLze1HNOOEzz66/Tsu5tWtatI7p5Mzoeb/d6IyMDq7gYz6BCfCecgGfwILK+8AX8paXtptOJBLFPPyW6ZSux7dvdNu5TT8EIBg8qUyA7m8D48Qc9nq76VQ0d4OH3H+aOdXfwwoKniF75NcxQBiP/9NRhz1gLIbqP3dRE/NNPSdTUYtfW0Pza32j485/R4TAA1nHHMeg73yEwdQq1jz1G3ePLsOvrUZaFf9IkApMnYw0bhmfwIKwhQ7CKizFzctq1VQ9UA6aGDjB/xHzuWHcHf638O5f8+79T9o0bqHn4EfKv+afeLpoQfYJ2HOzqajzdcE0CrTXYNtq20dEoza+/QcPzz9G0ek27GrYRCpF93nlkX3wRTlMTFb+4g/JvfQuSAZ0x/3TyvvhFAtOnY/h8R12ugarfBXpRRhGleaW88ukrXHX2w2TMm0flPfdg19ZgZGZh5uSQefpp3fJhbhXdtp3o1i1knXVWt81TiJ6QqKlh9//7Hs2vvUbWOedQ+O2b8A4ffsjpneZmGlb+mcaVK7Gbm0ADjoPT1Eiirg67tg4OGOHULCgg58orCM6YgScvDzMvD2vYMAy/PzVN6HOfo/6ZZ4jt/JScSy7GO2JET63ygNLvmlwA7t1wL/esv4dXLnuFnJo4Zdd/jdiOnakagwoEyPvqVeRfcw1mRsZRL2/XN26gafVqTnjj9VR/VSH6mpZ16yi/+TvYdXVknX02DStXom2bnEsvITh9Bt6SEqzBg4jt3Enkw48Ib9hA4yuvoFtasI47DmvoUHdGSmFmZmDm5mHm5GAE/GB6UB4PvrEnEDr5ZJSn39UV+4zOmlz6ZaBvr9/OwqcX8vXJX+f6KdenHneiUeKffkrVb39Lw4oXMXNzGfSdm8m+5JLP3DbnRCJ8POtz6EiE4nv+m8z587trNYQ4YrGdO2l8+RWiH3/s/uqxogJlmijTdLvxFhdR/Ktf4R83jvi+Cqr++7+p+9OfOvyhi5mbS+aZZ5J90YUEpkyR9us+YsAFOsCNr9zI+sr1rLxkJUHr4LPb4U3vU3HbbbSsW0do9myG/vhWrGHDjng5jX9dRdnXvw5A7he/yJClPzjqsgsBbgWk5c03SVTXpB5ThkrWhk2soiL848ejDAMdi1F1//1U3/s7dCyGp9DtCWING4bWDiRszPw8Cq6//qCj0taKTnTHDhJ79+EdcRy+sWPdH9NIiPc5A+qkaKtrJl7Dl1/8Msu3Lmdx6eKDng9MGM9xDz9E3f/9H/vuuJNt551P4c03k3vlFSjT7PJymlb9FSMjg8CkiTT//e/duQoiTcX3VRDeuIGMz3++3Y9Lolu20Pjyy2SecQa+MWM6fK22bZrWrKFhxYs0rVqF09zc6bLMvDwyTj2F8Kb3iX3yCVnnnM2g7353f/NIFxg+H74xYw5ZJpE++m0NHeCqF69iT/MeXrj4BSzj0N0WY2Xl7F26lOa//x3fuFKGLl1KYMqUw85fOw5b5swlNPMkApMns+/nt3H8Ky9jFRV152qINBH5+GNqHvyDO9pnPI6RmUnOZZeRMXcudf+3jIYXXwKtwTDIufRSCm+8IXVy3m5qpv5PT1Hz8CPEy8rck/dnnkHmWWfhHTly/0IcB52wwU4Q+fBDmtaspfnVVzEyMhiy9AdkzJ3bS2svjpUB2eQCsGbXGm746w387JSfcf7o8zudVmtN40svse/nt5GoqCB38WIG/b/vdtqFKrx+PTuuuJJhd9yB/8SxbDt/IUN+fCu5l13W3asi+hinpYW6P/6RxlWrSVRWkqisxGloQAUC5Fx8MRlz51C3fDmNf/4L2DZGMEjul79M9oUXULdsGTWPPY4yDIxQCCcScftna01g2jTyrrqKzPmnd/nEonYcUEqaRwaIARvojna45NlLAPjTwj916QNvNzVTefdd1D78CL6xYyn6z1/iGz26w2krfvmfVD/4ICf87TWMrCy2zp1HYPo0in/1q25dD3Hk4vsqqPyvu8k4dQ6ZZ53Z5bDTsRhNr/2NhuefI75nL0ZWJmZmFmZeLt6iIqyiIqLbtlPzhz9g19TgKy3FO3w4nsJCvMcNJ2vhQjy5ufvLsXs3zf/4Bxlz57Z7PLZzJzWPPoqOxzH8AYxAgIy5cwhMntzt74XoXwZsoAM898lzfP+173Pn3Dv5QskXuvy6prVr2f29JTiRCHlXfYXQyScTmDSpXZvoJ+edh6egkBF/eBCA3Uv+jaZVqxjz978dUTu86F7h9espu/GbJCorAQiefDKDv/9vGKEMwu+8TXj9BoxQEO+o0fhGj8JpaiLy8cdEP/yIptWrsevqMHNz8Y0di9PYiN3YSKKyMvUrR4DQqadScP3XCE6b1lurKQaoAXlStNWCkQt4+IOHue0ftzFr6CyyfV276EXGnDmMfPpp9vzgP6j+3X1U3/s7ME2CM2aQd9VVeEeWENv6CbmXL0q9JjT789Q//TSRDzYTmDihh9aoZzixGHZ19RGdTOsOWmvQmvju3bT84y1a3noLu76e4MyTCH3u83iLiwhv2OCO77FtO56CAqwhgzELCtzeHY4DgBEIYoRCxHbsoOL22/EMGcLI5X8ivH49lb++i+0XXJhaphEM4sTjcMBYIWZ+PsFZs8heuJCMU09pN1yE1hq7ro54WTlGwJ82AzmJgaXf19ABNldv5soXruS8Uefxk1N+csSvtxsakqHyNvXPPktizx637bO5mdEvv4y32D0JmqiqYsspp1L47W9T8C/Xdfdq9Ijotm3UPfFH6p9+Gruujvxrr6Xw2zcddmjQrop89DFmRqjdieLotu3sXbqUlrffdk8StmFmZ2PkZBPf+Wn7GRkGVnExdk0NTlNTp8sMff5zFP3nf6bGxrbr6qhd9n8YWZkEp01ze3NoTezTXcS2fYIRCrmDPeXnd8s6C9GTBnSTS6u73rmL+9+7n3vPuJfZRbM/83x0PE7Dn/9MzUMPY+Zkc9x997V7ftsFF6I8HrIXnk/4vU0k9u3DP66UwJQp+MeNwwiFUJaFCgQw2ow21x10IgFad2kgskRtLft+/nMann0OPB4yTz8dIxCg/plnyDjtNIbd8YtD/opWJxIkqmtIVLnNEE4kio5FMTMz8QwejJmXR9Oq1dT87yNENmwEwyBz/nzyrvoK4Q0bqLzrblQgQO5ll6J8fveXh7k5BGechG/M8SjDIL5nD82vv0F8924CkycTmDoVM8Nt7rKbmlKX+8IwQGuccNjt4mfbBKZOlV8qin5LAh2I2lEue+4yIokIyy9YTsjqmcHnK375S6p/fz8AnsGD8QweTPSjj9BtLhzbyszPxyouwltUjDVsKJ7BQ/AMGYwnvwBPfh5mdjbxfRVEt24htn0HZlYm3tGj8Y0ejTLN1Eh2kfffp/mNN90ar23jnziB4NSp7qWzcnLcWm9mFkYohBEK0fzqWvb++CfYDQ3kX3MNeV/+Ep6CAvcyXI89xr6f/RxryBA8gwfjNDZgNza5AzBpDYkEdn39QTXrjnhLSsj94hdJVFZS+8QTOPX1AGTMn8+QHy7FGjSoe998IQYACfSk9RXr+cqLX+Hzwz7PXaffhc/s/lHd7KZmwhvW4xszJhVYOh4n8uFHRD/+GCcaQcdi6HCY+O7dqQH2E3v3HjT+85Hwjh5N6OSZKMuiZf16Ih9sPqiNuC3/+PEM/dlP8Y8de9Bzza+/TtU9vwHLg5mZhZGR4Z7kNQwwFJ7cPDyDCjHz8zEzMlA+P8qysBvqSeyrIFFZiX9cKaHZs1NNN05LCw0vvoiZk0PG6adLFzshPiMJ9DaWb1nOD//+wx4N9c9COw52bS3xvXuxa2pIVFdj19a5g/kffzzekhKcpiaiWz8h+slW0BpPfj5mbh7ekSUH1XadSIT47j04DfXY9fXYDY04zc04zU2YOblkX7BQmiWESENHHehKqQXAXYAJ3K+1vu2A528G/hlIAJXAP2mtd3Y2z94KdHBDfenflzK7aDZ3ndZ3Ql0IIQ6ns0A/bFcGpZQJ3AOcDYwDrlRKjTtgsneBGVrrScCTwC+Orsg966IxF/Gjz/+Iv5X/jRteuYGWeEtvF0kIIY5aV/qmzQS2aq23aa1jwDLggrYTaK1Xaa1bU/ENoLh7i9n9Lh5zMT895ae8tfctrv3LtdRH63u7SEIIcVS6EuhFwK4298uSjx3KNcCLHT2hlLpOKbVOKbWuMvkrvt60cPRCfjnvl2yu3sxXX/oqFS0VvV0kIYT4zLrn1yNJSqkvATOAOzp6Xmt9n9Z6htZ6RmE3XgLuaMw/bj6/OeM3lDeVc/GzF7Ni2wp660SxEEIcja4EejnQ9qKDxcnH2lFKnQH8O7BQa31wp+s+bNbQWSw7bxkjMkfwvVe/x02rbqIqXNXbxRJCiCPSlUB/CxijlBqplPICVwDPtp1AKTUV+B1umKdlu8Wo7FE8fPbD3Dz9Zl4rf42Ln7mYV3a+0tvFEkKILjtsoGutE8ANwEpgM/CE1vp9pdStSqmFycnuADKAPyql1iulnj3E7Po00zC5esLVPHH+EwwJDeGm1Tfx76/9O42xxt4umhBCHNaA+2FRV8XtOPduvJf737ufPH8e35z6TRaOXohpyLC4Qojec1T90Acqy7S4ceqNPHrOoxRlFLH070tZ9Pwi/lb+NzlpKoTokyTQD2NCwQQeOfsR7phzB42xRr728te4/PnLWbFtBQkn0dvFE0KIFGlyOQIxO8YL217gwfcfZHv9doaFhvGV8V/houMvImgFe7t4QogBQAbn6maOdlizaw0Pvv8g71a8S44vh8vHXs6lYy5laMaxveKPEGJgkUDvQe9WvMsDmx5gza41AHy+6PNcMuYSTik6hYAn0MulE0L0NxLox0B5UznLtyxn+dblVLRUEPAEOKXoFM4acRZziudIk4wQoltIoB9DCSfBun3reHnny7zy6StUhasIeALMKZ7DgpIFfH7Y5yXchRCfmQR6L7Edm3cr3uWlHS/xl51/oSZSg8fwMH3QdE4pOoXPDfscY3LHYCjpbCSE6BoJ9D4g4SR4t+JdXi17lVfLX2Vr3VYAcn25zBw6k5MGn8SMITMYlT1KLs8mhDgkCfQ+aG/zXt7c8yb/2PsP3tjzRmro3lxfLhMLJzIhfwLjC8YzZdAUsrxZvVxaIURfIYHex2mtKWssY92+dazbt473q95nW/02NBpDGUzIn8DJQ09myqApnJh3IoWBQqnFCzFASaCnoeZ4Mx9Uf+DW4He/wXtV72FrG4A8fx6leaWMzRtLaV4p4/PHU5xZLCEvxAAggd4PNMeb+ajmIzbXbObDmg/5sOZDttZuJaHd4Qfy/HlMKpzEuLxxjMgawYisEZRklxCyQr1cciFEd+os0D3HujDiswlZIaYNnsa0wdNSj8XsGFvrtrKpahMbKjewsXIja3atQbN/Jz0sNIzjc49nTM4YTsg9gbF5YxmRNQKPIZteiP5Gauj9TNSOsqthFzsadrC9fjtb6rawtW4r2+u3pwYT85t+SvNLmVQwifEF4zku6ziGZw6Xk69CpAGpoQ8gPtPH8bnHc3zu8e0ej9txtjdsTzXbvFf5Ho9/+DgxJ5aaJseXw/DM4RRnFjM8cziDg4MZFBzEoOAgijOLJfCF6OOkhj6Axe042+q3UdZYxq7GXXza+Cm7Gnexq3EXe5r34Gin3fStgT80NNS9ZQxlWGgYxZnFFGUUyS9ghTgGpIYuOmSZFmPzxjI2b+xBzyWcBNXhairDlext3ktZY1kq8D+u/Zg1ZWuI2u2vBZ7pzaQgUEBBoIChoaGMyBrBcVnHMSQ4hGxfNjm+HLJ92fLLWCF6iNTQxWeitaY2Wkt5YznlTeWUNZVR2VJJVbiKqnAVZU1lqR9LteU3/RyXdRwlWSUUZRSlgj7Hl0OWLyt1P9efi2VYvbBmQvRtUkMX3U4pRZ4/jzx/HhMLJ3Y4TUu8hV2Nu6gMV1IXraM+Ws/upt3saNjB5prNrN61ul0b/oFyfDmp2v6wjGEMCg4i6AkS8AQIWSGyfFlu+PtyKQwWSs8dMeDJN0D0mKAVdJt0OLhJB9xafjgRpj5aT120joZYA/XRemojtVRHqqkOV1MRrmBv8142Vm2kPlp/yGWZymRQcBBDQ0MZEhrCkNAQBgcHk+HNwG/68Xv85PpyKQgUkB/Ix2t6e2q1heg1Euii1yilCFpBglawS1d6itkxWuIthBNhmuPN1MfqqYvUUROtYU/THvY072F30242VG7gzzv/3Ok1Xw1lYCoTU5lk+7IZEhrC0NBQsn3ZeE0vXsNLpjcz1csnx5eDz/ThM30ErSBZ3iz5Za7ocyTQRdrwml68ppcccg47raMdaiI1tMRbiNgRwokwdZG6VBt/1I5iaxvbsamN1rKveR8fVH9AU7yJqB0lakc73SF4lIdcfy45/hxCnhAhK+TunDzuDipkhcj2ZpPtc295/jzyA/nk+nIxDROFwlCGHCmIbiWBLvolQxkUBArgKK4C2BJvoTJcSUVLBfXReiJ2hJgdozHWSG2klppIDXXROloSLTTEGtjTvIdwIkxLooXmWHNqWIbO+Ewf2b5scn25ZPmyyPK6t4AngM/04TW9B/21DAuv6SXoCZLlyyLTm0mGlUHAEyDgCci5hAFMtrwQhxC0goyw3HFxjpTWmpZEC3XROrdZKFKT2gEknAQajaMdGmONqWkaYg3sbNhJQ6yBSMLdeUTtaLuhHLoi4Amkeg5lejNTzUsew0PICqVubY8ofKYPr+FN7WBau5gGPAH8Hn+7rqatPeOkyanvkUAXogcopVLBWZRR9Jnno7Um4SSI2lEidoS4HSdqR4k5McKJMI2xxtSt9eigMdaYOrncFG/C0Q5aa+JOnOZ4M03xJprjzQf9jqAzHsMDGmxto9EoVOpcQ5Yvi8JAIYXBQoKeIB7Ds/+m3L+WaeFR7l+f6SNkhVJHFZZhYZkWXsOL3+NPHZ20fY1HeWQH0gUS6EL0YUopLNMNvAwyunXeCSdBS6KFlnhLakcRtaOpXkd10ToidoRown1cKZU6kZzQidRr6qLuuYmtdVsJJ8LYjk3CSbg3nSDuxDs9H9EVCuWGvGGR0O68He3g9/hTRxkBTwC/6cfn8aFQ7pGNBp9n/w6kbbOV3+N3j1Q8QbymF6UUBgaG2n8zlZna2ViGhVIKhXK3i2GldlwKd2fTel6ktWfVsW7+6tLSlFILgLsAE7hfa33bAc/7gIeB6UA1sEhrvaN7iyqE6E4ew5Nqs+9pWut2O4HmeDPN8WZaEi0kHDf0Y3aMSMI9gR2xI6mdQutzMTtGzImlmo9MZRJOhFO9niKJCBE7QiQRSV0cBqChpYEd8R2pE94xO0bciff4OoP7HgfMQCrcW2/XT76es0ee3f3LO9wESikTuAc4EygD3lJKPau1/qDNZNcAtVrr45VSVwC3A4u6vbRCiLSklMJSFpZhEbSC5Ppze7U8WuvUjqUl3kLcibtNU8lzG452cHDcHYodJ+bE3HMfWqPR2LrNUUibow9HO6kjnZZEC9FENLWTaT1SSTgJsr3ZPbJeXamhzwS2aq23ASillgEXAG0D/QLgluT/TwL/rZRSurfGFRBCiE4opfB73GaR/EB+bxen23RllKQiYFeb+2XJxzqcRmudAOqBg94lpdR1Sql1Sql1lZWVn63EQgghOnRMh73TWt+ntZ6htZ5RWFh4LBcthBD9XlcCvRwY3uZ+cfKxDqdRSnmAbNyTo0IIIY6RrgT6W8AYpdRIpZQXuAJ49oBpngWuSv5/KfBXaT8XQohj67AnRbXWCaXUDcBK3G6LD2it31dK3Qqs01o/C/wP8IhSaitQgxv6QgghjqEu9UPXWq8AVhzw2NI2/0eAy7q3aEIIIY6EXAtMCCH6CQl0IYToJ3rtmqJKqUpg52d8eQFQ1Y3FSRcDcb0H4jrDwFzvgbjOcOTrPUJr3WG/714L9KOhlFp3qIuk9mcDcb0H4jrDwFzvgbjO0L3rLU0uQgjRT0igCyFEP5GugX5fbxeglwzE9R6I6wwDc70H4jpDN653WrahCyGEOFi61tCFEEIcQAJdCCH6ibQLdKXUAqXUR0qprUqpJb1dnp6glBqulFqllPpAKfW+UupbycfzlFJ/UUptSf7t3cu+9ACllKmUelcp9Xzy/kil1JvJ7f1/yQHi+hWlVI5S6kml1IdKqc1Kqc8NkG397eTne5NS6nGllL+/bW+l1ANKqQql1KY2j3W4bZXr7uS6b1RKTTvS5aVVoLe5HN7ZwDjgSqXUuN4tVY9IAN/RWo8DZgHfSK7nEuAVrfUY4JXk/f7mW8DmNvdvB36ltT4eqMW93GF/cxfwktb6RGAy7vr3622tlCoCvgnM0FpPwB34r/Xylf1pe/8BWHDAY4fatmcDY5K364DfHunC0irQaXM5PK11DGi9HF6/orXeo7V+J/l/I+4XvAh3XR9KTvYQcGHvlLBnKKWKgXOB+5P3FXA67mUNoX+uczYwB3fEUrTWMa11Hf18Wyd5gEDyGgpBYA/9bHtrrdfijkDb1qG27QXAw9r1BpCjlBp6JMtLt0DvyuXw+hWlVAkwFXgTGKy13pN8ai8wuJeK1VN+Dfw/wEnezwfqkpc1hP65vUcClcCDyaam+5VSIfr5ttZalwN3Ap/iBnk98Db9f3vDobftUedbugX6gKKUygCeAm7SWje0fS55AZF+0+dUKXUeUKG1fru3y3KMeYBpwG+11lOBZg5oXulv2xog2W58Ae4ObRgQ4uCmiX6vu7dtugV6Vy6H1y8opSzcMH9Ua/2n5MP7Wg/Bkn8reqt8PWA2sFAptQO3Ke103LblnOQhOfTP7V0GlGmt30zefxI34PvztgY4A/5/e3evEjEQzfLLHgAAAUhJREFURXH8P9WCndYWYmNruYWFYLevIGjhU4iVL+AbWFlYCKJi6UcvFqKioi6CNoKV9RbHYkZIE1iRGHI9PwiETZG5nHAhk1mGF0kfkkbAPvkZiJ431Gf76/7WtYY+znZ4nVfmjreBe0lblUvVrf5WgcO/HltTJK1LmpY0Q871TNIycE7e1hCC1Qwg6R14SynNlZ+WgDsCZ128Av2U0kR53r/rDp13UZftEbBSVrv0gc/K1Mx4JHXqAAbAIzAENtoeT0M1LpBfw66Bq3IMyHPKp8ATcAJMtT3WhupfBI7L+SxwATwDe0Cv7fE1UO88cFnyPgAm/0PWwCbwANwCO0AvWt7ALvkbwYj8NrZWly2QyKv4hsANeQXQj+7nv/6bmQXRtSkXMzOr4YZuZhaEG7qZWRBu6GZmQbihm5kF4YZuZhaEG7qZWRBfw58Gk810nskAAAAASUVORK5CYII=\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"TutiUGFIm81Q"},"source":["Training and validation accuracy improve instantaneously, but reach a plateau after around 30 epochs"]},{"cell_type":"code","metadata":{"id":"koMrvUyJm81R","executionInfo":{"status":"ok","timestamp":1604685184602,"user_tz":420,"elapsed":552520,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"d28b3257-f557-4ca2-8644-96cb7ec19ed0","colab":{"base_uri":"https://localhost:8080/"}},"source":["results = model.evaluate(X_test, y_test)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["313/313 [==============================] - 0s 1ms/step - loss: 0.3152 - accuracy: 0.9488\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"p7YpK2Pvm81T","executionInfo":{"status":"ok","timestamp":1604685184603,"user_tz":420,"elapsed":552515,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"76391ac6-eaf0-48c5-ccba-13c2769ad449","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Test accuracy: ', results[1])"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Test accuracy:  0.9488000273704529\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"fcqGvLFsm81V"},"source":["## 3. Optimizers\n","- Many variants of SGD are proposed and employed nowadays\n","- One of the most popular ones are Adam (Adaptive Moment Estimation)\n","- Doc: https://keras.io/optimizers/"]},{"cell_type":"markdown","metadata":{"id":"94XG6lbrm81V"},"source":["<img src=\"http://cs231n.github.io/assets/nn3/opt2.gif\" style=\"width: 400px\"/>\n","<br><center> **Relative convergence speed of different optimizers** </center></br>"]},{"cell_type":"code","metadata":{"id":"Po9y2epXm81W"},"source":["def mlp_model():\n","    model = Sequential()\n","    \n","    model.add(Dense(50, input_shape = (784, )))\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(50))\n","    model.add(Activation('sigmoid'))  \n","    model.add(Dense(50))\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(50))\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(10))\n","    model.add(Activation('softmax'))\n","    \n","    adam = optimizers.Adam(lr = 0.001)                     # use Adam optimizer\n","    model.compile(optimizer = adam, loss = 'categorical_crossentropy', metrics = ['accuracy'])\n","    \n","    return model"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Fzn5iX2Um81Y"},"source":["model = mlp_model()\n","history = model.fit(X_train, y_train, validation_split = 0.3, epochs = 100, verbose = 0)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1greMMf5m81a","executionInfo":{"status":"ok","timestamp":1604685416949,"user_tz":420,"elapsed":784848,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"0f66a1bb-5b29-47c8-bf25-75f64fe96045","colab":{"base_uri":"https://localhost:8080/","height":265}},"source":["plt.plot(history.history['accuracy'])\n","plt.plot(history.history['val_accuracy'])\n","plt.plot(history.history['loss'])\n","plt.plot(history.history['val_loss'])\n","plt.legend(['train acc', 'valid acc', 'train loss', 'valid loss'], loc = 'upper left')\n","plt.show()"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxU1f3/8deZfcky2chCAgFECFlYAoiggFUsilJFES3aaqu41lpbK/ptLW4t9mt/tVqXL7bWXau4V5RqC4ILsu8IBAhkIZA9mcw+c35/DEaWABESw4TP8/HIw8zcO+d+JsH3nJx77rlKa40QQojYZ+jqAoQQQnQMCXQhhOgmJNCFEKKbkEAXQohuQgJdCCG6CVNXHTg1NVXn5uZ21eGFECImrVixokZrndbWti4L9NzcXJYvX95VhxdCiJiklNp5uG0y5CKEEN2EBLoQQnQTEuhCCNFNdNkYeluCwSDl5eX4fL6uLiWm2Ww2srOzMZvNXV2KEOI7dEIFenl5OfHx8eTm5qKU6upyYpLWmtraWsrLy+nTp09XlyOE+A6dUEMuPp+PlJQUCfPjoJQiJSVF/soR4iR0QgU6IGHeAeRnKMTJ6YQL9KPxhXzs8ewhFAl1dSlCCHFCiblA94f91HhqOiXQGxoaeOKJJ47pteeffz4NDQ0dXJEQQrRfzAW6QUVL7owbcxwp0EOhI3+AzJs3D5fL1eE1CSFEe8VcoCui48MRIh3e9syZM9m2bRtDhgzhjjvuYOHChZx55plMnjyZQYMGAXDRRRdRXFxMfn4+c+bMaX1tbm4uNTU1lJaWkpeXx3XXXUd+fj7nnnsuXq/3kGO99957nHbaaQwdOpRzzjmHPXv2AOB2u7nmmmsoLCykqKiIN954A4APP/yQYcOGMXjwYM4+++wOf+9CiNh3Qk1b3N+9721gY2XTIc9HdARfyIvVVI9RGb9Vm4OyEvjdhfmH3T579mzWr1/P6tWrAVi4cCErV65k/fr1rVMAn3nmGZKTk/F6vYwYMYJLLrmElJSUA9rZunUrr7zyCk8//TSXXXYZb7zxBldeeeUB+5xxxhksWbIEpRR/+9vf+OMf/8if/vQn7r//fhITE1m3bh0A9fX1VFdXc91117Fo0SL69OlDXV3dt3rfQoiTwwkb6CeKkSNHHjCf+9FHH+Wtt94CoKysjK1btx4S6H369GHIkCEAFBcXU1paeki75eXlTJs2jd27dxMIBFqP8fHHH/Pqq6+27peUlMR7773H2LFjW/dJTk7u0PcohOgeTthAP1xP2hfysa1hG9nx2SRaEzu9DqfT2fr9woUL+fjjj/niiy9wOByMHz++zfneVqu19Xuj0djmkMvPfvYzbr/9diZPnszChQuZNWtWp9QvhDh5HHUMXSn1jFJqr1Jq/WG2T1dKrVVKrVNKfa6UGtzxZX6jM0+KxsfH09zcfNjtjY2NJCUl4XA4+Oqrr1iyZMkxH6uxsZGePXsC8Nxzz7U+P2HCBB5//PHWx/X19YwaNYpFixaxY8cOABlyEUK0qT0nRZ8FJh5h+w5gnNa6ELgfmHOEfY9bZ54UTUlJYcyYMRQUFHDHHXccsn3ixImEQiHy8vKYOXMmo0aNOuZjzZo1i6lTp1JcXExqamrr87/5zW+or6+noKCAwYMHs2DBAtLS0pgzZw5Tpkxh8ODBTJs27ZiPK4TovlR7erpKqVzgX1rrgqPslwSs11r3PFqbw4cP1wff4GLTpk3k5eUd8XWhSIjNdZvJcGaQYk854r4ns/b8LIUQsUcptUJrPbytbR09bfGnwAdHKGSGUmq5Ump5dXX1MR3g6yGXiO74HroQQsSyDgt0pdRZRAP9zsPto7Weo7UerrUenpbW5i3xjn6cfUMumo4fQxdCiFjWIbNclFJFwN+A87TWtR3R5hGOhVKqU06KCiFELDvuHrpSqhfwJnCV1nrL8ZfUjmOiOuWkqBBCxLKj9tCVUq8A44FUpVQ58DvADKC1fgq4B0gBnti3bGvocAP2HUV66EIIcaijBrrW+oqjbL8WuLbDKmoHAwYJdCGEOEjMLc4F0R76iTLkEhcXB0BlZSWXXnppm/uMHz+eg6doCiFER4vJQDeoE6+HnpWVxdy5c7u6DCHESSwmA12hOmXa4syZMw+47H7WrFk8/PDDuN1uzj77bIYNG0ZhYSHvvPPOIa8tLS2loCB63ZXX6+Xyyy8nLy+Piy++uM21XADuu+8+RowYQUFBATNmzGj9kCopKeGcc85h8ODBDBs2jG3btgHw0EMPUVhYyODBg5k5c2ZHv30hRIw7YRfn4oOZULWuzU2Z4X0BabR/uzYzCuG82YfdPG3aNG677TZuvvlmAF577TXmz5+PzWbjrbfeIiEhgZqaGkaNGsXkyZMPe+/OJ598EofDwaZNm1i7di3Dhg1rc79bbrmFe+65B4CrrrqKf/3rX1x44YVMnz6dmTNncvHFF+Pz+YhEInzwwQe88847fPnllzgcDlnPRQhxiBM30I9A0TmzXIYOHcrevXuprKykurqapKQkcnJyCAaD3H333SxatAiDwUBFRQV79uwhIyOjzXYWLVrErbfeCkBRURFFRUVt7rdgwQL++Mc/4vF4qKurIz8/n/Hjx1NRUcHFF18MgM1mA6LL6l5zzTU4HA5AltAVQhzqxA30I/Sk9zbtIhAJcIrrlA4/7NSpU5k7dy5VVVWti2C99NJLVFdXs2LFCsxmM7m5uW0um/tt+Hw+brrpJpYvX05OTg6zZs067jaFECe3mBxD78yTotOmTePVV19l7ty5TJ06FYguddujRw/MZjMLFixg586dR2xj7NixvPzyywCsX7+etWvXHrLP1+GdmpqK2+1uPaEaHx9PdnY2b7/9NgB+vx+Px8OECRP4xz/+gcfjAWQJXSHEoWIy0DtryAUgPz+f5uZmevbsSWZmJgDTp09n+fLlFBYW8vzzzzNw4MAjtnHjjTfidrvJy8vjnnvuobi4+JB9XC4X1113HQUFBXz/+99nxIgRrdteeOEFHn30UYqKihg9ejRVVVVMnDiRyZMnM3z4cIYMGcLDDz/csW9cCBHz2rV8bmc41uVzASrdlTQFmhiYfORgPZnJ8rlCdE/f5fK53wm59F8IIQ4Vk4FuwCDL5wohxEFiMtC/7qFLL10IIb4Rk4Eudy0SQohDxWSgy12LhBDiULEZ6PsuuZchFyGE+EZMBrphX9kd3UNvaGjgiSeeOKbXnn/++TQ0NLR7/68X/hJCiI4Sk4H+dQ+9o8fQjxTooVDoiK+dN28eLperQ+sRQohvIyYD/euToh095DJz5ky2bdvGkCFDuOOOO1i4cCFnnnkmkydPZtCgQQBcdNFFFBcXk5+fz5w5c1pfm5ubS01NDaWlpeTl5XHdddeRn5/Pueeee9jlc7+2evVqRo0aRVFRERdffDH19fUAPProowwaNIiioiIuv/xyAD755BOGDBnCkCFDGDp0KM3NzR36MxBCxK4TdnGuh5Y+xFd1X7W5LazD+EI+bCYbRmVsd5sDkwdy58g7D7t99uzZrF+/ntWrVwOwcOFCVq5cyfr16+nTpw8AzzzzDMnJyXi9XkaMGMEll1xCSkrKAe1s3bqVV155haeffprLLruMN954gyuvvPKwx/3Rj37EY489xrhx47jnnnu49957eeSRR5g9ezY7duzAarW2Duc8/PDDPP7444wZMwa32926GqMQQsRkD/27NHLkyNYwh2ivefDgwYwaNYqysjK2bt16yGv69OnDkCFDACguLqa0tPSw7Tc2NtLQ0MC4ceMA+PGPf8yiRYuA6NK706dP58UXX8Rkin72jhkzhttvv51HH32UhoaG1ueFEOKETYMj9aS9QS/bG7fTK6EX8Zb4Tq3D6XS2fr9w4UI+/vhjvvjiCxwOB+PHj29zyVur1dr6vdFoPOqQy+G8//77LFq0iPfee48HH3yQdevWMXPmTCZNmsS8efMYM2YM8+fPP+piYUKIk0NM9tA766RofHz8EcekGxsbSUpKwuFw8NVXX7FkyZLjPmZiYiJJSUksXrwYiK60OG7cOCKRCGVlZZx11lk89NBDNDY24na72bZtG4WFhdx5552MGDGCr75qe1hKCHHyOWF76EfSWfPQU1JSGDNmDAUFBZx33nlMmjTpgO0TJ07kqaeeIi8vjwEDBjBq1KgOOe5zzz3HDTfcgMfjoW/fvvzjH/8gHA5z5ZVX0tjYiNaaW2+9FZfLxW9/+1sWLFiAwWAgPz+f8847r0NqEELEvphcPjcYDrKlfguZcZkk2+RWbG2R5XOF6J665fK5IFeKCiHE/mIz0GUtFyGEOERMBrqstiiEEIc6aqArpZ5RSu1VSq0/zHallHpUKVWilFqrlBrW8WUecsxOva+oEELEovb00J8FJh5h+3lA/31fM4Anj7+so1NKyZCLEELs56iBrrVeBNQdYZcfAM/rqCWASymV2VEFHo5SSoZchBBiPx0xht4TKNvvcfm+5w6hlJqhlFqulFpeXV19XAc9Ue4rGhcXB0BlZSWXXnppm/uMHz+eg6doHul5IYQ4Ft/pSVGt9Ryt9XCt9fC0tLTjauvr+4qeKLKyspg7d25XlyGEOIl1RKBXADn7Pc7e91yn6owhl5kzZ/L444+3Pv76JhRut5uzzz6bYcOGUVhYyDvvvHPIa0tLSykoKADA6/Vy+eWXk5eXx8UXX9yutVxeeeUVCgsLKSgo4M47o+vYhMNhrr76agoKCigsLOTPf/4z0PayukII0RGX/r8L3KKUehU4DWjUWu8+3karfv97/JsOv05JKOQjrGCnsf3Lx1rzBpJx992H3T5t2jRuu+02br75ZgBee+015s+fj81m46233iIhIYGamhpGjRrF5MmTWy9wOtiTTz6Jw+Fg06ZNrF27lmHDjjzxp7KykjvvvJMVK1aQlJTEueeey9tvv01OTg4VFRWsXx+dYPT1ErptLasrhBDtmbb4CvAFMEApVa6U+qlS6gal1A37dpkHbAdKgKeBmzqt2gMK6/gmhw4dyt69e6msrGTNmjUkJSWRk5OD1pq7776boqIizjnnHCoqKtizZ89h21m0aFHr+udFRUUUFRUd8bjLli1j/PjxpKWlYTKZmD59OosWLaJv375s376dn/3sZ3z44YckJCS0tnnwsrpCCHHUNNBaX3GU7Rq4ucMq2udIPWmAnU07CeswvRP7duhxp06dyty5c6mqqmLatGkAvPTSS1RXV7NixQrMZjO5ubltLpvb0ZKSklizZg3z58/nqaee4rXXXuOZZ55pc1ldCXYhRExeKQp02oVF06ZN49VXX2Xu3LlMnToViC6b26NHD8xmMwsWLGDnzp1HbGPs2LG8/PLLAKxfv561a9cecf+RI0fyySefUFNTQzgc5pVXXmHcuHHU1NQQiUS45JJLeOCBB1i5cuVhl9UVQoiY7dZ11jz0/Px8mpub6dmzJ5mZ0en006dP58ILL6SwsJDhw4cf9YYSN954I9dccw15eXnk5eVRXFx8xP0zMzOZPXs2Z511FlprJk2axA9+8APWrFnDNddcQyQSfZ9/+MMfDrusrhCxSGuNPxTBH4qAhgS7qc1zU1prqt1+Kht8hMLR/x+UgiSHhcxEO3aLkVA4QlWTj4p6LyajgbQ4KylxFkJhzd5mH9XNfmpbAjR4gzS0BAiEI5gMBkxGRYLdzID0eAZkxJNoN+MJhKhrCeANhHFYTTgtRsxGA43eIPWeAM2+EJH9OpRGpTAZo3VXNwfY0+Rjb7MPg1LYzEasJgO1LQF2N3ipbPBx4eBMrjo9t8N/njG5fC5ARXMFLcEWTk0+tTPKi3myfG7sCoYjBEIRjAaFyaAIhCM0eUM0+YJ4A2FCEU1Ea/b/XzfJYeaUHnFthqHbH2J5aR0bKpvwBcP4QxHCEU2c1YTLYSbRbsZhMWIzG7GYDDR6gtS4/VQ3edlZ76O0poWddR56uuyM7pfC6H6pJNhNNHiC1HuC1Lr97G32U93sJxiOYLcYsZuNGA0KfzCCPxQmosFqMmA1GwiEIuys9VBdU4PLXcJWnU0zDgAcFiM9XXbSE2wEwxF8wTDN/hCVDV58wcN34OJtJjyBMOFI+/NMKWgr/qwmQ/QDph1s+Omjqtile9CC/YBtRoMiHNGYCOGihVxjNUMcNeRZq0nLG8vYST9sd60H1n345XNjuod+IlxYJMTXwhGNIhoUoYhmZ62HbdVuKuq9FGUnMrRXEkZDNHDL6z0s2bidKq+JlhB4/CHK671sq3ZTVu9tI5g08XgJYMKPpc3jZyTYGHdqGoOyEtjb7KOywUfJnma27q7Bqb3EKw8JykuS0YtDBdkTdLAXF7U6gQBmwhiwEeAsw2omGz/nMsNqGlUCpfY8ansUsNNnZ+cXHj74PEISbjJVLZmqDhdhcpSNiMVJkyGJHZEMtkbSaYo4SDV5SDZ4UEpRGklnRziNNEMD11k+4tzwh9isHgAaHL2pd/ajOWzGHYjQUqMIGuyETXYw20jL8pNqbCFBeQnEZ9PiGkhLQj/89btR1ZtwNJZgNoLFkYDVkUAEA95ACG8ghCPcSFKomjj/HpQ1nkjWMMy9hmN2ZRMJeIkEPDQ31lJfXYGnbjf4GrEZItgMIYwGAy3mZJqMSXiN8ThNGocxQly4kfiaVThr12PQITQKb3xvWhL7YyOILdiAyVcL3gZUYL+7oAWBkBHsOW3+Do9XTAe6XPrfvXgCIUprPOxt9mE3G3FaTRiUoqrJS0W9l9qWABkJNnqnOOmV4iDeZsJuNmJQipK9btaUNbBxdxNp8VYGZSYwKCuBHvHW1l5rjdvPB+t28/663bT4w2Qk2shMtJEaZyXJYcblsOCwGFvrcVhM9EtzkravDbc/RMleN1v3NLO9poXt1W521Xlp9ET/jPcEwhiIkIibRNVCUJvwYSGIkWxVQ4G9hjOTGkhq3EhusIRLVQ1ubWONPoV1hoH0s1n4iaWRrB512LUHQySIioSwhZpwBOswRgKEjTbq00+nPvtsQvYUEvYsI3Hvl2hfEyvNw3hlfT7PrUjme8Y1XGVZRUFkMxZrsO0fuLXtpwHCzh6ogVfRI9BMj/JlUPVpdMN+iRG2JBCOy8BotmEM7oWgB1qqIRL6ZqfQge2iDBAGvAoKpkDehVCzBVflalzVm0EHwRgBFYy21+wGHQazExzJYHFCyacQOujajsReoExQ1wK7971GGQAF9iRIyIK0IvDWw6Y3YM2zQPQkogFI2veFzQV2FygrYIFwGJrWQksN7N+BNFqhZzEMvBXS81F1O3DsXo2jZku0xvg0SB8YrdmeFG3X1QtS+oGrN5ja/lA+XidcoGutDzu/e38nyqX/J6KOGkbTWuMNhmmqr8YQDmIyGlBGE2tq4PPt9Xy5o44e8VbGD0hj/IAe2EwGdlbVEFnzGm6fn3VxZ7JHJxDY9+erNewl2WEgv28vhvVOwmwwsHDLXj7ZsIuG0jWYW6rIVLU48bFTp1Oie1KuU7ESxKl8mAmxU6cTOuif7dd/OtvwM868GR3ZjVHVUa/qcakWEg0B4g0+zKEWJiovU5UXrzGeksberAtlsy2YylbtoAknAUzYCGAjQAATu3Q6DZZMEqwGctyrGW3YyKmqnBRlYIzFgsNiwmVsJsHRgNPSiC3UhDrcv8sIUAu7jT3xZBRT02cYScE9jClfypg9b4BPgzENnD2jIWC0gMEMtgRwpkFcD4wNZaRu+YDUpQuibRqtkD0CUnsxbsdHjOOdb4I6dRD0vQ6cqWBNAGt89L+2BDDZwVsH7j3REA4HIRIGNPQegzH3DDB88+GGpw4CLd88trswWuPZb4+ocBAadkFtCQTc34RZJAx126NfAMOugsTs9vwjjL7WuN/vPBKGuh1QsxniMiBtAFjjjt5W6+sjULs1GtIWB5gdYImL/owPF7ThEPibor8TkxUMpug/vBPMCTWGvmPHDuLj40lJSTlqqO/17KXaU82glEHt+gCIZVprWgJhQuEIBqVQCgxKYVAKowFMBgMGwzd3caqtraW5uZk+ffoc0pbb3UzluoV85u3D0ko/JXvdOCxGEuxm0gzNuNzbcXl2kO4vpVdoJ6eoMtJU0wFttGgrJTqbWmdfNgUzWe1JZZfuwdmGVfzE9AGp+/aPaMVKlUedIYVTI9vpRSUGNJsj2SyLDKCROEYYvmKoYRvmQ7pybYuYbDQnFbI7Lo8mg4sWHAS1YpB3BVnVizHs67lFDBZarKm0GOLxYKdFWzHZE8lM74HLlQTuati7Aao3Qzhw9ONiwECEsMFCKGUAFqMB9XVP1JG87ys1Gp6OFLAlRsMt5Iu2n9Az2jtL7hvtwR3M746GhLkdF8ppDdVfgbcBsoZ+85qgD0oXQ2MZ9D0Lkg/9/YvYd6Qx9BMq0IPBIOXl5e2a490caKY50EymM7NbBLrWtJ4AMgA2YwR7xIOOhAlGFCEUYQyEtJEQRsIY0PuurlKA2aiwmAwopdjTEmbOikZ21ftJjbeSFmfFbFKU7W3gAe8DjDWuw6/NrDQWsidxMBn+HfTxbSI98s3FUn6DnVp7HxoT+uNLPIWQ0U44HEaHQ/RSe8nw78BY8xW07D3gfdRmjiV4+m0kpfTAuuU92PhONKwyB0PWEIJa4dm6GHvVckxhH97UAhynjkfljARXDiRkg9kOdduiYdtYHg1AS1y0R7R7DZQvj/437P/mwM4ekHdB9E/49MJoqBraMSs3HARPLfgao1/hQLT3arJGw7i+NPoVDkLv0ZAzMlqfEF0kZgL923hh4wv8cdkf+fTyT0m0JnZgZZ0vHNE07lyLt3oH/oY9uOv3sGCHl3VNdhITXVwY+pixgUV4sFKme9DD5CFRuTGFv/mgixjMeO2ZuG2ZNEWs+DzNhH1ujDqI2WTGYjHTYsvkncQrWRfIJBAM8jv/wwxp/oSS/FvJsvpw7PgI6ndEe4/ZIyB7OPTIg9QB0efaE4jehmj41m6P/umbeeSrYr/5IYSi4WlxHNsPUWsIesHfHB1rdfU6cIhAiG6qW85ysRqjA4X+/XtpJ5qGXVC5ih2Owby1NcB/v9pDuKGCnwX+zvnGpQfsWgRgAbyA2Yn/9FvZkH0lGZk9SUlxRgOspQZqtkDNZgz1pTgbynA2lpEerIM4J9qcQ8RgwkgkOs5YsYzCxoUw/CfR0Fv9CZz7IKeMviV6UK2j44K24/hAtLuiJ4d6Hnmu/SGMpgPHRb8tpaIfBsf6gSBENySBfjSeOtj1BexeC1VrwdcU7Yn2yENnFKF6FrcGU7MvyIvvfUR44zucw1IG6ugJoF5aMUIX0Dc+j/P0exjNEVb3uZHm7PFYXenEuXpwqktj8lZDSy1kDcXqTOG0/etQCuLSol+5Y9osVcGBJ6laamDhH2D5M9Gz/mf+Cr4O86/bPJ4wF0KcUGI+0APtOKF1THavhaVzYN3rEPKhlYEaay9qww567nyFeDwowGNwUpt2Oi3ObKzbP+LGfSsHl9rzed02g63mAUxybGR0/ccYG1+DUybA+f/LkLZOWCW146z/t+FMhUl/ghHXQdU6KGz7BhxCiO4hZgPdYoxOL+roHnrzjhW4/3U3mbVL8CsbCyzjec53OqtCvdBBO4OzXaQnWOlnayKtYS3x5Z8wvGol/fkP6y1FVIy4np6jLiM3IZPc/RvWGtx7Ia7Hdz/dqcfA6JcQoluL2UDvyB66LxjmizUbsHzye05v+pAgTh4xXMVnCZNwJKYwMM3J9aemMapvCjbz/oMa49H6Z2ypamZbfSNjBma3Th88hFIQn37ctQohxOHEfKD7wt9yGVu/G8wOwigWba3mvdWVfLVxDc9yD0nKzZKMK3B9/y5+3ieH29rRk1ZKMSAzgQGZCcfyNoQQosPEfKB/qx562TIiL05hryWHn/tv4sumJAbaGnjF8iBOg4IfL2R0VmEnVSyEEJ0rZgP9W4+hly0j/PxF7A7YifOV8rzhdraN/AV5Za+hPD64+j3IlDAXQsSumL3BRbunLWoNu5YQef4iKoJx3GL7PQ1XL8SaO4pBa2ej3HvgyrnRKxmFECKGxWwP/YhDLk2V8OYMqNkavaw7EqRCZXKDcRaPXzeJ3FQn9H4b1rwSnf3xbS+KEUKIE1DsBrpp30nR0EEnRb0N8OIl0FCGb8BktjRZ+KRC80ZwNI9fN5E+qfsWRjIYYOj077hqIYToPLEb6G310EN++OeV6JqtPJb1Bx5d0ZNQRFPYM5E/Tc4nP0uuihRCdF8xG+iHnBSNhOGtG6B0Ma9m38Oft2Xx0zG5TB2ew4CM+C6sVAghvhsxG+gmZcKgDNFA9zfDG9fClg/ZmP8r7loxkJvP6scd35erI4UQJ4+YDXSlFFajlYC3Fp45D/ZupG78bKYt6MvQXnHcdo7cPFoIcXKJ2UAHsBrM+Na9Dg3NhK/4Jz/92AHKzaOXD8VsjNkZmUIIcUxiOvUsWhMIeWH66ywIFbFqVwP3Ts4nJ1nWyBZCnHxiMtB1JILWGmskjN9gguwRvL9uN4l2MxcOzurq8oQQoku0K9CVUhOVUpuVUiVKqZltbO+llFqglFqllFqrlDq/40uNavrwQ74qKCSwoxRrKEjAnoAvDB9v3MP389NlqEUIcdI6avoppYzA48B5wCDgCqXUoIN2+w3wmtZ6KHA58ERHF/o1g9MJkQjhhnosQR8+azyLt9bQ7A8xqUh650KIk1d7urMjgRKt9XatdQB4FfjBQfto4Ov1YxOByo4r8UDGxOjFQeHd27FFggTMdt5fW4nLYWZ0v5TOOqwQQpzw2hPoPYGy/R6X73tuf7OAK5VS5cA84GdtNaSUmqGUWq6UWl5dXX0M5YLR5QIgvGsjFq3xmSx8vGkv3x+UIcMtQoiTWkcl4BXAs1rrbOB84AWl1CFta63naK2Ha62Hp6WlHdOBWgO9YhtWDXVBcPtDTCrKPI7yhRAi9rUn0CuAnP0eZ+97bn8/BV4D0Fp/AdiA1I4o8GCG+HgwGgnvLcNidlDn8+FymDldhluEECe59gT6MqC/UqqPUspC9KTnuwftsws4G0AplUc00I9tTOUolFIYExII1+zBYnXREvAyMV+GW4QQ4iN/zOgAACAASURBVKgpqLUOAbcA84FNRGezbFBK3aeUmrxvt18C1yml1gCvAFdrrXVnFW1MTCDc3ELI6EKrEN/Pz+isQwkhRMxo16X/Wut5RE927v/cPft9vxEY07GlHZ7RYSZcayBoSkap3fRKkStDhRAiJscpjJYIYb+BFkMKqBBp8dauLkkIIbpcbAa60Us4aKQlEo8yhIizGLu6JCGE6HKxGeg0EQ4Y8QWj5Qd1sIsrEkKIrhd7gR4OYQxVo4OagCd63rX1rkVCCHESi71Ar9uG0Ry9j6ihIfrfA+4rKoQQJ6nYC/SqdRgtkej3TdEglx66EELEYqD3+x7Gif8DgEkCXQghWsVeoDuSMRZOACDOGz0Z6g9JoAshROwFOt8s0BXnDwHSQxdCCIjxQI/3hQFoCjR1ZTlCCHFCiMlAVzYbEbOZeG902mKlu9PupyGEEDEjNgNdKYLOBOL8IUwGE5UtEuhCCBGTgQ7gs8fhCnrJcGSw2727q8sRQoguF7OB7rE6SA55yYrLYneLBLoQQsRsoDdZnCSEvGQ6M6WHLoQQxHCg15vsxPlbyIrLotpbTTAsC3QJIU5uMRvoNcqK3ecm05GBRlPVUtXVJQkhRJeKyUCPRDR7lQ1jOEymKRlAZroIIU56MRnoDd4gjebobecygk5A5qILIURMBnqN20/zvkBPCVgAZMhFCHHSi81Ab/bTZIkGuqG5hTR7mgy5CCFOejEZ6NVuP837Aj3c0EBmnExdFEKI2Az0Zj9NlujYeaihgSxnlvTQhRAnvZgM9Bp3gIA9GuiRxkYy4zKpaqkioiNdXJkQQnSdGA10P4kJDgxOZ3TIxZlJMBKkxlvT1aUJIUSXidlAT42zYnS5CO8bcgGZuiiEOLnFZKBXN/tJi7diTEwktO+kKMjURSHEya1dga6UmqiU2qyUKlFKzTzMPpcppTYqpTYopV7u2DIPFO2hWzC6XEQaGr/pocuJUSHEScx0tB2UUkbgcWACUA4sU0q9q7XeuN8+/YG7gDFa63qlVI/OKjgS0dS6A61DLsGKCuIsccRb4mXIRQhxUmtPD30kUKK13q61DgCvAj84aJ/rgMe11vUAWuu9HVvmNxq9QUIRvS/QEwk3NACQ5ZR10YUQJ7f2BHpPoGy/x+X7ntvfqcCpSqnPlFJLlFIT22pIKTVDKbVcKbW8urr6mAqucfsBSI3fd1K0qQkdiZAZlyk9dCHESa2jToqagP7AeOAK4GmllOvgnbTWc7TWw7XWw9PS0o7pQNX7Aj1t35ALWhNpaore6KJlN1rrY34TQggRy9oT6BVAzn6Ps/c9t79y4F2tdVBrvQPYQjTgO1x1875Aj7dgTEwEaJ262BJsoSnQ1BmHFUKIE157An0Z0F8p1UcpZQEuB949aJ+3ifbOUUqlEh2C2d6Bdbb6fn4G//3lOHolO6M9dCBUXy9TF4UQJ72jBrrWOgTcAswHNgGvaa03KKXuU0pN3rfbfKBWKbURWADcobWu7YyCbWYjfdPisJgMWPtH/wjwrllDz7josH55c3lnHFYIIU547RpD11rP01qfqrXup7V+cN9z92it3933vdZa3661HqS1LtRav9qZRX/NnJWFpW9fWj77nL6JfTEoAxvrNh79hUII0Q3F5JWi+3OOGYNn2TJsESP9XP3YULOhq0sSQoguEfOBHnfGGLTPh3fFCgpSCthQu0FmugghTkoxH+iOESNQZjPuTz+jILWABn8DFe6DJ+EIIUT3F/OBbnA4sBcX0/LZZ+Sn5AOwvnZ9F1clhBDfvZgPdADnmNH4N2+mbygJs8Es4+hCiJNStwj0uDPOAMC/ZCkDkgawvkZ66EKIk0+3CHTrgAEYU1Jo+fQz8lPz2Vi7UW5HJ4Q46XSLQFcGA87Ro2n5/HMKkgfhCXkobSzt6rKEEOI71S0CHaLTF8N1deTXOQA5MSqEOPl0m0B3jh4NgGvtLuwmu4yjCyFOOt0m0E1paVgHDMDz+RfkJefJTBchxEmn2wQ6RJcB8K5YweC4AXxV9xXBSLCrSxJCiO9MNwv00ehgkGG7bQQiAUrqS7q6JCGE+M50q0B3FBejrFayN0VX7p1fOr+LKxJCiO9Otwp0g82GY/hw1LI1TO43mWc3PMuGWhlLF0KcHLpVoEN0HD1Qso1f9rqaFFsKv/n0NwTDMpYuhOj+umWgAxiWreN3o39HSUMJT619qourEkKIztftAt16an+Maam0fPYZY7PHMrnfZP6+7u9sqt3U1aUJIUSn6naBrpQibvQYWr74Ah2J8OsRv8ZhcvDiphe7ujQhhOhU3S7QAZxnjCFcX0/Da69jr2rg7Jzv8d9d/8Uf9nd1aUII0WlMXV1AZ3COGYPB6aRq1iwArrBb2XlBkE/LP+Xs3md3bXFCCNFJumUP3ZScTP/Fi8h97Z9kPnA/5uRULlti4IPSD7q6NCGE6DTdMtBh363piopwXXopyT/8If3KQmxdtQBP0NPVpQkhRKfotoG+v8QfTEYbDZy+ysuCsgVdXY4QQnSKkyLQTampxI8fz1kbFP8umdfV5QghRKc4KQIdwHXJJSS4I7gXL6bR3whAYOdOwk1NXVyZEEJ0jJMm0OPOPBOd7GLsmhDzd3xI3XPPse38SVT++s6uLk0IITpEuwJdKTVRKbVZKVWilJp5hP0uUUpppdTwjiuxYyizmZSLplBcovHf9SB7/jAbU2oq7oULCZSWdnV5Qghx3I4a6EopI/A4cB4wCLhCKTWojf3igZ8DX3Z0kR3FdckUjBEYtinIwknZ5PzzZTCbqXv55a4uTQghjlt7eugjgRKt9XatdQB4FfhBG/vdDzwE+Dqwvg5l7deP9N/+hqrfX88TRVX8bfebJEycSOMbbxJ2t3R1eUIIcVzaE+g9gbL9Hpfve66VUmoYkKO1fv9IDSmlZiilliullldXV3/rYjtC8vTpnD3lNi465SLmrJ1D5XlDibS00Pj2211SjxBCdJTjPimqlDIA/w/45dH21VrP0VoP11oPT0tLO95DH5e7Rt5FbmIuv6p+CmNBHvUvvoiORLq0JiGEOB7tCfQKIGe/x9n7nvtaPFAALFRKlQKjgHdPxBOj+3OYHTwy/hF8IR9zB3sJlJbS8tlnXV2WEEIcs/YE+jKgv1Kqj1LKAlwOvPv1Rq11o9Y6VWudq7XOBZYAk7XWyzul4g7U19WX+8fcz2s9y/C67NT+7e9orbu6LCGEOCZHDXStdQi4BZgPbAJe01pvUErdp5Sa3NkFdrZzc8/lqqJreHW4H8+XX9L86eKuLkkIIY6J6qoe6fDhw/Xy5SdGJz4UCXHb/Fu49N5PCDtsOF94nFE9R3d1WUIIcQil1AqtdZtD2ifNlaJHYjKYeGzikxhmXElWpY9n/3Idv1jwC+p8dV1dmhBCtJsE+j5KKc68+i4sAwdw/RdxfFa6kCnvTGFR+aKuLk0IIdpFAn0/ymAg/Vd3YKtu4nn/lSTbk7n5Pzcze+lswpFwV5cnhBBHJIF+EOeY0TjHjIGnXuTZ3r/lyrwreWnTS9y1+C6CkWBXlyeEEIclgX4QpRRZD83GlJxM1S23cnvOj/hF8S/4oPQDbl9w+yE3mtZa8/729/n9l78nEA50UdVCCNFNbxJ9vEypqeT831OUXvFDym64kR+/9BJx5jgeWPIAV867kvP6nMeYrDH4w34eWvYQzevWkF2jeTJk5OdjZDleIUTXkGmLR+Be/CllN9yAbeBAXJdPY/lAI0/teImtdZtJboKiUs15a4zkVkSHYhocYL/mhxRddwcGm62LqxdCdEdHmrYogX4Ujf96n+rHHiW4cxeYTFiyswlUVkAgGuLmfn1JvvwKdG42nz70C04t8WFM70GfV17BnJXVxdULIbobCfTjpLXGt2EjTR/MI7irDHOvHCy9emMbOABbURFKKQBW713NQ/93FXfNhbiiwfR57nmUwdDaRrCsDEuvXl35VoQQMU4C/Tv011V/ZcsLT3LjvAjvT+qBb+oEfjroGtQjf6f+5VfI/P3vcU25uKvLFELEKAn075DWmmW7l+L/9X0krd7BfT+yceEXIYZvCmJITMRgt9Nv/ocYrNauLlUIEYPk0v/vkFKKkVmncfpfnseSmMy9z/kZvinIc2cbeO+neYSqqqh/SW55J4ToeBLoncSUkkLW7x/E6Eoi438f4pQbfsFL9tWs62ei8onH8DfWfyd16EiEmqefZteMGUQCMk9eiO5MAr0TxY0bR/9PF5N04WSuLbyWNye/yZopBZjcXp65ezJvl7xNU6DpgNeEqqvR4Y5ZZiDc3Ez5z26l+k//j5ZFi3F//HGHtCuEODHJGPp3TGvNyhumY/x8Na+dqQhajPRN6sfZ3j4YVqwnWFZG/IQJ9Hzkzyij8du1HQ4TrKwkWF5OoLycur/9nUBFBel3/Iq6557H3KsXvZ/9Rye9MyHEd+FIY+hypeh3TClF4d1/oHT6dK76by0QAb7CbdtC8uln4jz9dBpee42qe+8j495ZrVMiDyfi89Hy+Rc0/+dj3AsWEq77ZslfU3o6vZ97FkdxMRGvl+pH/kKgtBRLbm6nvkchRNeQQO8Clt696f/JJ0S8PrTfx4qyJVy/aiYTT3HxhzNmYXS5qJ0zh12mJtJnXE/K+nLcn0SX8bXl52PLzye0dw9N8z7AvWABEY8HQ1wctjNGo4cXkHZKEebsbMwZ6ShT9FecOGUK1X99nPrXXif913d05dsXQnQSCfQuooxGjHFOiHNyWsokblBlPL76cYrTixl6zYWsWfMWg1/+kKZ/fog3DIb4eDAYaHj99dY2jC4XCRdcQPy551LWP4EZi3/FHs8nTFPTuCntJiymb3695h49iD/rLBrfeou0236OwWLpircthOhEMoZ+gojoCDd9fBNLq5ai0cQbHPxudV821Gxgc148v53xMumJWQTLy/GsX4+Kd5Jw2ukos5n5pfP57We/JcGSwOis0byz7R3iLfFMPXUqNqONCBESLAmMr3TRdPOvyHr4YRIvmIQOhQjt3YspM/OoQztCiBODXFgUI+p8dVz/0fWcmnQqvxz+S5JtyWyo3cBP5/+UdEc6Pyn4CZ9VfManlZ/iDrhJsiWRbEumpKGEIWlD+PNZfybVnsqW+i38cekf+bLqywPaN2HgyaeNOBJTSS0qjo65NzRgHTgQ12VTSbzwQozx8d+qZq01/q1bafn0MxzFw7APHtyRPxIhxEEk0GPcsqpl3PDRDQQiAZJtyYzNHkumM5NqbzXVnmr6uvpyy5BbsBgPHEYJhAMopTAqI7uadvFWyVt4nnuFKf9uxmc34hh3JqkFxTTOm4d/4yaUzUbc2LEkTPw+cePGYXA6D1tTqL6e2jlP0zT/Q0KVu4HoEFCfd97GnJ7+TQ07d6JDIaz9+h3weh0Oo32+Ix5DCHEoCfRuYHPdZoKRIINSBmFQx375QCDg5YN/P8nD9a/jxscVA68gLzmPlJ2NJH68HOMnS9G19WA2Y4yPR1mtGOx27MOGkjBhAo4RI2h4402q//pXIm43cWPHEnfWeCy5uZRdfwP2/Hx6PfsPlMmEZ+VKymZcT8TrJeUn15B6880YbDbcixez5/d/IFRdTeb995Fw3nlt1updvZq6F14k5ac/wTZo0AHbglVVeJYtx7NiOf6SElyXXorroouO+ecCRC+8CgYP+JAJNzTQOG8e4fp6UmfMQJnNx3UMIY6XBLo4RI23hj+v+DPvbnv3gOdVRDOwHEaUmuhnyqCPLYuUgBXv0mVEWlrAYIBIBMfpo0i/6y5sp55KKBLi0ZWPkvt5KXlPfETqTTfhGDmSsptuwpyWhn3YMBrfegtLbi7m3r1o+WQRpt45RJwOIhs345o2jfS7Zh6whnzLkiWU3XQz2uMBo5HkH/2I1JtvxrN0KfUvvkjL558DYHA6MaamENy5i4TJF5Jxz++iJ5vbKeL14l60mOZ//xv3woVEPB4sffpgLywg4vXhXrAAHYwulewceybZjzyCweHogN+AEMdGAl0cVlOgiTpvHQ3+Bup99dT766nz1VHeXM7Huz6m0d+Iy+rCqS1kb65nwDYf9hHD+fH1jxNvjScQDvDrRb/mP7v+A8DP3ocz1oVQZjPW3r3p9cwzmNLSaPn8c3b/9h5CDQ3smjKSB3qvxhvy8tdtY0h4/T9Yevcm8ZJLSDj/PPwlJVTc+nMsvXvR85FHqHv2uejsHrMZgkFM6em4LptK3Pjx2AYMAKWoefIpap54AnNONgnnn4+5Rw9MPXpgysjAnJWF0eU64MRvuKGBupdeov75Fwg3NmJMSiL+nLMxpWfgW78e7/r1EImQeOEFJF50Ed6166i6915sBQXk/N9TKLOZQOlOIi0tOIYXf+uLwIQ4VhLo4pgEw0EWVyzmP7v+g0Lhsrrwh/28vuV1suKyuH/M/Ty97mk+q/iMmSNnclrGaTy/fA5j7vsXQaNm3i3FnDN4Cn0T+1LhrqCirpQPNr9LSaiS0zNPxxPysLZ6LQ+aLqXw3Y1416yJHthgwJaXR87fnsaUlASAZ/lyGt5+m7gzziD+7LPbHPrwLF/O7lmzCGzfAZHIAduU3Y4pLQ1TUhIGVyLeZcuJeDzEnXUWyT+6CseIEa1z9iF6shc44EOg+T//oeL2X4LW6P3WxTFnZ5P84x/jmnKxnBMQnU4CXXSoVXtX8etFv6aqpQqFYtboWUzpP6V1e0X9Tt4rncf7pfMobSo94LUDkwfy82E/54yeZ+AL+bhj0R0sLFvItAHTGG8aRM8vd2Lb20CPX/3qW8+4+ZoOhwnV1hLcs4fK7Wsp3bKM+tIt9Akn0SNgJVRXj7V/f1KuvRbbgFNbX9ccaCYUCZFkS2p9zhP08P6O96lqqeKS/peQVLKXhnfewZyZhSW3N4RC1L3wIt5VqzA4nTiGD8deXIwtL49g1W78W7cS2l0V/YvizDOP6f18GxGfj9CePVh69+70Y4muIYEuOlyjv5HHVj3GaZmnMaH3hDb30VqzsXYjtb5aesb1JCsuC7vJfsA+oUiI2Utn88/N/2x9zmq0YjfZsRqtuKwuJvaZyEWnXESqPZUKdwX/3PxPPir9CKUUceY44i3xjMwYyXl9zqNXQi9qvDW8ufVN3tz6JhXuCgDsJjvBSJC/n/t3hqUPaz3W9obtzNsxjy92f8H6mvVEdIQBSQMYlTmKkA7xTsk7uINuFNHZQpP6TuLawmvJTcw94H14Vq2i8c238KxYQWD79tbnld2OweEgXFtL4pQppM+8E2NCwoE/p1AI96LF0Q+E4mEH/KXwbQQrKym76Wb8W7aQ/pv/IfmHPzymdsSJ7bgDXSk1EfgLYAT+prWefdD224FrgRBQDfxEa73zSG1KoIv9Nfob2dawjZKGEsqay/CGvPjDfnY17WLl3pWYDCYGpQxiXfU6DMrAGT3PwGl24gl6qPZWs6F2AwD9Evuxs2knIR3itMzTOLf3uYzKHEWiNZHp86bTHGjm1UmvkuHMYO7Wucz+cjYhHaIgpYBRWaOwGq18uftLVu1dhUZzbu9zuWLgFWQ4M3h2w7O8seUNgpEgVw26ihsH34jDfOgJ0lBdHf4tWzH3zMLcsyc6GKTm8Seo/fvfMaWk4Lr0UpyjT8dWUEDzv/9NzRNPEigtBcCQmEjc2LGYs3uiAwG0P4A5M5O4752FtU+f6Lz/LVtonj+fsNtN/Pe+h2P4cHwbNlB28y1onw/boEF4li4l5bprSfvFL1pvgyi6h+MKdKWUEdgCTADKgWXAFVrrjfvtcxbwpdbao5S6ERivtZ52pHYl0EV7bW/YzutbXmdp1VLGZY/jsgGXkeHMOGCfqpYq5pfOZ1H5IgYmD2TqqVMP6UVvb9zO9Penkx2fTX9Xf97b/h6js0bz4BkPkmpPPWBfb8hLKBIi3nLgsE+tt5bHVj3GG1vfIMOZwZ0j7uR7vb7Xrqmku5d/yuZ7Z9JjWx0qoltnDFkHDiT1xhvBoHD/dwHuhdELvpTVijKbiTQ3A2Dp0weAwI4dYDCgzGa034/R5SLi9WLq0YOcJ5/AkptL1QMP0PDqP4k752wSJ03CNmgQ5pycA8Jdh8N4V67EvWgRgdKdBCrKCdc3kHbrrbguPnQKaLi5Ge/KlXhWr8Y2aBDx55xz2CuMdTgcPc8QiaC1JrR7N54VK/GsWAFak/nA/QfMahLtd7yBfjowS2v9/X2P7wLQWv/hMPsPBf6qtR5zpHYl0EVXWFy+mJv/czMANw65kRmFMzAavv0MlVV7V3HfF/dR0lBChjODC/pewITeE3CYHER0BJPBRE58Tmvg7WraxfUfXU+5uxyHT/NjfzEXtPTHVTwyGoz7B+1BJ2SDFRU0L1iIe+FC0Jr4CROIn3AOBocD9+LFNP/7IwDS/+fu1pPIWmtq//Y3ah59rHXapcHhwNyrF5acbAzOONyffkq4pgZlNmPOycGck024tg7fhg1k/e//knjBJAB8W7ZQdd99eFeshP3ywjl6NOm/+Q2mtFSa5n1A49tv49+2De31th7zYMbUVMK1td8sEb3vfXvXb6Bh7uuYUlKx9MrB0qcPtkGDjnn4CaLXFQR2lGJO74HR5frm5xsIUPvscwR27cQ+eDCOYcOw9OpFxB9A+7woq/WA8zfBykqq//IoLUuWEPe9s3Bdein2/Pxjrut4HW+gXwpM1Fpfu+/xVcBpWutbDrP/X4EqrfUDbWybAcwA6NWrV/HOnUcclRGiUywqX4TT7KQ4vfi42glGgny882Pe2/Yen1d+TlgfeGOSfon9mNJ/Cqcmn8qdi+4koiM8+r1HWVK5hDlr55BsS+bqgqs5p9c5ZMZlHlctX6vz1bG+Zj2FqYUk2ZLQgQD+khJ8Gzfi27yFYFkZgbIywnV1OEadRsK55xI3dmzr7JyI10vZ9TfgWbGCnn96mEBZGTWPPoYhPp6kK67AMWIEtoICGt96i+q//IWI348yGtE+H9b+p+AYeRoGhwNlt6HMZpTBCAYDRpcLx7ChmHv1ou7Z59j70EMk/+QnpP/6DhrmzqXqvvvBYED7/a0fGobEROLOPJO4ceOwntIPc8+eGJxOvGvX4l6wkJYlS1BmM6aUFEypKWit0R4vEY+HwM6d+Ldvh1AIg8NB0pVXknzN1QTLy9l99//g37oVQ0ICkaamQ3+ISmHLz8c5ejQ6HKL+hRcBcIw6Dc+XS9F+P7ZBg0i54XriJ0xo/eD1rltP07x5GJOTsPbti6VvXyw5Ocf1odSW7yzQlVJXArcA47TW/iO1Kz100Z3UeGtYunspESIYlZFGfyPvbXuPtTVrAchyZvHUhKfokxgdNtlQu4H7v7i/dey/IKWAywdezgV9LzimvxgA1lSv4fYFt7PXuxeAvOQ8xuWM4+r8q3Gav8XFVi0t7PrptXhXrwYgfsIEMu6dhSk5+YD9QtXV1Dz5FFpHcE2Zgq2goF2LvGmt2XP/A9S//DL24cV4l6/AOXo0WX96GIPTSbC8Av+WzbgXfoL7k08I139zu0ZlNkd7/0Yj9iFDUEYjodpawjU1YDBgsNtRdjvm7J7YBgzEeko/3As/oemDD1B2O9rnw5SWRsa9s4gbN47AjlK8q1b+//buPTiquwrg+PcseUFem82jhARISEITklSwxVJondb2D2ktdbSdkdYpnXHQP7RW7ejo6D86Ux0dR9v6GjuKj45jO7ZaGcaR2idqS6FACwECG0IgCYEkJNk8SPZ5/OPepgkJlELSNXfPZ2Yne+/e3fs7e5KTe3/3d+8l1t2NZGbhm59FrPcsI6+/7gyjjcfJ23AnJQ89RHpZGfHBQULbttH/xyeJtLWRuaIO/6c+zdD27ZzbvXv8XInx9mZkkFFV5fyzW73aOT4y4dIYl+MD6XIRkduAn+EU8+73apQVdJMKjvYfZUfHDjZUbaBkQcmU108MnuCFEy/wj+P/4Gj/UWoDtTx83cNU+6s52HuQg2cPktAEFfkVVOZVUppTSl5GHmm+yVt9zx59lkfeeISSBSV8ffXXaelv4fWu19l7Zu/4OQOrF66+5HbHh4Y48/0fkL12LXmfuGPGr8apsRgdX3qQ4VdeofALX6D4yw9Oe3KWxuOEjxwh0t5BtLOTWE8PWQ315Nx005TRQhcTDgY5u+V3+HJzKH7wwUsaEhsfHiYxNER66dS9J43FCG3bRu8vfkm0vZ200lIC99+P/567QZVIayvhY62EW1oIB4OMNR8m3tMLQGZdHYFN91/2pSqutKCn4RwUvRXoxDkoeq+qHpywzCrgGZwt+eClNMoKujHvSmiC7W3beXTPo5waOTU+/52DrQmdfKJUTnoOWWlZqCoJTdAf7ueG0hv40Ud/hD/r3f7ivWf28p3/fof2oXbuWX4P68rWUZVfRcmCElpDrRzsPUhwIMhobJRoPEpc41TkV3BN0TU0FjcSyJq8VQ6wv2c/oXCI4gXFFM0vojCr8LIKvkYiRDo7yXQP9s5FGo0yduQoWVcvv+h1fpzRSUGGdzh7HXnr1xO4777LWudMDFu8HXgUZ9jiFlV9RES+B7ypqltF5AWgEehy33JSVTdc7DOtoBszVTge5rngc4zFx2goaqAuUEeaL42TgydpG2yj+1w3oUiIwfAgo7FRfOJDEJbmLeXeununbLmDc3LUY3sf46kjT035xwCQm5FLXkYe6b50FKV9qH18ueuuuo6NtRu5ZcktNJ9t5vF9j7Oza+ek9weyAqxdtJa1i9ayIH0BTb1NNPU2EYlHaChqoKGogYXZC+kb66NvrA8fPq4vvZ7y3PIpbVFV9pzZw0vtL1Eyv4SK/AqW5S9jSd6SGfqG5z47scgYw3BkmOOh47SGWjk9cppl/mXUF9ZTmj35Bifnouc4dPYQe87s4W8tf6NzuJP8zHxC4RAFmQVsvmYzjUWN9Iz20H2um7d73mbnqZ30h52+7jRJo6aghvR56TSf8ugX/AAABthJREFUbSaSiEzbnoq8Cq4vvZ6leUspzyknHA/z5KEn2d+7nzRfGrFEbHzZm8pu4mvXfo3qgurxeQlNcGLwBE29TQQHgqwsXskti2+ZFMvpkdNkzsucdPbvxRzoOcCx0DFuLLtxylDWeCLOQHiAvrE+RmOjLMtfRk5GziV97kyygm6MuSzxRJx/d/6bba3bqPHX8NkVn532AGtCExzuO0w0HqU2UEtWmjPGPJqIEuwP0jfWRyArQCArwLnoOV479Rr/OfUf3up+i5HoyPjnlOeU80D9A9xVfRfheJgTgyfYdXoXWw5sYSQ2wh2Vd5A+L52W/haOhY6Nv9cnPhKaYHnBcjY3bmYkOsLWY1vZ270Xn/hYWbySmxffzJrSNVQXVJPum9w9cqTvCD/f93Ne6XgFAEFYVbKKxqJGTg6dpDXUSsdQx5SRTItzF1MbqGVR9iIWZi9kYfZC6grrWJS9aNbuAmYF3Rjzf0lVGQgP0DHUwWhslGuvunbaUT4DYwP8ev+vefrI0+Rm5FLtr6bKX0VdoI76onoq8irY3radJ/Y/MX79oMr8Su5cdieRRIRX21/lcN9hwLm0xNWBqynMKiQUDtEf7ud46Di56bk80PAA68rWsaN9B8+feJ62UBtL8pZQ5a9iad5S55jB/EIyfBm0DLTQ3NdMsD9I10gX4fi7A/uK5xfTWNRITkYOPvExT+ZR7a9m9cLV1BTUXNE9DaygG2M8IZaITXuc4B3xRJzXTr1GQVYB9YX1U7pf9nXvG+/jH4oO4c/048/0s7xgORtrN5KfmT/p8xKauKTiq6qEwiE6hjs40HuAt3ve5tDZQ4RjYRIkiMQj9I31AZCfmc/mxs1sqt90Wd+BFXRjjEmyruEudp/Zza6uXawrW8f6yunv1PVeLlbQZ/YUJmOMMdMqzSllQ84GNlRddADgFbHLsBljjEdYQTfGGI+wgm6MMR5hBd0YYzzCCroxxniEFXRjjPEIK+jGGOMRVtCNMcYjknamqIj0AJd7D7oioHcGmzNXpGLcqRgzpGbcqRgzvP+4l6pq8XQvJK2gXwkRefNCp756WSrGnYoxQ2rGnYoxw8zGbV0uxhjjEVbQjTHGI+ZqQX8i2Q1IklSMOxVjhtSMOxVjhhmMe072oRtjjJlqrm6hG2OMOY8VdGOM8Yg5V9BF5OMickREWkTkm8luz2wQkcUi8rKIHBKRgyLykDs/ICL/EpGg+/PSbmU+x4jIPBHZJyLb3OlKEXnDzfnTIpKR7DbOJBHxi8gzItIsIodF5IZUyLWIfNX9/W4SkT+LSJYXcy0iW0SkW0SaJsybNr/ieNyNf7+IfPj9rGtOFXQRmQf8AlgPrAA2isiK5LZqVsSAh1V1BbAG+KIb5zeBF1W1BnjRnfaih4DDE6Z/CPxUVauBfuBzSWnV7HkM+Keq1gIfwond07kWkTLgy8B1qtoAzAM+gzdz/Xvg4+fNu1B+1wM17uPzwK/ez4rmVEEHPgK0qGqrqkaAp4C7ktymGaeqXaq6130+hPMHXoYT6x/cxf4AfDI5LZw9IlIO3AH8xp0W4GPAM+4inopbRPKBjwK/BVDViKoOkAK5xrkF5nwRSQMWAF14MNequgPoO2/2hfJ7F/BHdewE/CJSeqnrmmsFvQxonzDd4c7zLBGpAFYBbwBXqWqX+9Jp4KokNWs2PQp8A0i404XAgKrG3Gmv5bwS6AF+53Yz/UZEsvF4rlW1E/gxcBKnkIeAPXg71xNdKL9XVOPmWkFPKSKSAzwLfEVVBye+ps54U0+NORWRTwDdqron2W35AKUBHwZ+paqrgBHO617xaK4LcLZGK4FFQDZTuyVSwkzmd64V9E5g8YTpcnee54hIOk4x/5Oq/tWdfead3S/3Z3ey2jdL1gEbRKQNpzvtYzj9y353txy8l/MOoENV33Cnn8Ep8F7P9W3AcVXtUdUo8Fec/Hs51xNdKL9XVOPmWkHfDdS4R8IzcA6ibE1ym2ac22/8W+Cwqv5kwktbgU3u803A3z/ots0mVf2WqparagVObl9S1fuAl4G73cU8FbeqngbaReRqd9atwCE8nmucrpY1IrLA/X1/J27P5vo8F8rvVuB+d7TLGiA0oWvmvanqnHoAtwNHgWPAt5PdnlmK8UacXbD9wFvu43ac/uQXgSDwAhBIdltn8Tu4GdjmPl8G7AJagL8Amclu3wzHuhJ40833c0BBKuQa+C7QDDQBTwKZXsw18Gec4wRRnD2yz10ov4DgjOQ7BhzAGQV0yeuyU/+NMcYj5lqXizHGmAuwgm6MMR5hBd0YYzzCCroxxniEFXRjjPEIK+jGGOMRVtCNMcYj/gcPVQx8tFecTQAAAABJRU5ErkJggg==\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"xCEHlMxrm81c"},"source":["Training and validation accuracy improve instantaneously, but reach plateau after around 50 epochs"]},{"cell_type":"code","metadata":{"id":"Zm-4ybkLm81d","executionInfo":{"status":"ok","timestamp":1604685417478,"user_tz":420,"elapsed":785370,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"cc6df5d1-bcfd-4656-8bc0-7588a149ab61","colab":{"base_uri":"https://localhost:8080/"}},"source":["results = model.evaluate(X_test, y_test)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["313/313 [==============================] - 0s 1ms/step - loss: 0.1801 - accuracy: 0.9465\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"DBHaJT5Um81f","executionInfo":{"status":"ok","timestamp":1604685417479,"user_tz":420,"elapsed":785364,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"f41904bb-c268-462e-b010-d33004890a07","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Test accuracy: ', results[1])"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Test accuracy:  0.9465000033378601\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"DNuN5ygtm81g"},"source":["## 4. Batch Normalization\n","- Batch Normalization, one of the methods to prevent the \"internal covariance shift\" problem, has proven to be highly effective\n","- Normalize each mini-batch before nonlinearity\n","- Doc: https://keras.io/optimizers/"]},{"cell_type":"markdown","metadata":{"id":"aTk5ndUCm81h"},"source":["<img src=\"https://raw.githubusercontent.com/torch/torch.github.io/master/blog/_posts/images/resnets_modelvariants.png\" style=\"width: 500px\"/>\n","\n","<br> Batch normalization layer is usually inserted after dense/convolution and before nonlinearity"]},{"cell_type":"code","metadata":{"id":"wXD9H1wfm81h"},"source":["from keras.layers import BatchNormalization"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"mZEOsHbVm81k"},"source":["def mlp_model():\n","    model = Sequential()\n","    \n","    model.add(Dense(50, input_shape = (784, )))\n","    model.add(BatchNormalization())                    # Add Batchnorm layer before Activation\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(50))\n","    model.add(BatchNormalization())                    # Add Batchnorm layer before Activation\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(50))\n","    model.add(BatchNormalization())                    # Add Batchnorm layer before Activation\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(50))\n","    model.add(BatchNormalization())                    # Add Batchnorm layer before Activation\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(10))\n","    model.add(Activation('softmax'))\n","    \n","    sgd = optimizers.SGD(lr = 0.001)\n","    model.compile(optimizer = sgd, loss = 'categorical_crossentropy', metrics = ['accuracy'])\n","    \n","    return model"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ziG6hzqZm81m"},"source":["model = mlp_model()\n","history = model.fit(X_train, y_train, validation_split = 0.3, epochs = 100, verbose = 0)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"4cVWf4x8m81o","executionInfo":{"status":"ok","timestamp":1604685663470,"user_tz":420,"elapsed":1031344,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"b9ace8eb-0ab3-4096-b772-49d788799a0f","colab":{"base_uri":"https://localhost:8080/","height":265}},"source":["plt.plot(history.history['accuracy'])\n","plt.plot(history.history['val_accuracy'])\n","plt.plot(history.history['loss'])\n","plt.plot(history.history['val_loss'])\n","plt.legend(['train acc', 'valid acc', 'train loss', 'valid loss'], loc = 'upper left')\n","plt.show()"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxU1f3/8deZO1sm+wZkgwCyJqwBRIOCgoqoLCqixaXa4lK1tYtfqbXWtWJrtV9blx+2WvVbQQQ3BEVtQdxQAwKCIGsgCWTfM5NZz++PGWKQAAESQiaf5+MxjyR3/Vx4PN45Offcc5XWGiGEEOHL1NEFCCGEaF8S9EIIEeYk6IUQIsxJ0AshRJiToBdCiDBn7ugCWpKUlKQzMzM7ugwhhOg01q5dW661Tm5p3SkZ9JmZmeTl5XV0GUII0WkopfYcbp103QghRJiToBdCiDAnQS+EEGHulOyjb4nX66WwsJDGxsaOLqXTstvtpKenY7FYOroUIcRJ1GmCvrCwkOjoaDIzM1FKdXQ5nY7WmoqKCgoLC+ndu3dHlyOEOIk6TddNY2MjiYmJEvLHSSlFYmKi/EUkRBfUaYIekJA/QfLvJ0TX1KmC/ki01pQ6S6n31Hd0KUIIcUoJm6BXSlHhqqDOW9cux6+urubpp58+rn2nTJlCdXV1G1ckhBCtEzZBD2CYDPwBf7sc+0hB7/P5jrjv8uXLiYuLa4+yhBDiqMIr6JWBX7dP0M+dO5edO3cyfPhw7rzzTlatWsVZZ53F1KlTGTx4MADTp08nJyeHrKws5s+f37RvZmYm5eXl5OfnM2jQIObMmUNWVhbnn38+LpfrkHMtXbqU008/nREjRjBp0iRKSkoAqK+v5/rrr2fIkCEMHTqUJUuWAPDee+8xcuRIhg0bxsSJE9vl+oUQnVenGV7Z3P1LN/PtvtpDljf6gyNK7Ma+Yz7m4NQY/nBJ1mHXz5s3j02bNrF+/XoAVq1axbp169i0aVPTcMXnn3+ehIQEXC4Xo0eP5rLLLiMxMfGg42zfvp0FCxbw3HPPccUVV7BkyRKuvvrqg7YZN24ca9asQSnFP/7xD/70pz/xl7/8hQcffJDY2Fi++eYbAKqqqigrK2POnDmsXr2a3r17U1lZeczXLoQIb50y6A9HoQjowEk735gxYw4ak/7kk0/yxhtvAFBQUMD27dsPCfrevXszfPhwAHJycsjPzz/kuIWFhcyaNYv9+/fj8XiazvHhhx+ycOHCpu3i4+NZunQpZ599dtM2CQkJbXqNQojOr1MG/eFa3vvq91HrqWVgwsCTUkdkZGTT96tWreLDDz/k888/x+FwMGHChBbHrNtstqbvDcNosevm9ttv51e/+hVTp05l1apV3Hfffe1SvxCiawi/PvqAH611mx87OjqaurrDj+ipqakhPj4eh8PB1q1bWbNmzXGfq6amhrS0NABefPHFpuXnnXceTz31VNPPVVVVjB07ltWrV7N7924A6boRQhwivILeZAC0S/dNYmIiubm5ZGdnc+eddx6yfvLkyfh8PgYNGsTcuXMZO3bscZ/rvvvuY+bMmeTk5JCUlNS0/J577qGqqors7GyGDRvGypUrSU5OZv78+Vx66aUMGzaMWbNmHfd5hRDhSR2t9auUygBeAroDGpivtf7fH2yjgP8FpgBO4Mda63WhddcB94Q2fUhr/SJHMWrUKP3DF49s2bKFQYMGHXG/qsYq9tXvo198P6yG9Win6ZJa8+8ohOh8lFJrtdajWlrXmj56H/BrrfU6pVQ0sFYp9YHW+ttm21wI9At9TgeeAU5XSiUAfwBGEfwlsVYp9bbWuuoEruewDBVs0bfXEEshhOiMjtp1o7Xef6B1rrWuA7YAaT/YbBrwkg5aA8QppVKAC4APtNaVoXD/AJjcplfQzIGum/Z6aEoIITqjY+qjV0plAiOAL36wKg0oaPZzYWjZ4Za3dOwblVJ5Sqm8srKyYymribTohRDiUK0OeqVUFLAEuENrfejTSidIaz1faz1Kaz0qObnFF5kflbTohRDiUK0KeqWUhWDI/1tr/XoLmxQBGc1+Tg8tO9zydiEteiGEONRRgz40ouafwBat9eOH2ext4FoVNBao0VrvB1YA5yul4pVS8cD5oWXtwqRMmJRJgl4IIZppTYs+F7gGOFcptT70maKUulkpdXNom+XALmAH8BzwMwCtdSXwIPBV6PNAaFm7OfDQ1KkgKioKgH379nH55Ze3uM2ECRP44VBSIYRoS0cdXqm1/gQ44quJdHAw/q2HWfc88PxxVXccDFP7zWB5vFJTU1m8eHFHlyGE6KLC6slYaL+piufOnXvQ9AP33Xcfjz32GPX19UycOJGRI0cyZMgQ3nrrrUP2zc/PJzs7GwCXy8WVV17JoEGDmDFjRotz3QA88MADjB49muzsbG688camaR127NjBpEmTGDZsGCNHjmTnzp0APProowwZMoRhw4Yxd+7ctr58IUQn1iknNePduVD8TYurevgbg1MgmB3HdsweQ+DCeYddPWvWLO644w5uvTX4h8uiRYtYsWIFdrudN954g5iYGMrLyxk7dixTp0497PtZn3nmGRwOB1u2bGHjxo2MHDmyxe1uu+027r33XgCuueYa3nnnHS655BJmz57N3LlzmTFjBo2NjQQCAd59913eeustvvjiCxwOh8x3I4Q4SOcM+iNQR+5lOm4jRoygtLSUffv2UVZWRnx8PBkZGXi9Xu6++25Wr16NyWSiqKiIkpISevTo0eJxVq9ezc9//nMAhg4dytChQ1vcbuXKlfzpT3/C6XRSWVlJVlYWEyZMoKioiBkzZgBgt9uB4PTF119/PQ5H8JebTFUshGiucwb9EVre1Q0lVLgqGJQ46LCt6uM1c+ZMFi9eTHFxcdPkYf/+978pKytj7dq1WCwWMjMzW5ye+Fg0Njbys5/9jLy8PDIyMrjvvvtO+JhCiK4r/ProTQYa3S4zWM6aNYuFCxeyePFiZs6cCQSnFO7WrRsWi4WVK1eyZ8+eIx7j7LPP5pVXXgFg06ZNbNy48ZBtDoR6UlIS9fX1TTdyo6OjSU9P58033wTA7XbjdDo577zzeOGFF3A6nYBMVSyEOFj4BX07PjSVlZVFXV0daWlppKSkADB79mzy8vIYMmQIL730EgMHHvmlJ7fccgv19fUMGjSIe++9l5ycnEO2iYuLY86cOWRnZ3PBBRcwevTopnUvv/wyTz75JEOHDuXMM8+kuLiYyZMnM3XqVEaNGsXw4cN57LHH2vbChRCd2lGnKe4IxztNMUCtu5aCugL6xPUhwhzRXiV2WjJNsRDh6UjTFIdfi17muxFCiIOEX9DLfDdCCHEQCXohhAhz4Rf00nUjhBAHCbuglxkshRDiYGEX9IAEvRBCNBOWQW+Y2n6q4urqap5++unj2nfKlClUV1e3evsDE6YJIURbCJug11rTuG0b3tJSzMrc5i36IwW9z+c74r7Lly8nLi6uTesRQojWCpugV0pBQKM93nZp0c+dO5edO3cyfPhw7rzzTlatWsVZZ53F1KlTGTx4MADTp08nJyeHrKws5s+f37RvZmYm5eXl5OfnM2jQIObMmUNWVhbnn3/+YacpPmD9+vWMHTuWoUOHMmPGDKqqqgB48sknGTx4MEOHDuXKK68E4KOPPmL48OEMHz6cESNGUFdX16b/BkKIzumok5oppZ4HLgZKtdbZLay/E5jd7HiDgGStdaVSKh+oA/yA73BPbR2rR798lK2VWw9ZHnC5QCl8FhM+7cNxDFMVD0wYyF1j7jrs+nnz5rFp0ybWr18PwKpVq1i3bh2bNm2id+/eADz//PMkJCTgcrkYPXo0l112GYmJiQcdZ/v27SxYsIDnnnuOK664giVLlnD11Vcf9rzXXnstf/vb3xg/fjz33nsv999/P3/961+ZN28eu3fvxmazNXULPfbYYzz11FPk5uZSX1/fNLulEKJra02L/l/A5MOt1Fr/WWs9XGs9HPgt8NEPXhd4Tmh9m4T8kSiTCQ5M6XASZnYYM2ZMU8hDsJU9bNgwxo4dS0FBAdu3bz9kn969ezN8+HAAcnJyyM/PP+zxa2pqqK6uZvz48QBcd911rF69GghOcTx79mz+7//+D7M5+Ps6NzeXX/3qVzz55JNUV1c3LRdCdG2teZXgaqVUZiuPdxWw4EQKao3Dtby9+/fjq6qiITOZEmcpAxMGNo2rbw+RkZFN369atYoPP/yQzz//HIfDwYQJE1qcWthmszV9bxjGUbtuDmfZsmWsXr2apUuX8vDDD/PNN98wd+5cLrroIpYvX05ubi4rVqw46iRrQojw12Z99EopB8GW/5JmizXwvlJqrVLqxrY612FrsFggEMDQwctqy6mKo6Ojj9jnXVNTQ3x8PA6Hg61bt7JmzZoTPmdsbCzx8fF8/PHHQHDmyvHjxxMIBCgoKOCcc87h0Ucfpaamhvr6enbu3MmQIUO46667GD16NFu3Htq9JYToetryb/tLgE9/0G0zTmtdpJTqBnyglNqqtV7d0s6hXwQ3AvTs2fO4ClAWCwBmf7Dfxqd9WLAc17F+KDExkdzcXLKzs7nwwgu56KKLDlo/efJknn32WQYNGsSAAQMYO3Zsm5z3xRdf5Oabb8bpdNKnTx9eeOEF/H4/V199NTU1NWit+fnPf05cXBy///3vWblyJSaTiaysLC688MI2qUEI0bm1apriUNfNOy3djG22zRvAa1rrVw6z/j6gXmt91AHixztNccDpxL1rF4G07uT7S+kV04soa9TRTtelyDTFQoSndp+mWCkVC4wH3mq2LFIpFX3ge+B8YFNbnO+wQi16ky/YZSNPxwohROuGVy4AJgBJSqlC4A8Q7A/RWj8b2mwG8L7WuqHZrt2BN0LvbTUDr2it32u70luo1WwGpTD5A2CWoBdCCGjdqJurWrHNvwgOw2y+bBcw7HgLOx5KKZTZjPL6gkEvM1gKIUT4PBl7gLJY0D4fSilp0QshBOEa9F4vhmr7aRCEEKIzCtugtxpWPAFPR5cjhBAdLvyC3mwBrbFjwePv2KCPigoO7dy3bx+XX355i9tMmDCBHw4lPdJyIYQ4VuEX9NbgEEurNvAFfKdE901qaiqLFy/u6DKEEF1U+AW9ORT0fgXQZt03c+fO5amnnmr6+cDLQerr65k4cSIjR45kyJAhvPXWW4fsm5+fT3Z28Fkzl8vFlVdeyaBBg5gxY0ar5rpZsGABQ4YMITs7m7vuCs7z4/f7+fGPf0x2djZDhgzhiSeeAFqevlgI0bV1yukNi//4R9xbWp7HRWtNwOkEqwWFl31mG2Z19Mu0DRpIj7vvPuz6WbNmcccdd3DrrbcCsGjRIlasWIHdbueNN94gJiaG8vJyxo4dy9SpUwk9P3CIZ555BofDwZYtW9i4cSMjR448Yl379u3jrrvuYu3atcTHx3P++efz5ptvkpGRQVFREZs2BZ9BOzBVcUvTFwshurawa9GjFChQoZkdAq2Y4qE1RowYQWlpKfv27WPDhg3Ex8eTkZGB1pq7776boUOHMmnSJIqKiigpKTnscVavXt00//zQoUMZOnToEc/71VdfMWHCBJKTkzGbzcyePZvVq1fTp08fdu3axe233857771HTExM0zF/OH2xEKJr65RJcKSWN0Djd9swOSLIj3QRYXGQHp3eJuedOXMmixcvpri4mFmzZgHw73//m7KyMtauXYvFYiEzM7PF6YnbWnx8PBs2bGDFihU8++yzLFq0iOeff77F6Ysl8IXo2sKvRQ8oi/n7IZZtOPJm1qxZLFy4kMWLFzNz5kwgOD1xt27dsFgsrFy5kj179hzxGGeffTavvBKc923Tpk1s3LjxiNuPGTOGjz76iPLycvx+PwsWLGD8+PGUl5cTCAS47LLLeOihh1i3bt1hpy8WQnRtYdnUUxYrAWcDViOaWk9tmx03KyuLuro60tLSSElJAWD27NlccsklDBkyhFGjRh31RR+33HIL119/PYMGDWLQoEHk5OQccfuUlBTmzZvHOeecg9aaiy66iGnTprFhwwauv/56AoHgBG6PPPLIYacvFkJ0ba2apvhkO95pig/wFhfjq6igoXd3SpwlDEgYgNkUlr/TjplMUyxEeGr3aYpPNcoSfGjKFvqDpaMfnBJCiI4UvkEPWAKhsfQS9EKILqxTBX1ru5m+f6Vg8Gd3wN1eJXUqp2I3nRCi/XWaoLfb7VRUVLQqrA4EPe0w8qaz0lpTUVGB3W7v6FKEECdZp7lDmZ6eTmFhIWVlZa3a3ltSiqm+nmqbj0AgQL1Dhhna7XbS09vmmQIhROfRmlcJPg9cDJS29HJwpdQEgu+K3R1a9LrW+oHQusnA/wIG8A+t9bzjLdRisdC7d+9Wb7/zN3dizcxk6bVpvLnjTdb8aM1hpyUQQohw1pqum38Bk4+yzcda6+Ghz4GQN4CngAuBwcBVSqnBJ1LssbCddhruHdvpFdMLp89JRWPFyTq1EEKcUo4a9Frr1UDlcRx7DLBDa71La+0BFgLTjuM4x8XWrx/evQX0svYAYE/tkZ9YFUKIcNVWN2PPUEptUEq9q5TKCi1LAwqabVMYWnZS2Pr3A61JqwjevJWgF0J0VW1xM3Yd0EtrXa+UmgK8CfQ71oMopW4EbgTo2bPnCRdl6xcsIaawBrPJLEEvhOiyTrhFr7Wu1VrXh75fDliUUklAEZDRbNP00LLDHWe+1nqU1npUcnLyiZaFtWdPlNWKd8dOekb3ZHfN7qPvJIQQYeiEg14p1UOFhrMopcaEjlkBfAX0U0r1VkpZgSuBt0/0fK2uyzCwntYX9/btDEgYwJbKLSfr1EIIcUo5atArpRYAnwMDlFKFSqmfKKVuVkrdHNrkcmCTUmoD8CRwpQ7yAbcBK4AtwCKt9eb2uYyW2fv1w719O9mJ2RQ3FFPuKj+ZpxdCiFPCUfvotdZXHWX934G/H2bdcmD58ZV24mz9+lHz1ttk2TIB2Fy+mfEZ4zuqHCGE6BCdZgqE43HghmzvSjMmZWJTxaYOrkgIIU6+LhH0alcBfWL7sLn8pPYcCSHEKSGsg96ckoIpMjLYT5+UzeaKzTKDoxCiywnroFdKYQvdkM1KzKKysZL9Dfs7uiwhhDipwjroIdh94962jezE4AO7m8qln14I0bWEf9D374+/upo+/kTMJjObK6SfXgjRtYR/0IduyOpd+fSP7y83ZIUQXU74B33/YNAfeHBqc8VmAjrQwVUJIcTJE/ZBb05IwEhMpHHbNrKTsqn31ssEZ0KILiXsgx7APmgQjZs2k5UUvCEr/fRCiK6kSwR9xIjhuLdto5fRDbthl356IUSX0iWC3jFiBGiN95vNDE4czMayjR1dkhBCnDRdIujtQ4eCyYTr6/XkdM9hc8VmGrwNHV2WEEKcFF0i6I2oKGz9++P6+mvGpIzBr/2sK1nX0WUJIcRJ0SWCHoL99K4NGxiWMASLycKXxV92dElCCHFSdJmgd4wYQaC+HpVfxLDkYXyx/4uOLkkIIU6KLhP0ESNGAOD6eh1jUsawtXIrNe6aDq5KCCHaX2teJfi8UqpUKdXibGBKqdlKqY1KqW+UUp8ppYY1W5cfWr5eKZXXloUfK0t6OkZSEq6vv+b0Hqej0eSVdGhJQghxUrSmRf8vYPIR1u8GxmuthwAPAvN/sP4crfVwrfWo4yuxbSilcIwYjvPr9QxJGkKEOYIv90s/vRAi/B016LXWq4HKI6z/TGtdFfpxDZDeRrW1uYgRI/Hu3YuqqmFEtxFyQ1YI0SW0dR/9T4B3m/2sgfeVUmuVUje28bmOWcSI4QC41q9nTI8x7KjeQbmrvIOrEkKI9tVmQa+UOodg0N/VbPE4rfVI4ELgVqXU2UfY/0alVJ5SKq+srKytyjqIPSsLZbHg/PprTk85HYC8YumnF0KEtzYJeqXUUOAfwDStdcWB5VrrotDXUuANYMzhjqG1nq+1HqW1HpWcnNwWZR3CZLViz87GlbeWgQkDibJE8UWxDLMUQoS3Ew56pVRP4HXgGq31tmbLI5VS0Qe+B84HOvw9fpFnjMX1zTeoeiejuo/i832fywvDhRBhrTXDKxcAnwMDlFKFSqmfKKVuVkrdHNrkXiARePoHwyi7A58opTYAXwLLtNbvtcM1HJPI3FwIBGhYs4bxGeMpqi9iW9W2o+8ohBCdlPloG2itrzrK+p8CP21h+S5g2KF7dKyIoUMxRUbS8OlnTDjrNh74/AH+u/e/DEgY0NGlCSFEu+gyT8YeoCwWHGPH0vDJJyTaExnebTj/LfhvR5clhBDtpssFPUBk7pl4i4rw7t3LxJ4T2Vq5lcK6wo4uSwgh2kWXDPqo3FwA6j/9lHN7ngvAf/dKq14IEZ66ZNBbevbEkp5OwyefkhGdQf/4/vxn7386uiwhhGgXXTLolVJEjsvFuWYN2utlYs+JfF36NRWuiqPvLIQQnUyXDHoIDrMMOJ24NmxgYs+JaDSrClZ1dFlCCNHmum7Qn346GAb1n35K//j+pEWlSfeNECIsddmgN2JiiBg6lIaPP0EpxaSek/h8/+dUN1Z3dGlCCNGmumzQA0RPPJfGTZvwFBRwcd+L8QV8rMhf0dFlCSFEm+rSQR8zZQoAtcuWMzBhIP3j+/P2zrc7uCohhGhbXTroLampRIwcSe2yZQBM7TuVjeUb2V2zu4MrE0KIttOlgx4g5qIpuLdvp3HbNqb0noJJmVi6c2lHlyWEEG1Ggn7yZDAMapctJ9mRzBmpZ/DOrncI6EBHlyaEEG2iywe9OTGRyLFjqV22DK01U/tMZX/DftaWrO3o0oQQok10+aAHiLnoIryFhTRu2MA5Pc8h0hIpN2WFEGFDgh6IPm8SymqlZtlyIswRnN/rfN7Pf596T31HlyaEECdMgh4woqOJGn82te++i/Z6mTVgFk6fk9e3v97RpQkhxAlrVdArpZ5XSpUqpVp856sKelIptUMptVEpNbLZuuuUUttDn+vaqvC2FjvjUvzl5dStWkVWUhYju43kla2v4A/4O7o0IYQ4Ia1t0f8LmHyE9RcC/UKfG4FnAJRSCcAfgNOBMcAflFLxx1tse4o6+yzM3btTveg1AK4ZfA1F9UXy9ikhRKfXqqDXWq8GKo+wyTTgJR20BohTSqUAFwAfaK0rtdZVwAcc+RdGh1FmM3GXXUbDJ5/gLSrinIxzSItK4+VvX+7o0oQQ4oS0VR99GlDQ7OfC0LLDLT8lxV1+GQDVS5ZgmAxmD5rN16Vfs6m8xR4rIYToFE6Zm7FKqRuVUnlKqbyysrIOqcGSmkrkWeOoXrwE7fMx47QZRFoieenblzqkHiGEaAttFfRFQEazn9NDyw63/BBa6/la61Fa61HJycltVNaxi7/iCnylpdSv/pgoaxSX9ruUD/I/kJeHCyE6rbYK+reBa0Ojb8YCNVrr/cAK4HylVHzoJuz5oWWnrKjx4zEnJ1O9aBEA1w2+DpMy8dw3z3VwZUIIcXxaO7xyAfA5MEApVaiU+olS6mal1M2hTZYDu4AdwHPAzwC01pXAg8BXoc8DoWWnLGWxEHvZpdSvXo2noIDukd25YsAVvLXjLQpqC45+ACGEOMUorXVH13CIUaNG6by8vA47v7eklB2TJhF/5ZX0+N3dlLvKuXDJhZyfeT4Pj3u4w+oSQgBaQ8AHfi+gAQVKQcAfXH7Qxw/aD4FA6KsPfI3gcwe/N2xgtoIyCLjraWyoweNqwEQAExqT0mhlJmCyEFAG2u9FB3xovw8CXvAHz+Pz+/F6fXh9PvweFwFPA9rjwufz4vVrPH4NSmExKSyGCcMUbGUrpcHvI+BtRPsa8RsRZN34z+P6Z1FKrdVaj2ppnfn4/7XDl6V7N2KnTKF6yRKSb7uVpNgkZg2YxctbXmbOkDlkxmZ2dImiq/N7g4Hl94Y+nmbB4z046JQp+DEZgAodQIO7DlxV4KoOhqAygtv4PeBxgrchGKomA0wW0IHgOl8j2ufG73Xj97gJ+D3QFIB+fBo8AQgENCrgQ2kfKuDF5HOj/B5M/sbgx9eICnhC1SgCmAhg4FcGfmXGoj1YA41YAi5UwI8iAFpj0D4PMZoAR+jTFtzagh8TCo1Co5v+7YOCS8GPQSNW3NpCpSmhjc5+MAn6w0i4/sfUvPUWVYsWkTRnDtdnX8+ibYt4duOzzDtrXkeXJ9qS1sEA87qCX3Xg+5agz/39p3lL0e8Fvzu4fdNxAtBYAw0V4KoMHtewBD/QrMXpD4Wy5/tg9nvA50a769EeJwGvC+3zfB/gAT8q4A0Gp68Rk/Z1zL8V4NUGXsx4MDd99WkDHwaBZsGm0Pgw8GHGi4EbC43aigc7jcTQiA2PNqNRKAIYaCzKj1X5MasAjQEzDdqGExs+DJQyYTUbYDITMJnxY0YphWECQ0FABc/vw4wyzNhtVmxWKybDwOUDlxc8WqEsEZgsdgzDjNXkx4YHq9KYbFFYHNEYtqjgL5sA+AIagwBm7cOMDwwzymRGm4L/rwFloJWB1WrBbrFit5qxRTiw2iKx2yxE2sxE2sw4LAYALq+fBo8Pjy9AIAC+QADDpIiymUm2mUk3t89ASAn6w7APHIjjjLFUvfx/JF53HYkRiVw18Cpe2PQCN2TfQP/4/h1dYudyIOT83lBwNoaCslmL1NMAnvrg1+b8HvA6wdsYDF8dCH48DdBYGwxXnysYmn5P8DwHwjrgbXaOYGDr0EcdaPn6vaG2VdtxEoFGYeDHgjd4GRgElAk/RjAstdEUlr5QENZrOy5to5EovBihEDWHAjMYYo2hwHRjwYMFbyhIfdrAbLVitdpo9Jtw+hQun8ZuVkRaFA4zwW4DQ2EoRa22U+ZzUOa1Y7VaSHCYibcbWKw2ApZIAoYdw2xgVRqrKYBJKQJmGygzZsNEhMXAbjGFuiKCxzQbwdCKtluIsBoHtWEdJkVUaBub2YTNbGAzm7CGPmaTQqnv99Ba4/EHaHD7sVuC52u+vjM6EPwnmwT9ESRefz0FN95E7XvvETt1Kjdk38Br217jibVP8MykZzq6vKPTulnL0xtssXpdwVB01wdD1V3X1JoMtmYPBJ4OhvGBP/i4WLgAACAASURBVOEPrD/QZeBtDB7H2/h9n6fX+X1Y+9zN6ggEj9fG/JhwmqJwqkg8yoqXYCvPiwmfVngDJnyYCJisBEwReLSZOp+JOl+oFUqwFerFCAVnMDyDXQgKPybc2tIUqD4M/JjwhwL6QEhrFGYTmA0DU0QcRlQiMVGRWAwTWmv8AY1fg88fwOfXRFgNEiKtxDks2C0GAa3RwS5c7GYDu8UgwmLCYTUTYTWIMpuwGArDZMKkwFAq1JJVRFgMIqwGkbbgMW1mo83/nTuKUir0yyB8rqmjSNAfQeS4cVj79qXihX8Rc8klxNpiuWnoTTyW9xif7fuMM1PPbJsT+TzBVqm79uCvjTXBFmtTSzcUzO76YPgeaAn7PKGwbRa6vsaDuxVOgN9kw2ey4TNZgu1TkxWvsoW+hgJSR+LS3anTdmpNVhrNZlAKhcKnNXVucPpVqOUaao1qc6g1GmyROrFTryNwYjvo/B7MNGobjVgJKBNxDhvxkXYMqx2TSWFSKhR+YJiCrUWHzUyk1UChaPT5cXn8GCZFUpSNpCgbMRFm7EawFWlSoEJtT7OhiI+0kuAIBrHNbGA1B1usPn8Ar1/j9QeCLdJQi9ZqmDp9S1OENwn6I1AmE4k3XM/+391D/cqVRJ97LlcNvIoFWxfweN7jnH7x6RimZq0Nnxuq90L1nuANrgOh3VAO9SXBj6fh+yB21wWD3Oc6ejHmCLBGErBF4zdH4jUi8CkLXiLxmWLw2Gx4bFY8mHFpK66ABWfAwOk3qPcrGnwG1V4zVV6Dao+ZWm2jQUfQgL0pdH2Y8Tf7Y9uNFRdWdLNRuCYFNrOBSYEpNILgwJ/wDquZKJuZKLsZm9kUbMPrYPgmRllJjraRFGkjzmEhzmEl0mbg9Pipa/Ti9PiJtluId1iIsVuwWYLhajGZMDUL0Si7GcMkoSrEsZDhlUehfT52XXwJymKm9+uvoxr28+6WhfzPtpd5KCqbaV5TMMBr90NtES12UVgcENUdorqBLRpttuHBSl3ATlXAToXPTh2RuEyRNKhIqgN2Sjx2it02Sj1WqrxWGnxQ7/bh9LRuxEGk1SDSZiY2wkJMhIVoe/D72ND3drOBxXygRft9cDqsBlH2UGCH+hMjrWYibcHj2czSehXiVCTDK4+Xx4naspTksVaKFm6j5qa+xPWsYTLwUmp3nvRv5HxXJBFRPSAzF+IzCcRlUmZOYU9jBHsaDHbWGFR5zbi8fpweP/srXeypcFLvPnjUhDV0Q8tsKBxWg3iHlYRIK0kxZnpag/2wDquZhEgr8aFuhUibmahQADssZuzWYOs60mrGJK1eIUSIBP0PBQJQsAY2LITNb4C7luioFOwpEZRvcRAz5x5MKVncSSPXfXQH9wy4hOFR17KtuI4tW2vZWlxHvdsJOAGwW0zERlhC3RsGKbF2Rmcm0DPBQa9EB+nxDtLjIzrkTrwQomuQdDmgcjesfQG+WQK1hcHulsHTYPhsVK9cks/4nIKf/JSvPvXz3oA4Ptlejsd0Oiv0a7yRn4RD92FQSjSXjUxjYEoMfZOjyEx0kBxtk64OIUSHkqBvrIWPH4M1zwSHAfadCJP+AAOmgC2KPRUNLFu9i4+/MzEt+TR6vfRP3riwB0P6p3JJn1+wuPgOeg19lyVTF2E32zv6aoQQ4hBdO+g3vwnL74SGUhj2I5h4L8Sk0Oj1s/yb/bz61Td8sTs4B9uglBiqrr6RoX+9i2XR35L2kxkAjCy6n1s+vIX5G+fz85E/78irEUKIFnXNoNcaVj0CHz0KqSPhRwshLQeANbsq+O3r37C7vIFeiQ7uvGAAl45MIyU2AoD9xeuofuUVEi+dgX3wYMaljWNa32k8v+l5JvacSFZSVkdemRBCHKLrDa/0uuDNW4I3WofPhoufALONGpeXee9uYcGXBWQkRHD/1Cwm9O92yOgVf20tOy+cgiUtjcwFr6AMgxp3DZe9fRkR5ghevfhVHJa2mhZJCCFa50jDK0+ZVwmeFAE/LLgy2GVz3gMw7Skw21ixuZjzHv+IRXmF3HR2H96/YzznDuze4hBFIyaG7nPvonHjRqpfew2AWFssD497mD21e/hL3l9O9lUJIcQRda2g/+Rx2LUq2IrP/QUVDR5ufWUdN728lsQoG2/dmstvpwwiwnrkuTViLr4Yx9ixlP7lcbylpQCcnnI612Vdx6Jti1hVsKr9r0UIIVqp6wT93i9g5SOQfRnk/Jgap5dZ89fwweYSfnN+f96+LZfstNhWHUopRY9770V7POy/5x4OdH/dPuJ2BiYM5A+f/YFyV3l7Xo0QQrRaa18lOFkp9Z1SaodSam4L659QSq0PfbYppaqbrfM3W/d2Wxbfaq5qWPITiE2Hi5/A49fc/H9r2VPRwIs3jOG2c/thMY7td56tT2+6/c+dNKz+mKoFCwCwGlbmnTUPp9fJL1f+Ek8bTSomhBAn4qjpppQygKeAC4HBwFVKqcHNt9Fa/1JrPVxrPRz4G/B6s9WuA+u01lPbsPbW0RqW/gLq9sPlL6BtMcx9fSOf76rgT5cP5Yy+icd96Pgf/YjIs86i9NE/4d65E4C+cX15aNxDrC9bz4NrHuRUvNkthOhaWtOMHQPs0Frv0lp7gIXAtCNsfxWwoC2KaxMbFsK3b8I5v4P0HP7+3x28vq6IOyb1Y8aI9BM6tFKK1D8+jMnhYN+d/4P2BFvwF2RewM3DbubNHW/y8rcvt8VVCCHEcWtN0KcBBc1+LgwtO4RSqhfQG/hvs8V2pVSeUmqNUmr6cVd6PKr2BB+I6nkm5P6CD74t4S8fbGPGiDR+MbFfm5zCnJxMyoMP0Pjtt5T+5fGm5bcMu4VJPSfxl7V/4ePCj9vkXEIIcTza+mbslcBirXXzuXR7hcZ2/gj4q1Kqb0s7KqVuDP1CyCsrKzvxSgJ+eOPm4PcznmVHuZNfvrqeoemxPHLpkDadfyZ60iTir7mGyhdfpHbF+wCYlImHxz3MgPgB/PqjX7O5fHObnU8IIY5Fa4K+CMho9nN6aFlLruQH3TZa66LQ113AKmBESztqredrrUdprUclJye3oqyj+OxvsPczmPJnauypzHlpLXaLiWevzsFuaftXk3W/8zdEDBvG/rvvxpOfD4DD4uDpSU+TYE/gZ//5GQW1BUc+iBBCtIPWBP1XQD+lVG+llJVgmB8yekYpNRCIBz5vtixeKWULfZ8E5ALftkXhRxTww8ePQ//JMOxK7n79GwoqnTw9O4fUuIh2OaWyWkn76xMoi4XCn/+CgCv41qikiCSemfQMfu3n5g9vpsJV0S7nF0KIwzlq0GutfcBtwApgC7BIa71ZKfWAUqr5KJorgYX64GEmg4A8pdQGYCUwT2vd/kFfvBHcNTBkJjvKGlj2zX5+NqEvY3ontOtpLSkppP75z7i3b6foN3eifcGXi/SO7c3fz/07pc5S5nwwR8JeCHFStaqPXmu9XGvdX2vdV2v9cGjZvVrrt5ttc5/Weu4P9vtMaz1Eaz0s9PWfbVv+YeR/EvzaK5d/fbYbq2Hi2jMzT8qpo84aR/ff/Y76//yH4gcfahpeObzbcP428W8U1BbwkxU/kQeqhBAnTXg+GZv/CSSeRrU5kSVri5g2PJWkKNtJO33C1bNJnDOH6ldfpfyZZ5qWj00Zy9OTnmZfwz5uWHEDpc7Sk1aTEKLrCr+gD/hhz2eQOY6FXxXg8vq5YVzvk15G8q9+Sey0aZQ/+TeqFr7atHx0j9E8O+lZShpKuHr51Wyv2n7SaxNCdC3hF/TFG8Fdi69nLi9+ls+ZfRMZlBJz0stQSpHy0INEjR9P8f33U7N0adO6kd1H8sLkF/AFfFz77rV8WvTpSa9PCNF1hF/Q7w4+nPRfV3/21zRyQ+7Jb80foCwW0v73rzhGj2bf3N9S9+GHTesGJw7mlYteITUqlVv/cyuvbn31CEcSQojjF35BH+qff269k8xEB+cO7Nah5ZjsdtKffhp7dhZFv/wV9R991LSuR2QPXrrwJXLTcnnoi4d4aM1DeAPeDqxWCBGOwivo/T7Y+zlkjmPL/jomDDj0DVEdwYiKpOf8+Vj7nUbBz26lesmSpnWRlkiePOdJrs++nle/e5WbPriJqsaqDqxWCBFuwivoQ/3z7vRc6t0+usfYO7qiJkZsLL1eepnIsWPZ/7t7KPv7U01DLw2Twa9yfsUfx/2RDaUbuOKdK1hfur6DKxZChIvwCvrQ+PnShOBrE7tFn7whla1hREWS8ewzxE6fTvnf/x58aYnn+znrL+l7CS9d+BJmZebH7/2Yf37zTwI60IEVCyHCQfgFfWI/9gfiAEg+xYIegjdoUx75I0k/+xk1S15n75wb8Vc3vaeFrKQsFl2yiEm9JvHXdX/lxg9uZG/t3g6sWAjR2YVP0Dfrny+tawSgW8ypF/QQHHqZ/PPbSf3To7jWrSP/yquaJkIDiLZG8+ez/8wfzvgDm8s3M+OtGTy74Vl5Y5UQ4riET9CjYcazkHMdZXVuALpFnzp99C2JnTqVnv96AX91Nbsvn3nQ8EulFJf3v5y3pr/FOT3P4an1TzFz6Uy+KfumAysWQnRG4RP0hgUGXgSpIyitc2MxFHERlo6u6qgcOTlkLl6MtVcvCm+7ndLHHmuaDA2gm6Mbj41/jKcnPo3T5+Tqd6/mibVP4Pa7O7BqIURnEj5B30xprZukKNspMbSyNazpafR65d/EzZpFxT/+yZ5rrsW9e/dB25yVfhavT32d6adN5/lNz3P525ezIn+F3KwVQhxVeAZ9XeMpN+LmaEw2Gyn33xec5njnTnZPn0HF8y+g/d+/rCvaGs39Z97Ps5OexVAGv/noN8x6ZxarClbJS8iFEIcVlkFfVucm+RTvnz+c2Esups87S4nMzaX0T38i/6of4dp88GsIc9NyWTJ1CX8c90cavA3c/t/bmbl0Ju/tfg9/wH+YIwshuqqwDfpTdcRNa1i6dSP9qb+T+thjeIuKyJ95BcUPPoS/trZpG8NkcEnfS3hr+ls8lPsQbr+bO1ffyfS3pvN+/vvSwhdCNAm7oPf6A1Q0eEg+ifPPtwelFLEXX0Tfd5cTf+WVVL3yCjsvnEL14sXowPf98haThWmnTePNaW/y+ITHMZvM/PqjX3P1u1ezrmRdB16BEOJU0aqgV0pNVkp9p5TaoZSa28L6HyulypRS60OfnzZbd51Sanvoc11bFt+S8vrQ0MpO3KJvzoiJoce9vyfztdewZmSw/57fkz/zCpx5eQdvZzI4r9d5LL5kMfefeT/F9cVc9951XLP8Gt7Lf08mSxOiC1NH+xNfKWUA24DzgEKCLwu/qvm7X5VSPwZGaa1v+8G+CUAeMArQwFogR2t9xFm7Ro0apfN+EGSttaGgmmlPfcpz147ivMHdj+sYpyqtNbXvLKP0scfwlZQQMSqHpDlziDz7bJQ6eISR0+tkyfYlLNi6gIK6Aro5ujH9tOlMP206GdEZHXQFQoj2opRaq7Ue1dK61rToxwA7tNa7tNYeYCEwrZXnvgD4QGtdGQr3D4DJrdz3uHz/sFR4tOibU0oRe8nF9H13Od1/Oxdv0T4KbrqZ3dOmU/veewd16TgsDq4ZfA1Lpy/l7+f+nX7x/Xhu43NMeX0KP13xU5btWiZj8YXoIloT9GlAQbOfC0PLfugypdRGpdRipdSBJmNr920zpaGgPxXnuWkrJoeDhOuu47T3V5DyyCNor5eiO37J7mnTqFm2DO39vpvGMBmMzxjPs5Oe5f3L3+e24bdRWF/I3I/ncu6ic5n35Tx5naEQYa6tbsYuBTK11kMJttpfPNYDKKVuVErlKaXyysrKjruQA/PcnMyXgXcUZbEQN2M6fd5ZSupfHkNrzb5f/4bt55xL6eNP4CksPGj7HpE9uGnYTSy/dDnzz5vPmalnsui7RVz69qVcs/wa3tzxJvWe+g66GiFEe2lNH/0ZwH1a6wtCP/8WQGv9yGG2N4BKrXWsUuoqYILW+qbQuv8HrNJaLzjSOU+kj/7uN77hvU3FrPv9ece1f2emAwHqV6+m+tVFwTdZaU3UhAkkXHctjtNPP6QfH6CqsYq3d77N4m2Lya/Nx2wyM6r7KManj2dSr0n0iOzRAVcihDhWR+qjb03QmwnejJ0IFBG8GfsjrfXmZtukaK33h76fAdyltR4buhm7FhgZ2nQdwZuxlUc654kE/U9fzKOwysl7d5x9XPuHC29xMVWvvkr1wlfxV1Vh69eP2EsvJWbKhVi6H3qTWmvN+rL1rCxYyUcFH7GrZhcKxdiUsUw9bSrnZpyLw+LogCsRQrTGCQV96ABTgL8CBvC81vphpdQDQJ7W+m2l1CPAVMAHVAK3aK23hva9Abg7dKiHtdYvHO18JxL00576lBi7mZd/cvpx7R9uAm43te8so2rBAho3bQKlcIwZQ9zllxN9wfmYrNYW99tTu4dlu5bx9s63KaovwmwyM7LbSM5IPYNxaeMYED+gxb8QhBAd44SD/mQ7kaA/85H/MLZvIo9fMbyNq+r83Lt3U7tsOTVvv413716MhATiLruU6MmTsQ8ahDIdessmoAOsK1nH6sLVfLrvU7ZVbQMgKSKJ3NRczkw9kzEpY0iKSDrZlyOEaKbLBL3Wmv73vMtPxvVh7oUD26Gy8KADARo++5yqhQuo/+9KCAQwkpOIGncWURMmEJmbixEV2eK+Zc4yPt33KZ8Wfcpn+z6j1hOclqF3bG/GpoxlXNo4RvcYTYQ54mRekhBdXpcJ+soGDyMf/IB7Lx7MDeN6t0Nl4cdXXk79J5/QsHo19Z98SqC2FmWx4BgzhqgJE4iaMB5rRssPWPkCPr6r/I4vi7/ky+IvWVuyFpfPhc2wkdM9hxHdRjCy20iGJA+R4BeinXWZoN9aXMvkv37M364awSXDUtuhsvCmfT6c69ZRv3IV9StXNr3e0Nq7N5G5uUSOPR3H6NEYsbEt7u/2u1lbvJaPiz7mi+Iv2FG1A43GrMxkJ2UzqscoRnUfxdDkoURbo0/ilQkR/rpM0H+8vYxr/vklr944ltP7JLZDZV2LZ88e6ld/TP1HH+HMy0M3NoJS2LOziRyXS9S4cUQMG4Yym1vcv9ZTy4bSDawtWUteSR6byzfj0z4Uin7x/RiSNITUqFS6ObqREplCdlI2kZaWu4yEEEfWZYJ+ydpCfv3aBlb+ZgK9kyQw2pL2eHBt3EjD52to+OwzXBs2QCCAcjiIGDoUx8gRRIzMIWL48MP27zu9TjaUbWB92XrWl67n24pvqXZXN603lEFWYhY5PXIYkTyCYd2GkWBPOFmXKESn1mWC/plVO3n0va1svv8CIm0ttzJF2/DX1NDw+RqcX32F8+t1uLd+B4EAGAb2wYNxjByBPTsbe1YW1szMFkf0ADT6GilzlVFQW8Da0rXkFeexsXwjvkDwvbk9o3syIGEA/eL6cVr8aQxNGkr3yPCarE6ItnCkoA+rNCytayTSakjInwRGbCwxky8gZvIFAPjrG3BtWI8zLw/XV3lULXwV7X4JCM7NY+vfH9vAAdgHDsQ2YAD2/v0xRUZiN9vJiM4gIzqDM9POBILh/23Ft6wvW8/Gso18V/kdH+75EE2wUZIRnUFO9xz6x/cnPSqd9Oh0esX0wmq0/EyAEF1dWLXob31lHZuLalh15zntUJU4Ftrnw71zF42bNtH47bc0frcV93fbCNTVNW1j6dkT22mnYevbF1u/04gYORJLWlqLD2K5fC52Vu/k69KvySvOY13puoO6fcwmMwPjB5KdlE3v2N4kRSSRFJFEN0c3ukd2x2KynJTrFqKjdJmumyv+3+egYdHNZ7RDVeJEaa3xFu3Dve073N99R+N32/Ds3IF7dz74gl015h49cOTkYO3bB2tGRvCTmYkRF3fIsWrcNRTWF7K3di9bq7ayqXwTm8s34/Q5D9rWUAbdHd3JiMmgX1w/+sf3D3YHxfeTXwAibHSZrpuyOjeDU2M6ugxxGEoprOlpWNPTiD733Kbl2uvFvWsXzrVrceXl4Vy3jtplyw7a14iPx9qnD/bBg4kYPgzH8OHEpqYSZ48jOymbKUwBgk/yVrurKXOWUe4qp8RZQlF9EUX1ReTX5LN422Ia/cEZTu2GncGJgxmQMIBEeyLx9niSI5LpG9eXtKg0DJNx8v5xhGhHYRX0pbWNTBiQ3NFliGOkLBbsAwZgHzAAfvQjAAKNjXgLC/Hs3Ytndz6e/N24d+6i+rXXqHr55eCOZjNGXBzm+DisffriyBlJRE4Ocf36kZCQwAAGHHIuf8BPYX0hWyq2sKFsAxvLN7J051LqvQdPz2w37PSO7U2fuD70julNr5hexNvjibPFEWeLo5ujm8z1IzqNsOm6CQQ0C78qoF/3KEZnypC8cKW9Xhq/24Zr4wZ8JaX4q6rwVVbg/nYL3n37mrYzxcZiTkrC0qMH1l49sfTsiSUtDUv37pi7dcOclHTQ+H+P30O1u5rihmJ2Vu9ke/V2dlbvJL8mn30N+w6pI9oazcCEgQyIH0CPyB4kRiSSFJFEWmQaKVEpmE1h1YYSnUCX6aMXXZt3/36ca9fhLSzAV1qGr6wM7/79ePbsOegmMABmM5a0VKw9e2Ht1QvbaX2xnXZa8H5AfDzK+L7bxuVzUVhXSLW7mmp3NeWucrZXbWdr5Va2V21v6go6wFAGqVGpZMZk0ie2D33i+pAckUyMLYYYawzdHd1lymfR5rpMH73o2iwpKcRefNEhy7XW+Kur8e3fj7ekBF9pGd59+/AW7MWzZy81a9cScDa7gasURmwsRmIilh49MKf0IC41le59+mI7rS/WPuNRoemdtdbUeeuocFVQ7iqnsK6QgroCCuoK2F2zmy+Lv2zx3bxJEUn0jO5JWlQaPSJ7NH2SI5JJdiSTYE/ApNrqBXCiq5OgF2FPKYU5Ph5zfDz2wYMPWa+1xrd/P+6dO/Hs2Yu/shJfVSX+8gq8xcU0bvsOf1n59zsYBkZCPOakZMxJSZi7JRPVvTtx3XswOCMdS88xWIb1QBkG/oCffQ37qHBVUOuppcZdQ4mzhL21e9lTu4evSr6izFmGX/sPqslispAWlUZaVFrTNBHdHN1Iikgi1hZLnC2OWGssMbYY+YUgjkqCXnR5SiksqalYUlPhrJa3CbhceHbvxr1jB+5du/CVl+Mvr8BXVoZ761Z85eXQrBtUWSwYiYkY8fEYcbEkxyfQIzERc2IiRmIC5sRzMKclYk5KQifEUemvpcRZQpmzjFJnKcUNxRTWF1JYV8jmis0HPTPQnEmZiLXGkhiRyODEwQxJGkJ2UjY9InsQb4uXkUMCkD56IdqE9vnwlZXh2VuAZ08+3r178VVU4q+qCnYbVVXir6gkUN/yy9dNsbFYunfHkpGBNT0dc48eGDHRmKKjg88QdE+iKsZEha+GGvf3nwP3DUoaSthYvpHKxu/f0mlSJuJt8WREZ5AZm0mvmF44zA7MJjMWk4XEiERSIlNIjUrFYXbIKKJOri1eJTgZ+F+CrxL8h9Z63g/W/wr4KcFXCZYBN2it94TW+YFvQpvu1VpPPdr5JOhFuAo0Nga7hioqgn8VVAT/KvCVleEtLsFbUICnsBDtch26s1IYiYmY4+MxEhIwJyZizeyFtXdvLOnpmCIjKaOO7e5CSswNVDRWUu4qZ0/tHvJr8yl3lR96zBCzMhNhicBhdhBtjW7qHoq3x5NoTyQxIpHUyFR6x/YmNSpVRhWdgk705eAGwZeDnwcUEnw5+FVa62+bbXMO8IXW2qmUugWYoLWeFVpXr7WOOpaCJehFV6a1JlBXR6CuDn9dXfCXQnEx3v3F+EpKgn8dVFXjKy3FW1QUnEzuB5TdjiUlBUtKCkZCAkZcHIFoB/4IK9puIxBhpSbKRGmMptDRSK124vQ6afA2UO+tp9pdTY27hsrGSqoaq5rmGYLg/YOM6AzSo9NJj0onwZ6ARqO1xmpY6RXTi8yYTFKiUvBrP16/F5MyEWeLk78a2tGJjroZA+zQWu8KHWwhMA1oCnqt9cpm268Brj7+coXo2pRSGDExGDExHG2ChoDHg7egAG9hIQGnk4DTRaC+Du/+4uDIouJiPAX/v717i42jvAI4/j87uzO79np9SYiT2AQSGtGiSm0oakFUpaJUAlqVPvBALyoPSLy0Kq0qVVQ8tU+9qTepQiCgpVVFKy6iERJFTaDqUylJQdxLcGixnRDbiXftXXtvM6cPMzYLsWObeNn02/OTRrszu+v5To5zxvN9s9+MExaLp11i6gOjwKgI3tYtZLbvILN9GG9gkFR+J6l8L6lsDvUzLKYazAYhE701xrJzjOkJJivHOHziMJVGZV1x5dI5RvIjjPaNMtwzzLaebZyXOy8eWA763x5kDvrtjGGTredfcwQYb1mfAD5xhvffAjzesp4VkUPE3To/VNVHN9xKY8yKUr4fTwp30UVrvlfDkGhxkaiyQFSp0Jw6QWPyWHJAOE7zrRPUjr5BOPccUblyWvdRBtidLJ8NgmRQeRepwQG8/n7SAwOEfT0Ue2AqW2Mm20D7+5D+AvVen8mFY4zPjzMxP8GzU89SqpVWbWuf30da0iiKIOzM74y/oDZ0MQW/QKghYRQymB1kT/8em7JiDZt62BSRrwKXAVe1bL5AVSdFZA/wpIi8oKpjK3z2VuBWgF27dm1ms4wxgHgeXj6Pl497UoM9Z76vsjabRNUaWq+htRrh7Cz1yUkak5M0p6bj8YVT8SBz4+gbhKUS0fw8KWB7srT6WC5HamnJb0V6d9HIZWgWeqgVsiz0+ZRzUMpGnEo1qGc9mtkMjWyaseYxDrx5gIePPLxiW5cGl/2Uj+/5FPwCo32jjORHGAgGUJRIIzKpDEPZIbbktjCUHWIgGKDgF5w/SKyn0E8CrXeHHk22vYOIXAPcAVylqsvfEFHVyeTxqIj8DdgHnFboWORr6AAABsBJREFUVfVu4G6I++jXH4Ixph0kncbLp4H4jmGZHTtW/B5CK63Xac4WCU+dpDlzkrBYXL7yKFpcJFpciLuYkrOKzOwccnSc1MmTZOt1zjR5iQQB9PZBNoDAR3yfZm/AfCHDyZ6IuSCknlZqnlJKzTDujXFAisznYCGAxQAaHrDCOEGf30fBj7+53Of3kU1nCbyAnnQP23u3M5KPp7bIelkyqQzpVJpsOksunSOXzpHP5M/p8Yf1FPpngL0ispu4wN8EfLn1DSKyD7gLuFZVp1q2DwILqloTka3AlcCPN6vxxphzi/g+meFtZIa3behzqkpUqSQHhmJ8dlAuE1UqRJX4MSyX4y6l6iJRrY5Wq4TFIsGRaQamp9F6fe0deR4EaSI/Q5jxCH2PRkao+SnKfQ3m8rMUe05RlZC6hCxQ59VUmUOBshAIDQ+aybIQwHwOGhlZHn/Ymd9J1ssuD04PZYfiifH6dy/fFlOQ5QNF4AXk0jl8z2/rlNlrFnpVbYrIN4AniC+vvE9VXxKRHwCHVHU/8BMgDzyYHNWWLqP8EHCXiERAiriP/uUVd2SM6Voi8na30ujohj+vqtBoENXraK1GVC4n31+YJSqVCCsVonKFqFJBq1WiWhWt1t5+rFRozszQfG369HmR1hD5aWq9IQu5CeaCN6n50EynCNNCKVVjMtPgSCAs+lDLxMtiAOWcUM7CXA+Uc5BKpRnuGeaJG5/YcPxrsS9MGWNMC63X0WYTDUO0Xo8PGvNlovJ8/FqjEY9ZzJcJS6X4LGQufoyKJaJqNT7Y1GtElQXCchmq1TPvMyXUCzkWtw9wxaMH31O7bVIzY4xZJ/H95UnrANiy5ax/pjYayRhFFV1cICwn3VSlIuGpWZonZ2jOzLStn98KvTHGtJlkMniZDF6hM3fAs2nvjDHGcVbojTHGcVbojTHGcVbojTHGcVbojTHGcVbojTHGcVbojTHGcVbojTHGcefkFAgiMg389z1+fCuw+j3T3NSNMUN3xt2NMUN3xr3RmC9Q1fNWeuGcLPRnQ0QOrTbfg6u6MWbozri7MWbozrg3M2brujHGGMdZoTfGGMe5WOjv7nQDOqAbY4bujLsbY4bujHvTYnauj94YY8w7ufgXvTHGmBZW6I0xxnHOFHoRuVZE/i0ir4vI7Z1uT7uIyPki8pSIvCwiL4nIbcn2IRH5q4gcSR4HO93WzSYinog8KyKPJeu7ReTpJOd/EhF/rZ/x/0ZEBkTkIRF5VUReEZErXM+1iHw7+d1+UUQeEJGsi7kWkftEZEpEXmzZtmJuJfarJP7nReTSjezLiUIvIh7wa+A64BLgSyJySWdb1TZN4DuqeglwOfD1JNbbgYOquhc4mKy75jbglZb1HwE/V9UPALPALR1pVXv9EviLqn4Q+Ahx/M7mWkRGgG8Cl6nqhwEPuAk3c/1b4Np3bVstt9cBe5PlVuDOjezIiUIPfBx4XVWPqmod+CNwQ4fb1BaqelxV/5U8nyf+jz9CHO/9ydvuB77YmRa2h4iMAp8D7knWBbgaeCh5i4sx9wOfAu4FUNW6qhZxPNfEtzjNiUga6AGO42CuVfXvwKl3bV4ttzcAv9PYP4ABEdmx3n25UuhHgPGW9Ylkm9NE5EJgH/A0MKyqx5OX3gKGO9SsdvkF8F0gSta3AEVVbSbrLuZ8NzAN/CbpsrpHRHpxONeqOgn8FHiTuMCXgMO4n+slq+X2rGqcK4W+64hIHngY+JaqzrW+pvE1s85cNysinwemVPVwp9vyPksDlwJ3quo+oMK7umkczPUg8V+vu4GdQC+nd290hc3MrSuFfhI4v2V9NNnmJBHJEBf5P6jqI8nmE0uncsnjVKfa1wZXAl8Qkf8Qd8tdTdx3PZCc3oObOZ8AJlT16WT9IeLC73KurwHeUNVpVW0AjxDn3/VcL1ktt2dV41wp9M8Ae5OReZ948GZ/h9vUFknf9L3AK6r6s5aX9gM3J89vBv78fretXVT1e6o6qqoXEuf2SVX9CvAUcGPyNqdiBlDVt4BxEbk42fQZ4GUczjVxl83lItKT/K4vxex0rlusltv9wNeSq28uB0otXTxrU1UnFuB64DVgDLij0+1pY5yfJD6dex54LlmuJ+6zPggcAQ4AQ51ua5vi/zTwWPJ8D/BP4HXgQSDodPvaEO9HgUNJvh8FBl3PNfB94FXgReD3QOBiroEHiMchGsRnb7eslltAiK8sHANeIL4qad37sikQjDHGca503RhjjFmFFXpjjHGcFXpjjHGcFXpjjHGcFXpjjHGcFXpjjHGcFXpjjHHc/wCg3MgUw5PfAQAAAABJRU5ErkJggg==\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"IvyC1A4Nm81q"},"source":["Training and validation accuracy improve consistently, but reach plateau after around 60 epochs"]},{"cell_type":"code","metadata":{"id":"89wGTNv0m81r","executionInfo":{"status":"ok","timestamp":1604685663946,"user_tz":420,"elapsed":1031814,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"8a58f27e-1b76-41a4-9574-7220cb03b7c7","colab":{"base_uri":"https://localhost:8080/"}},"source":["results = model.evaluate(X_test, y_test)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["313/313 [==============================] - 0s 1ms/step - loss: 0.1866 - accuracy: 0.9481\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"2OlzhJIrm81t","executionInfo":{"status":"ok","timestamp":1604685663947,"user_tz":420,"elapsed":1031809,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"e00f0778-d942-46d6-b8a6-9a14d100e740","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Test accuracy: ', results[1])"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Test accuracy:  0.9480999708175659\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"M_n7L3yrm81w"},"source":["## 5. Dropout (Regularization)\n","- Dropout is one of powerful ways to prevent overfitting\n","- The idea is simple. It is disconnecting some (randomly selected) neurons in each layer\n","- The probability of each neuron to be disconnected, namely 'Dropout rate', has to be designated\n","- Doc: https://keras.io/layers/core/#dropout"]},{"cell_type":"markdown","metadata":{"id":"cgmuU0_Em81w"},"source":["<img src=\"https://image.slidesharecdn.com/lecture29-convolutionalneuralnetworks-visionspring2015-150504114140-conversion-gate02/95/lecture-29-convolutional-neural-networks-computer-vision-spring2015-62-638.jpg?cb=1430740006\" style=\"width: 500px\"/>"]},{"cell_type":"code","metadata":{"id":"idwYUgbqm81x"},"source":["from keras.layers import Dropout"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"H6sbqxj0m81z"},"source":["def mlp_model():\n","    model = Sequential()\n","    \n","    model.add(Dense(50, input_shape = (784, )))\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dropout(0.2))                        # Dropout layer after Activation\n","    model.add(Dense(50))\n","    model.add(Activation('sigmoid'))\n","    model.add(Dropout(0.2))                        # Dropout layer after Activation\n","    model.add(Dense(50))\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dropout(0.2))                        # Dropout layer after Activation\n","    model.add(Dense(50))\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dropout(0.2))                         # Dropout layer after Activation\n","    model.add(Dense(10))\n","    model.add(Activation('softmax'))\n","    \n","    sgd = optimizers.SGD(lr = 0.001)\n","    model.compile(optimizer = sgd, loss = 'categorical_crossentropy', metrics = ['accuracy'])\n","    \n","    return model"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"JWT0Rcwom811"},"source":["model = mlp_model()\n","history = model.fit(X_train, y_train, validation_split = 0.3, epochs = 100, verbose = 0)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"28sHxARXm813","executionInfo":{"status":"ok","timestamp":1604685863059,"user_tz":420,"elapsed":1230909,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"7361997b-5c42-4e85-9e4d-43446e488a72","colab":{"base_uri":"https://localhost:8080/","height":268}},"source":["plt.plot(history.history['accuracy'])\n","plt.plot(history.history['val_accuracy'])\n","plt.plot(history.history['loss'])\n","plt.plot(history.history['val_loss'])\n","plt.legend(['train acc', 'valid acc', 'train loss', 'valid loss'], loc = 'upper left')\n","plt.show()"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXQAAAD7CAYAAAB68m/qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwU9f3H8dd3dje72dwXRyAYDpUQiCCHCFSxXiACoiK04EFVWmu11mql+vvh2Vb9eaKtipaeCioqgkVRWhGsRwsKyKXckgRC7nPv+f7+mE0MISQBcpDl83w81uzOfGfmOzvy3tnvfuc7SmuNEEKIzs/o6AoIIYRoHRLoQggRISTQhRAiQkigCyFEhJBAF0KICCGBLoQQEaLZQFdKZSilPlRKbVFKbVZK/byRMmOVUuVKqfXhx9y2qa4QQogjsbegTBD4pdb6C6VUHLBOKfWB1npLg3JrtNaXtn4VhRBCtESzga613g/sDz+vVEptBXoADQP9qKSmpurMzMzjWYUQQpx01q1bV6S1TmtsXkvO0OsopTKBIcDnjcw+Wym1AcgH7tBab25qXZmZmaxdu/ZoNi+EECc9pdTeI81rcaArpWKBN4DbtNYVDWZ/AZyita5SSl0CLAFObWQds4HZAL169WrppoUQQrRAi3q5KKUcWGH+stb6zYbztdYVWuuq8PPlgEMpldpIufla62Fa62FpaY1+YxBCCHGMWtLLRQF/BLZqrZ84Qplu4XIopUaE11vcmhUVQgjRtJY0uYwGrga+UkqtD0+7G+gFoLV+HrgSuEkpFQQ8wHR9DMM4BgIBcnNz8Xq9R7uoqMflctGzZ08cDkdHV0UI0Y5a0svlY0A1U+ZZ4NnjrUxubi5xcXFkZmYSPuEXR0lrTXFxMbm5ufTu3bujqyOEaEcn1JWiXq+XlJQUCfPjoJQiJSVFvuUIcRI6oQIdkDBvBfIeCnFyOuECvTmBUID91fsxtdnRVRFCiBNKpwv0mmANJZ4SCqoLWn3dZWVl/OEPfzimZS+55BLKyspauUZCCNFynS7QE5wJpESnUOItoczbugHaVKAHg8Eml12+fDmJiYmtWh8hhDganS7QAbq6u+J2uMmvzscT9LTaeufMmcPOnTsZPHgwd955J6tWreJ73/sekyZNYsCAAQBcdtllDB06lOzsbObPn1+3bGZmJkVFRezZs4esrCxuvPFGsrOzueiii/B4Dq/jsmXLOOussxgyZAgXXHABBQXWN46qqipmzZrFoEGDyMnJ4Y033gDgvffe48wzz+SMM87g/PPPb7V9FkJEjqMay6U93b9sM1vyG44w8B2Nxhv0AAewG3YMpVAYhC9vouF/AQakx3PvxOwjrvPhhx9m06ZNrF9vdbdftWoVX3zxBZs2barrArhgwQKSk5PxeDwMHz6cK664gpSUlEPWs337dhYuXMiLL77IVVddxRtvvMHMmTMPKTNmzBg+++wzlFK89NJLPProozz++OM8+OCDJCQk8NVXXwFQWlpKYWEhN954I6tXr6Z3796UlJS06D0UQpxcTthAb45C4bS58IV8BEL+5ksrKPdpcitzsRk27MqOzbChUGitMTEp95VjapOQGcJm2AAYMWJEXZhrrXn66adZsmQJAPv27WP79u2HBXrv3r0ZPHgwAEOHDmXPnj2H1Sg3N5dp06axf/9+/H5/3TZWrlzJokWL6solJSWxbNkyzjnnnLoyycnJR/1+CSEi3wkb6E2dSTcUMkP4TT+BUAAT0wpobaIJ/9WaoBkkYAbwBD0EzWCjvWQKqgsImAG2lWzDUAbfVnwLUbC9dDshHeKzNZ+x7L1lLHhnAW63m2snX8vOop10r+hOSIcoqC7A6/Fij7JT7CnG1CaekAeP10OVvwqH4cBQBoYyuOWWW7j99tuZNGkSq1at4r777mvFd08IcTI6YQP9aNgMG9FGNNH26BYvY2qToGn90KmUwsAgyUzCV+Oja0xXgmYQt8ONTdmItkdjKAPlVaSmpNIzuSfffP0NG9duxK7sBM0gWmvKfeVUeaoImkEOVB8AoCpQRY2/hr0Vh454WVBcgD/Gz9clX/PM/GfwBD1sK9nGkDFDePDxB5n78Fzsyk5FWQXpA9L58KMP+Xjjx/Tu05uK0opDvhUoFEqpur8A1YFqXt32KnbDjsPmwGE4cNlcuOwu3A43ya5kUqNT696zkBkiqIN1HzpCiM6n0wW6DoXQgQDKZgObDWUcW/gYyiDKFnXItPQu6YwZPYaxw8cyfvx4JkyYgMvuomdcTwCmXzadRX9exNhhYzn99NMZOXIkXWO60jexL3bDzmnJp1HhqMBpc3J68ukYyqCruysVoQoyEzIJmAFMbWJqkzn/O4df3vBLEhITGHXOKPL35RMfFc+v7/41c34xhwmjJmAYBrfedSsXT7qY3zz1G2bPnI1pmiSnJvPi4heb3L9yXzkPfflQs++D0+asC/NadcFvdxMTFUOsIxa3w239tbsJ6RDeoBdfyAeA3bBjUzZshg2H4cBu2Im2RxMXFUecIw6nzYnGGtonyhZFQlQCCc4E4p3xRNuicdqdOAxH3TcqhSI+Kh6H7dCxaLTWctGUEE1QxzCGVqsYNmyYbniDi61bt5KVldXkcqHycvz79tW9VoYBSkHtP/Twc1V/Guq730aV+u51bdl6y323YnXYX1V/XYet+9BtKBop08g6rT+NTG+wT41uO7xdjfUjce1fpRRff7OdLl3iCBmaACZ+FcKn/XhDPqoD1RR7iin2FlPuK8dhOOrO4v0hP96gF0/QQ02whupANVX+KqoD1VQHq6n2V2MzbLjsLlw2FwBBHSRoBq0PBtN67gl6qAxUNnEkmxdtt751BUIBvCEvATNArCOW+Kh468PAHl334VP712lz4rBZHyoOw4Fd2a0PnPDvJrXfWFw2F3FRccQ6Yol2WOuJskXhtruJd8bjMGRgM3FiUkqt01oPa2xepztDV243URkZ6GAQHQpBKARaY30uadD1H9Y060/4g6t2utZgmuFJGl1/fnhV1voa/A2X0Q1en2jMgoMUX3nVIdMMwG0YxNhsdLHbUXb7od90DANsBspmt16H5yu7Pfw8FmVLqJuO3WaVbey53YG2GQQNjWkQXsaOaVN4VYAa/HhViIBhElAmQUODzaqTaUA1Pqp9Pmr8AWxOJ/ZoFyoqimp8VISqqfB7qA74qKaCMoqowYc35LN+JDcDBEIBAmag7pvB0YpxxOC0Oet+e1EoEpzWN4tYRyx2w/pwcNqcxDpirW8j4Uft69oPGJfNhdPuxGlz1pWPccTItw3R6jpdoBsOByQkdHQ1DnP4B4KuP9P6EGhsfiN/db1lmirb6DbDz20eD13uugvMEDoYQgcDEDLRpvUhaE0LQiiIDplg1s4L/w2GrOatUBACQbRpWmUDQbTfjxkKWmVME4JB6wPWDFllg8Hvpoes7ehgEAKBurfEHX60JhUVZT0cDlRUXPi1AxxREGUHhwMcdnSUA9NhI2hX+B2KkN0gEKUI2g28dk2NLUS1LYDXrsEZhXI6CUYZlHu9lPm8lBkePHZNuSOEhwBVwWoq/ZVUB6pbXFdDGYeEf1xUnPXNIyqeBGcCSa4k0qLTSI1OJcGZQLQ9GrfdTVxUHG5Ha79zIlJ0ukA/UR3WbNJYmXaqC4Bx8CAps65rxy22TG2460Ag/AhCMHDINy4dCHxXzu+3HoHAd38DwQav/XXlzEPKBw5d1u9He3yY5VUonw/D58Pu86FrH/7mur82wmbDcLsxoqNR7iSIicZ0uwhFR9U9gtEOAtEO/C47PqdBjRPKHSFKHQHKHQGK7T7KQtYP5xW+Csr95XW/TzQm2h5NkjOJLu4u9IrvRc+4nqTHpNc1RSVEWR8Iic7Euu634uQggS7alQo39eBydXRVDqNNE+3zYXq9aI8H0+Oxnvt8mB6P9bfGg+mpsebXhMt4ajBratA1NYSqqzGrqjEPlGJWV2NWVRGqqrKaBpugHA6MxARsCQnYErpDfCz+BDeeWAeeuCi8sVHUxNopj1EcjPZTYPew33OAz/d/ztKdSxtfJ4okVxLpMen0jOtJRlwGp8SfQmZCJqfEnUKCM0GafSKMBLoQYcowUNHRGNHRkJTUauvVWlsfBlVVhCorMSsrCVVUYlaUE6qoJFRRYT0vryBUXk6orIxQ3n74qoSo0lKiQiEOa2S02bCnpmJPS8NI64c/KQ5/ahyeZDcVyS6KE20UxoYo9BWTV5nH5uLNrNy78pDeTHZlJ8GZQHJ0MlnJWeSk5jAobRCZ8ZnSrNNJSaAL0caUUiiXC8Plwp562L3Tm6RN0wr50lJCxcUEi0sIFhYSLCqy/hYWEsw/gP5iPUZ5OTFADNAdwG7H0b07URk9cWSMwNYzncquceSnKPbEeCgxKyj1llLoKeTjvI8POdNPciaRHptOv8R+ZKVkMSBlAP0S+xEXFdeab41oZRLoxyk2Npaqqiry8/O59dZbWbx48WFlxo4dy2OPPcawYY32NBLiiJRhYE9Kwp6UBH36NFnWrKkhcKCAwP58Anl5BHLzCOTuw5+bh/f99wmVlgKQCqTabESdcgrOvn2J6ncaztMnUpGTzGZnEbnV+eRX5ZNbmcuavDW8vfPtum10ie5Cn8Q+nJF2BmN6jGFQ6iBppz+BSKC3kvT09EbDXIj2YrjdOPv0xtmn8XvJhioq8O/Zg3/3bny7duPbuQPf9u1U/vOfdV14e7vdZA0YgGvgQFwDJ+MalUNZchTbSraxo2wHu8p3saNsBy9+9SIvbHyB+Kh4RqWPYkyPMYzuMZrU6KP7BiJalwR6PXPmzCEjI4Obb74ZgPvuu4/Y2Fh+8pOfMHnyZEpLSwkEAjz00ENMnjz5kGX37NnDpZdeyqZNm/B4PMyaNYsNGzbQv3//RofPBXjggQdYtmwZHo+HUaNG8cILL6CUYseOHfzkJz+hsLAQm83G66+/Tt++fXnkkUf4+9//jmEYjB8/nocffrjN3xMROWzx8UTn5BCdk3PIdNPnw7d9B75tW/Fu2Yp3yxZKFy5E+8JXAqel0XfIEAYNH477rGtxjj6VCn8Fn+7/lI9zP+bf+f/mvT3vAdAvsR8DUgaQlZzF8G7DOT359Hbfz5PZiRvo786BA1+17jq7DYLxRw7BadOmcdttt9UF+muvvcaKFStwuVy89dZbxMfHU1RUxMiRI5k0adIRewg899xzuN1utm7dysaNGznzzDMbLfezn/2MuXPnAnD11VfzzjvvMHHiRGbMmMGcOXOYMmUKXq8X0zR59913efvtt/n8889xu90yhK5oNYbTSfTAbKIHfjcgng4E8O3YgWf9emq+/BLPui+ofP99AGwpKcSMHMnZ53yPC0bfhjH6Ab4u+Zo1eWvYULiBT/M/rWuPz0nLYfrp07k48+LDhtoQre/EDfQOMGTIEA4ePEh+fj6FhYUkJSWRkZFBIBDg7rvvZvXq1RiGQV5eHgUFBXTr1q3R9axevZpbb70VgJycHHIanBHV+vDDD3n00UepqamhpKSE7Oxsxo4dS15eHlOmTAHAFe7et3LlSmbNmoXbbfU+kCF0RVtSDgeurCxcWVkk/eAHAATy8qj+/D9Uf/Yp1f/+hIp//AMAV04OaRdcwLUXXogzZzYAB2sO8v6e93n161e5++O7+e3nv+Xs9LM5p+c5jOkxRppm2siJG+hNnEm3palTp7J48WIOHDjAtGnTAHj55ZcpLCxk3bp1OBwOMjMz8Xq9x7Udr9fLT3/6U9auXUtGRgb33Xffca9TiLbk6NGDxMunkHj5FLRp4t26leo1a6j8578ofOIJCp94AueppxI/YQKJEy5h5oCZzMiawWf7P2PFnhWsyVvDB3s/wFAGo9JHMbnvZM7rdR5Om7Ojdy1iyDipDUybNo1FixaxePFipk6dCkB5eTldunTB4XDw4Ycfsnfv3ibXcc455/DKK68AsGnTJjZu3HhYmdrwTk1Npaqqqu4H1bi4OHr27Fl3Ew2fz0dNTQ0XXnghf/rTn6ipqQGQJhfRoZRhEJ2dTepPfkLv11+j34f/ous992DExVH41FPsvPAi9vzgh5S/+RZnJZ7BfaPuY+WVK3l94utcP/B6tpdu587Vd/L9177PI/95hF3luzp6lyKCBHoD2dnZVFZW0qNHD7p37w7AjBkzWLt2LYMGDeKvf/0r/fv3b3IdN910E1VVVWRlZTF37lyGDh16WJnExERuvPFGBg4cyMUXX8zw4cPr5v3tb39j3rx55OTkMGrUKA4cOMC4ceOYNGkSw4YNY/DgwTz22GOtu+NCHAdH9+4kXz2TzFdept8/V5L2y9sJlZWx/5572P69c9h///34d+2if3J/bj3zVlZcsYL5F85nVPooFn29iMlLJnP9iuv5ouCLjt6VTq3TDZ8rWkbeS9HRtNZ4vviCstdep+Ldd9F+PzGjRpF87TXEnHNOXaeCYk8xS3Ys4eWtL1PoKWRsz7H8/Myf0y+pXwfvwYmpqeFzJdAjlLyX4kQSLCmh7LXXKV24kGBBAc7TTydl9o3EjxtnDbkMeIIeXt76Mgu+WkBVoIpxmeOYnTNbgr2BpgJdmlyEEG3OnpxM6k9+TL+VH5D+yMPoYJD8X97BzksuoXzZMnQoRLQ9mhsG3cDyy5cza+AsPsr9iClLp3D7qtvZUbqjo3ehU5BAF0K0G+VwkDB5Mn2WLaXHvKcxXNHk3/krdl92GRUffIDWmkRXIr8Y+gtWXLGC2Tmz+ST/Ey5fejl3r7mb3Mrcjt6FE5oEuhCi3SnDIP6ii+j91pv0ePIJdDBE3i23svfqq/Fs2gxAoiuRW4bcwnuXv8d12dfx/t73mbhkIr/7/HeUeKWXV2Mk0IUQHUYZBvHjx9Nn2VK63Xcf/l272TN1Kvlzfk2wqAiwgv32Ybez/PLlXNbvMl79+lUuefMSXtjwAt6gXLtRnwS6EKLDKbudpOnT6LviPVKu/xHl//gHO8dfQsnLL1t3sgK6uLtw79n38ubkNzmr21k8u/5Zpi6byleFrTxESCcmgV5PWVkZf/jDH45p2UsuuYSysrIWl7/vvvukL7kQDdji4uhyxx30efttXAOzKXjwIfZcNQ3vli11Zfok9OHp7z/N/Avn4w15ufrdq3nmy2cIhAJNrPnk0GygK6UylFIfKqW2KKU2K6V+3kgZpZSap5TaoZTaqJRqfDSqE1xTgR4MBhudXmv58uUkJia2RbWEOOk4+/Sm14IF9HjicQIHC9g99SoOPvEkpu+7e62enX42b056kwl9JjB/43wufetSFn+z+KQO9pacoQeBX2qtBwAjgZuVUgMalBkPnBp+zAaea9VatpM5c+awc+dOBg8ezJ133smqVav43ve+x6RJkxgwwNrlyy67jKFDh5Kdnc38+fPrls3MzKSoqIg9e/aQlZXFjTfeSHZ2NhdddNERh8+ttX79ekaOHElOTg5TpkyhNHwjgnnz5jFgwABycnKYPn06AB999BGDBw9m8ODBDBkyhMrKyjZ6N4ToWEop4i+5hL7vvEPC5MkUz5/P7sumUPPll3Vl4qLi+M2Y3/DCBS+QGp3K/Z/ez6VvXcqyncvoqGtsOtJRX1iklHobeFZr/UG9aS8Aq7TWC8OvvwbGaq33H2k9zV1Y9Mh/HmFbybajqltz+if3564Rdx1xfv0xzQFWrVrFhAkT2LRpE717WzcNKCkpITk5GY/Hw/Dhw/noo49ISUkhMzOTtWvXUlVVRb9+/Vi7di2DBw/mqquuYtKkScycOfOQbdWOtX7HHXeQk5PDM888w7nnnsvcuXOpqKjgqaeeIj09nd27d+N0OikrKyMxMZGJEycyZ84cRo8eTVVVFS6XC7v98DHW5MIiEWmqPv43++f+L8EDBaRc/yNSb7kFI+q7IXm11vw7/988++WzbC7ezLCuw/ifkf9D38S+HVjr1tdqFxYppTKBIcDnDWb1APbVe50bntbpjRgxoi7MwTprPuOMMxg5ciT79u1j+/bthy3Tu3dvBg8eDMDQoUPZs2fPEddfXl5OWVkZ5557LgDXXnstq1evBqyhd2fMmMHf//73utAePXo0t99+O/PmzaOsrKzRMBciEsWOGU2fpUtJvOJyil98iT1XXHFI27pSijE9xvDKhFeYe/Zcvin9hiuXXsmzXz5LwDw5mmFanAZKqVjgDeA2rXXFsWxMKTUbq0mGXr16NVm2qTPp9hQTE1P3fNWqVaxcuZJPP/0Ut9vN2LFjGx3y1un8bjhQm83WbJPLkfzjH/9g9erVLFu2jN/85jd89dVXzJkzhwkTJrB8+XJGjx7NihUrmh0sTIhIYYuNpfuDDxJ34YXsv+d/2D1tOmk/+xkpN1xfN4SAoQymnjaV83udz+NrH+eFjS/wSf4nPPy9h+kV33TudHYtOkNXSjmwwvxlrfWbjRTJAzLqve4ZnnYIrfV8rfUwrfWwtLS0Y6lvm4qLi2uyTbq8vJykpCTcbjfbtm3js88+O+5tJiQkkJSUxJo1awBrpMVzzz0X0zTZt28f5513Ho888gjl5eVUVVWxc+dOBg0axF133cXw4cPZtq11m6WE6AxizzmHPsuWEnf++RQ++SR7r74G/759h5RJdiXzmzG/4bFzH2NvxV6uXHYlr3/zOqY2O6jWba8lvVwU8Edgq9b6iSMUWwpcE+7tMhIob6r9/ESVkpLC6NGjGThwIHfeeedh88eNG0cwGCQrK4s5c+YwcuTIVtnuX/7yF+68805ycnJYv349c+fOJRQKMXPmTAYNGsSQIUO49dZbSUxM5KmnnmLgwIHk5OTgcDgYP358q9RBiM7GlphIjyefIP3/HsW3fTu7L5tC2VtLDvsx9OLMi3lj0hvkpObwwKcPMOu9Wews29lBtW5bzf4oqpQaA6wBvgJqP9ruBnoBaK2fD4f+s8A4oAaYpbVe28jq6shoi21L3ktxMgnk5ZF/1xxq1q4lbtw4ut93L7YG3Yi11izZsYTH1z1OdaCaawdcy48G/Yj4qPgOqvWxkeFzT0LyXoqTjQ6FKF6wgMKn52FPTSX9kUeIOWvEYeVKvCU8vvZxlu5cSnxUPDcMuoEf9P8BLrurA2p99GT4XCFExFM2G6k33kjmokUYLhffXncdB598Ch04tIdLbdv66xNfJycthyfWPcHEJRNZsWdFp++7LoEuhIgo0QOz6f3mGyReeQXFL7zAnh/8EN/u3YeV65/cn+cueI4FFy8gISqBOz66gxvev6FTj70ugS6EiDiG2033Bx+kx9NPE9i3j92XX0Hpq681egY+vNtwXr30Ve456x62lWzjymVX8sh/HqHCf0y9szuUBLoQImLFX3wRvZe+jXvIEA7cey+5N/2UYGHhYeVsho3p/afzzpR3uPzUy3l568tMfGsib3zzBkGz6XGcTiQS6EKIiObo2pWMl16k692/pvrTT9k1cRIV77/faNkkVxJzz57LwksXkhGXwX2f3lc36Jc/5G/nmh89CfTjFBsbC0B+fj5XXnllo2XGjh1Lwx49TU0XQrQuZRgkX3MNvd98A0ePHuTd+nPyfvUrguGB8BrKTsnmb+P/xjPff4YkZxL3f3o/E96awJIdSwiZoXaufctJoLeS9PR0Fi9e3NHVEEI0wdm3L5mLFpJ6881ULH+XXZdOpGJF42frSinGZozllQmvWKM5ulL533//L1PfmcrHeR+fkD1iJNDrmTNnDr///e/rXtfehKKqqorzzz+fM888k0GDBvH2228ftuyePXsYOHAgAB6Ph+nTp5OVlcWUKVNaNJbLwoULGTRoEAMHDuSuu6xxbEKhENdddx0DBw5k0KBBPPnkk0Djw+oKIVpGORyk3fIzei9+HUfXruT9/Ofk3nIrgYKDjZdXilE9RvHKhFf4v3P/D0/Aw00rb+LHH/yYr0u+bufaN+2EHarvwG9/i29r645T4szqT7e77z7i/GnTpnHbbbdx8803A/Daa6+xYsUKXC4Xb731FvHx8RQVFTFy5EgmTZqEdYHs4Z577jncbjdbt25l48aNnHlm0/f7yM/P56677mLdunUkJSVx0UUXsWTJEjIyMsjLy6sbzrf2jkgPP/zwIcPqCiGOnqt/fzJfe5XiBX+i6Nlnqf70U7r88nYSp01DGYef6yqlGJc5jvMzzmfR14t4fsPzTF02lcn9JvPjnB/TM65nB+zFoeQMvZ4hQ4Zw8OBB8vPz2bBhA0lJSWRkZKC15u677yYnJ4cLLriAvLw8CgoKjrie1atX141/npOTQ05OTpPb/e9//8vYsWNJS0vDbrczY8YMVq9eTZ8+fdi1axe33HIL7733HvHx8XXrbDisrhDi6Cm7ndTZN9Jn2VJcgwZy4P4H2PvDGXibGPTOYXNw9YCrWX75cq4ZcA3/2PUPLn3rUu75+B52le1qx9of7oRNg6bOpNvS1KlTWbx4MQcOHGDatGkAvPzyyxQWFrJu3TocDgeZmZmNDpvb2pKSktiwYQMrVqzg+eef57XXXmPBggWNDqsrwS7EsYs65RR6LVhA+dtvc/DR/2P35VeQNHMGabfeii3c8aGhBGcCdwy/g6sHXM2fN/+Zxd8sZunOpZzd/WyuOO0Kvp/xfRw2R7vuh5yhNzBt2jQWLVrE4sWLmTp1KmANm9ulSxccDgcffvghe/fubXId55xzDq+88goAmzZtYuPGjU2WHzFiBB999BFFRUWEQiEWLlzIueeeS1FREaZpcsUVV/DQQw/xxRdfHHFYXSHE8VFKkXjZZfR9dzmJ066i9G9/Z+f48dYIjuaRh9ztGtOVu0bcxYorV/DTwT9ld8Vu7vjoDi5YfAHzvpjHwZrG2+bbgpzWNZCdnU1lZSU9evSge/fuAMyYMYOJEycyaNAghg0b1uwNJW666SZmzZpFVlYWWVlZDB06tMny3bt35+GHH+a8885Da82ECROYPHkyGzZsYNasWZjh/5l+97vf1Q2rW15ejta6blhdIUTrsCUk0P3ee0m8/HIOPPQQ+3/9a0pfeYWud/8a95AhR1wu2ZXMTWfcxOxBs/kk/xNe/+Z1XvrqJf60+U+MyxzHNQOuISulbQfMk9EWI5S8l0IcP22aVLPwBiwAABkzSURBVLzzDgcfe5zgwYPEXXQRabfdhrNP7+YXBvZV7OOVba/w1o63qA5Uc1b3s7gu+zpGp48+YqeK5shoi0IIcQyUYZAwaRJ9311O6i0/o/rjj9k1cSL75957xG6O9WXEZ3DXiLv44MoPuH3o7ewu281NK2/i0f8+2jb1lTP0yCTvpRCtL1hcTNFzz1P66qsom43kq2eScsMN2BISWrR8IBRg+e7l9EvqR3ZK9jHVoVOdoZ+IV191NvIeCtE27CkpdPufe+i7/B/EXXQhxS/9kR0XXkTRc88RqqpudnmHzcHkfpOPOcybc0IFusvlori4WALpOGitKS4uxuXqHHdfEaIzisrIoMejj9J7yVu4hw6l8Ol57LzgAopefBGzuvlgbysnVJNLIBAgNze3Xfp4RzKXy0XPnj1xONq3D6wQJyvPV19ROO8ZqteswZaYSPJ115I0Ywa2uLhW31anuaeoEEJ0Zp716yl67nmqPvoIIy6OpJkzSL7mGuxJSa22DQl0IYRoR94tWyh67nkqV65EuVwkXTWV5OuuwxG+tuV4SKALIUQH8O3cSfGLL1G+bBkoRfz48aTMug7XgAHHvM5O1ctFCCEihbNvX9If/h39Pnif5JkzqfrnP9l9+RUcfOLJNtmeXPovhBBtzJGeTtc5d5H605soe/11opsZgfVYSaALIUQ7scXHk3L99W22fmlyEUKICCGBLoQQEUICXQghIoQEuhBCRAgJdCGEiBAS6EIIESEk0IUQIkJIoAshRISQQBdCiAjRbKArpRYopQ4qpTYdYf5YpVS5Ump9+DG39asphBCiOS259P/PwLPAX5sos0ZrfWmr1EgIIcQxafYMXWu9Gihph7oIIYQ4Dq3Vhn62UmqDUupdpVTb3P1UCCFEk1pjtMUvgFO01lVKqUuAJcCpjRVUSs0GZgP06tWrFTYthBCi1nGfoWutK7TWVeHnywGHUir1CGXna62Haa2HpaWlHe+mhRBC1HPcga6U6qaUUuHnI8LrLD7e9QohhDg6zTa5KKUWAmOBVKVULnAv4ADQWj8PXAncpJQKAh5guu6oG5UKIcRJrNlA11r/oJn5z2J1axRCCNGB5EpRIYSIEBLoQggRISTQhRAiQkigCyFEhJBAF0KICCGBLoQQEUICXQghIoQEuhBCRAgJdCGEiBAS6EIIESEk0IUQIkJIoAshRISQQBdCiAghgS6EEBFCAl0IISKEBLoQQkQICXQhhIgQEuhCCBEhJNCFECJCSKALIUSEkEAXQogIIYEuhBARQgJdCCEihAS6EEJECAl0IYSIEBLoQggRISTQhRAiQkigCyFEhJBAF0KICCGBLoQQEUICXQghIoQEuhBCRAgJdCGEiBAS6EIIESGaDXSl1AKl1EGl1KYjzFdKqXlKqR1KqY1KqTNbv5pCCCGa05Iz9D8D45qYPx44NfyYDTx3/NUSQghxtJoNdK31aqCkiSKTgb9qy2dAolKqe2tVUAghRMu0Rht6D2Bfvde54WlCCCHaUbv+KKqUmq2UWquUWltYWNiemxZCiIjXGoGeB2TUe90zPO0wWuv5WuthWuthaWlprbBpIYQQtVoj0JcC14R7u4wEyrXW+1thvUIIIY6CvbkCSqmFwFggVSmVC9wLOAC01s8Dy4FLgB1ADTCrrSorhBDiyJoNdK31D5qZr4GbW61GQgghjolcKSqEEBFCAl0IISKEBLoQQkQICXQhhIgQEuhCCBEhJNCFECJCSKALIUSEkEAXQogIIYEuhBARQgJdCCEihAS6EEJECAl0IYSIEBLoQggRISTQhRAiQkigCyFEhJBAF0KICCGBLoQQEUICXQghIoQEuhBCRAgJdCGEiBAS6EIIESEk0IUQIkJIoAshRISQQBdCiAghgS6EEBFCAl0IISKEBLoQQkQICXQhhIgQEuhCCBEhJNCFECJCSKALIUSEkEAXQogIIYEuhBARQgJdCCEiRIsCXSk1Tin1tVJqh1JqTiPzr1NKFSql1ocfN7R+VYUQQjTF3lwBpZQN+D1wIZAL/FcptVRrvaVB0Ve11j9rgzoKIYRogZacoY8Admitd2mt/cAiYHLbVksIIcTRakmg9wD21XudG57W0BVKqY1KqcVKqYzGVqSUmq2UWquUWltYWHgM1RVCCHEkrfWj6DIgU2udA3wA/KWxQlrr+VrrYVrrYWlpaa20aSGEENCyQM8D6p9x9wxPq6O1LtZa+8IvXwKGtk71hBBCtFRLAv2/wKlKqd5KqShgOrC0fgGlVPd6LycBW1uvikIIIVqi2V4uWuugUupnwArABizQWm9WSj0ArNVaLwVuVUpNAoJACXBdG9ZZCCFEI5TWukM2PGzYML127doO2bYQQrSrkt1QshOqDlqP9CHQ59xjWpVSap3Welhj85o9QxdCCHGMCjbDh7+Fbe8cOn3ULccc6E2RQBdCiNYS8MCBTbB/Pez+CLa+A844OPcu6Pt9iEmD2K7gjG2TzUugCyFES2gNuWth/d9h33+h3/lwxg+gSxZ8+ymsXQBb3oaQ3yrvToUxv7DOxt3J7VJFCXQhhGhKKADrX4FPfw9FX4PDDd0Hw2d/gE/mWcFdUwTOBBh6HfQ+F9IHQ3wPUKpdqyqBLoQQ9WkN/iqoKYZdH8Gax6DsWyvEJ86D7CngiofqItj0Juz9GE69CLIvhyh3h1ZdAl0IIQD2/QfWPA47//VdswlA+plwyeNw6oWHnnHHpMJZs63HCUICXQhxcqkqtH6wLNtrvdYadq2CPWsgOhmGXQ/x3cGdAkm94ZRR7d50cqwk0IUQnYtpQsEmyP0vxHaBboMg8RSriWTf59YPl2YQopOsHyODfqg+CFUFkPclFHx1+DrjusPFv4Uzr22zHijtQQJdCNH+/DWwaTFsfx8q8qFiP/iroccQOGU0pPWHg1ut0D64BRzR1tmzIxr2bwBv2aHrc7ghUGM9N+ygbBDyfTdfGdaPl2mnw/lzoc95Vu8UFR79xHCA0flv4CaBLoQ4fp5S+O9LkL8evOXWA6x2Zneq1XwRnQiuBCjda/Ua8ZVbZ9bJfaBvFtgc1tn1h78FNKCs0M38nhXONSXgq4Csida0jBHWtAMbrfCPT4eMs6yrMB0u60PDUwI2p3Wmbtg68h1qFxLoQoiW0xoKt1mBbXdaZ8Ob34L/vGiFbVqWFdzx6VbZmiIo3mkFr7/SWofhgAGTYPgN0Ovsw9unPWVQvANST7N6kzQluTf0PMLgrlHuDu91AlDpDbApr4KCCi/F1X5Kq/0MzUzivNO7tPq2JNCFOBnUXsGYPgRsTfyzr9hvNXEUbbfGHjEc4E4CZzzs3wg7VkLVgQYLKRgwGc65E7oNPPK6Q0Er9A1700EdnQg9Gx2q5IRT6Q2wMbecggovRVU+iqv9+AImvqBJjT/IlvwKdhRWUX/ILEPBT8f2k0AXQhyDvZ/C2zdbAZ1yKpz3axgwxWozNkPWwFHblsHmJdYl67Wi4kCbEKi2XrsSoe950Pd86ww86IOgB7rlQOqpzdfDZm+3KyZbi2lqQloTMjUef4hvS2r4tqSGbwoq+WRnMev3lREyv0trh00R7bARZbfhchic2iWWS3PSOSMjgYxkNykxUcS7HBhG2/SakUAXIlLVlMBHj8Lnz0NiBox7GL74Kyz+ESQ9YIV1Rb7VIwSs/tYX3G+dHaeeZo07ohQEvFYTizul6bP7Tqq8JsCBCi9VvgCV3iD7Sj18ubeUL74tZU9xTaPLGApyeibyk3P7MLJPCj0So0mNcxLntKM6sItj5B0dIU5moQBs/wA2vALfrLAukBkxG86/1+qON2K21ea9YZHVrS+hJyT2sgaOSjql8XU6XNajk6r2BTlQ4eVAuZe8Ug/7SmvYV1LD3pIa9hRVU1oTOGyZ1FgnZ/ZKZNIZ6ThsBoahcDlsZCRF0yvFTa9kN+6oEy8+T7waCSGOXtF26+x7w0KoLrTOroffCIN/eGi7tmGDQVdajwgRMjW5pTUUVfko9wQorQ6wo7CKTXnlbMmvoLjaf0h5Q0H3hGgykqMZN7A7vVPdpCdGE+dyEOu00yXOSc+k6A490z5WEuhCdEamCfu/tM7Cv3nP6putbHD6eBgyE/pdYHUDjBAHK718sKWAf+8owh/UOGwKQyn2llSzvaAKX9A8pLzdUJzWNY7v9+9Cn7RYuie46BrvokdiNN0TXThsnb/PeWMk0IU4kVUVQt5a6yrHqoPWD5gHt0DRN9aFNMqAniPgwgcgZzrEde3oGh8TrTX55V425ZWzKa+cHQetkA6ETMo9Ab7KK0dr6JEYTXy0g2DIJKQ1PRKjuXrkKZzWNY6uCS4Soh0kRDtIT3ThtEd+v/OGJNCFOFGV7IY/Xmg1odSK7QZd+lvDtKYPsXqcxKR0WBVbwjQ1BZVe8su85Jd5KK7y4Q2aePwhSmv8bDtQybb9FVR4rR9nDQWZKTG4nTZshoE7ysZt55/GxQO7cnrXuE7ZFNJeJNCFOBFVF8Hfr7B6oFzzNqT0s9rF7c6OrtlhqnxB9hRVs7uomsJKH9W+IFX+IAfKvWwvqGJXURXegNnosnFOO6d1i+PSM9LJ6hZHdo8EsrrFEx118p1dtwYJdCHaW9BvtXtvftO64Ccq1uqB0mUA9BkLCRmwcDpU5ME1S6HXWR1a3Rp/kB0HrVC2GaCU4psDlXy+u4T/7C4hr8xz2DJOu0FqrJN+XWI5u28KvVNj6tqv02KdREfZcNpt2NqoP/bJSgJdiLZkmlC8HUp2WTdJKNxm3aaspti6t2RsV+tmCt4KWPdnaxmH2wr6aX9rlzCv9Fpt1Ov3lbE5vwJf+Gw6ZJrsKa5hT3H1IVc61kqNjWJE72RmjjyF3qluMlNj6BrnItZlj9gfHU90EuhCHA+tDx+LJBSAL/8Ouz6EPR9b4V3L4bZulDDkamvEv/oX6pTuse6Qs2eN1S88a2KLq1HjD1JS7ae0OkClN0AwfIWjaWrruanxBkLsL/eyv9xDfpmX/eVeDpR7DumHnZEcTZzT6h2jFJzeNY7LBvfg9G5xxLnshMLrykh20zctRtqzTzAS6EIci4LN8K/fWIF95QI49QJreigIb9wAW5ZAfE/r1mSZY6zhYBN7fXf1ZWOSMmFoJp5BM6nwBqgpqqbGH6TaF6LaF6TSF6TSG6DcE6DCE+RghZdd4bbrcs/hF8ccSaLbQfeEaNITXJzZK5H0xGiy0+M5o2ciSTFRx//eiA4jgS7E0ajYDx/Mha9eB2ecdYOFhdNgyguQPQXv4h/j2rqET/rcxu7TZpEW5yIlNoqAX1OTG6TKt5+yGj/FVX5Kqv1UegNU+YJUeoMUV/spqPBSGe7t0RSHTZEW6yQzNYZLc7rTM8kaJyQpJoo4lx27oTAMhU0pbIbCblNE2Qy6JbhOyCscRetQurHGsXYwbNgwvXbt2g7ZtohM1b4gBRVeguHBkhRgtxk4bKquTTcQMgmZmtKaAAfCTQ7eoEmUzSDKbpXxBkL4gia+QIiAqQkETap8QSj6mtsPzCFeV/Cm41Leck/FYbfz6/L7yQ5uZqPqz2C9lccCU3k2NKXZ+tb2mY5x2olz2kmJjaJrvIu0OCeJbgfuKBvRDjuxTjuxLjuxThtxLmsZp92Q5o6TlFJqnda60eEo5aP6JFR/BLmgqfGFAywY0qTERhHjPPR/C601Vb4gxVV+iqv9BEMmSimUAkMp7IZ1FmhqTSBk4g9a6za19dDaujy7dptWGZNASBM0re16AiFKq/2U1Pgpq7HagSu9QbyBEAnRDhLdUcS67FR6g5TV+OuaGGzhUCuq8tX1Y24pO0Gy1LdoYJvuRbDePwcDEzshsDmx2xRnOXbytPk7tGHn+cw/sMvejwR/CF8wxDMxj3Jz0UMMrvmELzNv4Nyx9zC7Wxw1vhCFlT6Kq31E2Q1iouzEOG0kuqNIjHZglx8ORSvrfIFumlCRiy78Gm/+FnyFu9BmENO07nFiKKyvmIbCYTdwGAZKWYFS4w9R4w+itVXGZihCWuMPWgHjD1nhEgiZKCDKbuCwGdhtCoWqa/oMmppQyAorlHUmCBAILxsyNXZDYbcZ2A1FsO6HKav3gEKhAU8gSI0vRI0/hKGw6mszsKlwKWUFls1m7U/IBH8oVBeGWlthqxTYDQObzfqKramdBwHzu30Kmtq6ws5s+luZw24Q7bBZAR3eltlG3+QMICr8SDEMnA7rTDnKFn7v7RDwmvgrgwRC1iXfUQ47DrsBStX1vnClGrij7EQ7rK5wGmuG1tT9OFh7DA0FSTV7SCjZiBHyWuUcbgLdhmC6knCU7sQo3YUK+SAu3Rq0Kn89JHWHmW9yW3Lvw3cktAwObmFIt0F1beTxLgfdEjrvoFai8+l0gb7l/RcZ8NmvUEA0ENBufPV2IxR+1A7Ho6gNUOufeP1zolD4b22gUG8ZgGOJMFX3X33Y8or6X5E1hrLaOQ2lQIP2hYO4XpnawKqdZiirH7CqW6c1T4cL1d+mwsqW2vLKZj1Qh9ZE1ftQMjXWmbUZXt4efiirnoaCQxZusN3aD73vitR7Vm+58GdW3ftVvw51zPADBY7wg/AnVbDBuxsEDu8OfWQJPWHYLOs2ZmjUt58Tte8zKCmwxgw/7QKrf3jZt1bvk77nwcR5EJvW+PpsduiecxQVEKL1dbpA1xlns3DXL/AlnYaRdjrxKd1wOQxshoHNAH/QxBsw8QRCVHmtXgEV3iCJbgc9EqNJT4zGZii8gRDegInTYdS1Zca7HMQ4bUQ7bGgNVf4gFZ4AHn+orrlAoay2zSgbDptR17SggFiXdYaolMI0rWYETyBEtMNaZ1sNai9awcArOroGQhy3Thfo2dk5ZGe3/ZmQUtZX5njXsY1YZxiKGKf9sPZoIYRoK/KrjBBCRAgJdCGEiBAS6EIIESFaFOhKqXFKqa+VUjuUUnMame9USr0anv+5UiqztSsqhBCiac0GulLKBvweGA8MAH6glBrQoNj1QKnWuh/wJPBIa1dUCCFE01pyhj4C2KG13qW19gOLgMkNykwG/hJ+vhg4X8l1yUII0a5aEug9gH31XueGpzVaRmsdBMqBw+6LpZSarZRaq5RaW1hY2HC2EEKI49CuP4pqredrrYdprYelpR3hijshhBDHpCVXveQBGfVe9wxPa6xMrlLKDiQAxTRh3bp1RUqpvUdR1/pSgaJjXLYzOxn3+2TcZzg59/tk3Gc4+v0+5UgzWhLo/wVOVUr1xgru6cAPG5RZClwLfApcCfxLNzMur9b6mE/RlVJrjzR8ZCQ7Gff7ZNxnODn3+2TcZ2jd/W420LXWQaXUz4AVgA1YoLXerJR6AFirtV4K/BH4m1JqB1CCFfpCCCHaUYsGGtFaLweWN5g2t95zLzC1dasmhBDiaHTWK0Xnd3QFOsjJuN8n4z7DybnfJ+M+Qyvud4fdgk4IIUTr6qxn6EIIIRrodIHe3LgykUAplaGU+lAptUUptVkp9fPw9GSl1AdKqe3hv0kdXde2oJSyKaW+VEq9E37dOzxG0I7wmEFRza2jM1FKJSqlFiultimltiqlzj4ZjrVS6hfh/783KaUWKqVckXislVILlFIHlVKb6k1r9Pgqy7zw/m9USp15NNvqVIHewnFlIkEQ+KXWegAwErg5vJ9zgH9qrU8F/hl+HYl+Dmyt9/oR4MnwWEGlWGMHRZKngfe01v2BM7D2PaKPtVKqB3ArMExrPRCrB910IvNY/xkY12DakY7veODU8GM28NzRbKhTBTotG1em09Na79dafxF+Xon1D7wHh46Z8xfgso6pYdtRSvUEJgAvhV8r4PtYYwRBhO23UioBOAer6y9aa7/WuoyT4Fhj9bKLDl+M6Ab2E4HHWmu9Gqs7d31HOr6Tgb9qy2dAolKqe0u31dkCvSXjykSU8FDEQ4DPga5a6/3hWQeArh1Urbb0FPArwreHxhoTqCw8RhBE3jHvDRQCfwo3M72klIohwo+11joPeAz4FivIy4F1RPaxru9Ix/e4Mq6zBfpJRSkVC7wB3Ka1rqg/L3wlbkR1UVJKXQoc1Fqv6+i6tCM7cCbwnNZ6CFBNg+aVCD3WSVhno72BdCCGw5slTgqteXw7W6C3ZFyZiKCUcmCF+cta6zfDkwtqv36F/x7sqPq1kdHAJKXUHqzmtO9jtS8nhr+WQ+Qd81wgV2v9efj1YqyAj/RjfQGwW2tdqLUOAG9iHf9IPtb1Hen4HlfGdbZArxtXJvzr93SscWQiSrjd+I/AVq31E/Vm1Y6ZQ/jv2+1dt7aktf611rqn1joT69j+S2s9A/gQa4wgiLD91lofAPYppU4PTzof2EKEH2usppaRSil3+P/32v2O2GPdwJGO71LgmnBvl5FAeb2mmeZprTvVA7gE+AbYCdzT0fVpo30cg/UVbCOwPvy4BKs9+Z/AdmAlkNzRdW3D92As8E74eR/gP8AO4HXA2dH1a+V9HQysDR/vJUDSyXCsgfuBbcAm4G+AMxKPNbAQ63eCANY3suuPdHwBhdWTbyfwFVYvoBZvS64UFUKICNHZmlyEEEIcgQS6EEJECAl0IYSIEBLoQggRISTQhRAiQkigCyFEhJBAF0KICCGBLoQQEeL/AQtwxGsF/cjzAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"R6YP4RHVm816"},"source":["Validation results does not improve since it did not show signs of overfitting, yet.\n","<br> Hence, the key takeaway message is that apply dropout when you see a signal of overfitting."]},{"cell_type":"code","metadata":{"id":"aAPAlxKEm816","executionInfo":{"status":"ok","timestamp":1604685863413,"user_tz":420,"elapsed":1231257,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"f85e06a9-e19c-40f6-9afd-4883e4523561","colab":{"base_uri":"https://localhost:8080/"}},"source":["results = model.evaluate(X_test, y_test)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["313/313 [==============================] - 0s 1ms/step - loss: 1.6782 - accuracy: 0.4227\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"47lVW1hXm818","executionInfo":{"status":"ok","timestamp":1604685863414,"user_tz":420,"elapsed":1231250,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"2c690eb9-22c7-4e51-c3d3-5aa2b061b505","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Test accuracy: ', results[1])"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Test accuracy:  0.4226999878883362\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"YtjeI1Erm81-"},"source":["## 6. Model Ensemble\n","- Model ensemble is a reliable and promising way to boost performance of the model\n","- Usually create 8 to 10 independent networks and merge their results\n","- Here, we resort to scikit-learn API, **VotingClassifier**\n","- Doc: http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.VotingClassifier.html"]},{"cell_type":"markdown","metadata":{"id":"TSAYvxCmm81-"},"source":["<img src=\"https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRs1CBSEtpp5yj6SJ5K_nHd1FNfyEYa9KLjWfoMY_v7ARTq3tdpVw\" style=\"width: 300px\"/>"]},{"cell_type":"code","metadata":{"id":"Pb8HFb6Tm81_"},"source":["import numpy as np\n","\n","from tensorflow.keras.wrappers.scikit_learn import KerasClassifier\n","from sklearn.ensemble import VotingClassifier\n","from sklearn.metrics import accuracy_score"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"kKsR6al3m82B"},"source":["y_train = np.argmax(y_train, axis = 1)\n","y_test = np.argmax(y_test, axis = 1)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"nRxuL1cwm82D"},"source":["def mlp_model():\n","    model = Sequential()\n","    \n","    model.add(Dense(50, input_shape = (784, )))\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(50))\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(50))\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(50))\n","    model.add(Activation('sigmoid'))    \n","    model.add(Dense(10))\n","    model.add(Activation('softmax'))\n","    \n","    sgd = optimizers.SGD(lr = 0.001)\n","    model.compile(optimizer = sgd, loss = 'categorical_crossentropy', metrics = ['accuracy'])\n","    \n","    return model"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"NsC1_PiCm82E"},"source":["model1 = KerasClassifier(build_fn = mlp_model, epochs = 100, verbose = 0)\n","model2 = KerasClassifier(build_fn = mlp_model, epochs = 100, verbose = 0)\n","model3 = KerasClassifier(build_fn = mlp_model, epochs = 100, verbose = 0)\n","model1._estimator_type = \"classifier\"\n","model2._estimator_type = \"classifier\"\n","model3._estimator_type = \"classifier\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2pBnptVLm82H"},"source":["ensemble_clf = VotingClassifier(estimators = [('model1', model1), ('model2', model2), ('model3', model3)]\n","                                , voting = 'soft')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"vOx2UEAjm82I","executionInfo":{"status":"ok","timestamp":1604686456466,"user_tz":420,"elapsed":1824279,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"8ff8b858-c620-4bed-fe25-806aedebd854","colab":{"base_uri":"https://localhost:8080/"}},"source":["ensemble_clf.fit(X_train, y_train)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["VotingClassifier(estimators=[('model1',\n","                              <tensorflow.python.keras.wrappers.scikit_learn.KerasClassifier object at 0x7f3b1c1f9438>),\n","                             ('model2',\n","                              <tensorflow.python.keras.wrappers.scikit_learn.KerasClassifier object at 0x7f3b1c1ce7b8>),\n","                             ('model3',\n","                              <tensorflow.python.keras.wrappers.scikit_learn.KerasClassifier object at 0x7f3b1c10d978>)],\n","                 flatten_transform=True, n_jobs=None, voting='soft',\n","                 weights=None)"]},"metadata":{"tags":[]},"execution_count":131}]},{"cell_type":"code","metadata":{"id":"uikYSUBmm82L","executionInfo":{"status":"ok","timestamp":1604686457386,"user_tz":420,"elapsed":1825192,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"44fa4445-330f-47aa-ff35-3190b964558a","colab":{"base_uri":"https://localhost:8080/"}},"source":["y_pred = ensemble_clf.predict(X_test)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/wrappers/scikit_learn.py:264: Sequential.predict_proba (from tensorflow.python.keras.engine.sequential) is deprecated and will be removed after 2021-01-01.\n","Instructions for updating:\n","Please use `model.predict()` instead.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"1_5lAEucm82O","executionInfo":{"status":"ok","timestamp":1604686457388,"user_tz":420,"elapsed":1825189,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"cca4d7a3-600e-47e7-d55f-d9620245d139","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Test accuracy:', accuracy_score(y_pred, y_test))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Test accuracy: 0.9002\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"UeDuUWLPm82Q"},"source":["## Summary\n","\n","Below table is a summary of evaluation results so far. It turns out that all methods improve the test performance over the MNIST dataset. Why don't we try them out altogether?\n","\n","|Model           | Baseline  | Weight initialization  | Activation function        | Optimizer        | Batchnormalization  | Regularization   | Ensemble   |\n","|----------------|-------------|------------|-------------|-------------|------------|-----------|------------|\n","|Test Accuracy   | 0.1134     | 0.8625     | 0.9488      | 0.9465      | 0.9480     | 0.4226    | 0.9002     |\n","\n","<br>\n"]},{"cell_type":"code","metadata":{"id":"MuUkwGu4T0si"},"source":[""],"execution_count":null,"outputs":[]}]}