{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Python imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from keras.utils import to_categorical\n",
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import optimizers\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.metrics import classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read MNIST data (download if necessary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ../data/train-images-idx3-ubyte.gz\n",
      "Extracting ../data/train-labels-idx1-ubyte.gz\n",
      "Extracting ../data/t10k-images-idx3-ubyte.gz\n",
      "Extracting ../data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets('../data/', one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Verify data shape\n",
    "\n",
    "* Training set\n",
    "* Validation set\n",
    "* Test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((55000, 784), (55000, 10))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.train.images.shape, mnist.train.labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5000, 784), (5000, 10))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.validation.images.shape, mnist.validation.labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10000, 784), (10000, 10))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.test.images.shape, mnist.test.labels.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configure neural network\n",
    "\n",
    "* 784 input units\n",
    "* 15 hidden units with sigmoid activation functions\n",
    "* 10 softmax output units"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Dense(15, activation = 'sigmoid', input_shape = (28 * 28,)))\n",
    "model.add(layers.Dense(10, activation = 'softmax'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Specify loss function and optimizer\n",
    "\n",
    "* Categorical cross-entropy loss function\n",
    "* RMSprop optimizer (learning rate is adapted for each parameter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer = optimizers.RMSprop(lr = 0.001),\n",
    "              loss = 'categorical_crossentropy',\n",
    "              metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1_input (InputLayer)   (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 15)                11775     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                160       \n",
      "=================================================================\n",
      "Total params: 11,935\n",
      "Trainable params: 11,935\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 55000 samples, validate on 5000 samples\n",
      "Epoch 1/20\n",
      "0s - loss: 1.8000 - acc: 0.5405 - val_loss: 1.4466 - val_acc: 0.7438\n",
      "Epoch 2/20\n",
      "0s - loss: 1.2511 - acc: 0.7997 - val_loss: 1.0558 - val_acc: 0.8444\n",
      "Epoch 3/20\n",
      "0s - loss: 0.9418 - acc: 0.8498 - val_loss: 0.8051 - val_acc: 0.8674\n",
      "Epoch 4/20\n",
      "0s - loss: 0.7376 - acc: 0.8697 - val_loss: 0.6376 - val_acc: 0.8878\n",
      "Epoch 5/20\n",
      "0s - loss: 0.6003 - acc: 0.8815 - val_loss: 0.5252 - val_acc: 0.8968\n",
      "Epoch 6/20\n",
      "0s - loss: 0.5063 - acc: 0.8911 - val_loss: 0.4472 - val_acc: 0.9022\n",
      "Epoch 7/20\n",
      "0s - loss: 0.4416 - acc: 0.8975 - val_loss: 0.3940 - val_acc: 0.9094\n",
      "Epoch 8/20\n",
      "0s - loss: 0.3960 - acc: 0.9028 - val_loss: 0.3581 - val_acc: 0.9128\n",
      "Epoch 9/20\n",
      "0s - loss: 0.3627 - acc: 0.9073 - val_loss: 0.3305 - val_acc: 0.9178\n",
      "Epoch 10/20\n",
      "0s - loss: 0.3377 - acc: 0.9109 - val_loss: 0.3097 - val_acc: 0.9210\n",
      "Epoch 11/20\n",
      "0s - loss: 0.3182 - acc: 0.9144 - val_loss: 0.2936 - val_acc: 0.9228\n",
      "Epoch 12/20\n",
      "0s - loss: 0.3026 - acc: 0.9171 - val_loss: 0.2808 - val_acc: 0.9258\n",
      "Epoch 13/20\n",
      "0s - loss: 0.2898 - acc: 0.9208 - val_loss: 0.2708 - val_acc: 0.9268\n",
      "Epoch 14/20\n",
      "0s - loss: 0.2789 - acc: 0.9226 - val_loss: 0.2616 - val_acc: 0.9294\n",
      "Epoch 15/20\n",
      "0s - loss: 0.2696 - acc: 0.9246 - val_loss: 0.2554 - val_acc: 0.9302\n",
      "Epoch 16/20\n",
      "0s - loss: 0.2613 - acc: 0.9270 - val_loss: 0.2486 - val_acc: 0.9316\n",
      "Epoch 17/20\n",
      "0s - loss: 0.2543 - acc: 0.9286 - val_loss: 0.2427 - val_acc: 0.9334\n",
      "Epoch 18/20\n",
      "0s - loss: 0.2477 - acc: 0.9301 - val_loss: 0.2383 - val_acc: 0.9338\n",
      "Epoch 19/20\n",
      "0s - loss: 0.2421 - acc: 0.9313 - val_loss: 0.2343 - val_acc: 0.9346\n",
      "Epoch 20/20\n",
      "0s - loss: 0.2367 - acc: 0.9328 - val_loss: 0.2292 - val_acc: 0.9352\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(mnist.train.images,\n",
    "                    mnist.train.labels,\n",
    "                    epochs = 20,\n",
    "                    verbose = 2,\n",
    "                    batch_size = 512,\n",
    "                    validation_data = (mnist.validation.images, mnist.validation.labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate model accuracy on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test accuracy:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.93069999999999997"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_loss, test_acc = model.evaluate(mnist.test.images, mnist.test.labels, verbose = 0)\n",
    "print()\n",
    "print(\"Test accuracy:\")\n",
    "test_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute predicted values and predicted probabilities in test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = model.predict_classes(mnist.test.images, verbose = 0)\n",
    "y_actual = mnist.test.labels.argmax(1)\n",
    "\n",
    "digits = [str(i) for i in range(10)]\n",
    "\n",
    "y_prob = model.predict(mnist.test.images)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Diagnostic measures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Classification report\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.95      0.97      0.96       980\n",
      "          1       0.98      0.98      0.98      1135\n",
      "          2       0.94      0.92      0.93      1032\n",
      "          3       0.90      0.92      0.91      1010\n",
      "          4       0.93      0.95      0.94       982\n",
      "          5       0.91      0.87      0.89       892\n",
      "          6       0.94      0.96      0.95       958\n",
      "          7       0.94      0.93      0.93      1028\n",
      "          8       0.89      0.90      0.89       974\n",
      "          9       0.92      0.91      0.92      1009\n",
      "\n",
      "avg / total       0.93      0.93      0.93     10000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print()\n",
    "print(\"Classification report\")\n",
    "print(classification_report(y_actual, y_pred, target_names = digits))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Confusion matrix\n",
      "[[ 950    0    1    2    1   10   10    4    2    0]\n",
      " [   0 1107    3    3    1    2    4    1   14    0]\n",
      " [  10    0  948   18    7    0   14    7   25    3]\n",
      " [   0    1   16  930    1   23    1   14   18    6]\n",
      " [   1    2    5    0  929    0    7    3    5   30]\n",
      " [   8    4    3   36    7  777   12    9   31    5]\n",
      " [   7    2    6    0    6   11  919    0    7    0]\n",
      " [   2    9   23    7    6    0    1  954    2   24]\n",
      " [   6    2    7   22   11   24    9   12  874    7]\n",
      " [  12    3    1   12   30    9    1   15    7  919]]\n"
     ]
    }
   ],
   "source": [
    "print()\n",
    "print(\"Confusion matrix\")\n",
    "print(confusion_matrix(y_actual, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Identify test examples where the model is wrong"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Errors\n",
      "693\n"
     ]
    }
   ],
   "source": [
    "errors = np.where(np.not_equal(y_actual, y_pred))\n",
    "print()\n",
    "print(\"Errors\")\n",
    "print(len(errors[0]))\n",
    "# print(errors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utility functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def show(image):\n",
    "    \"\"\"\n",
    "    Render a given numpy.uint8 2D array of pixel data.\n",
    "    \"\"\"\n",
    "    from matplotlib import pyplot\n",
    "    import matplotlib as mpl\n",
    "    fig = pyplot.figure()\n",
    "    ax = fig.add_subplot(1,1,1)\n",
    "    imgplot = ax.imshow(image, cmap=mpl.cm.Greys)\n",
    "    imgplot.set_interpolation('nearest')\n",
    "    ax.xaxis.set_ticks_position('top')\n",
    "    ax.yaxis.set_ticks_position('left')\n",
    "    pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def review(idx):\n",
    "    img = mnist.test.images[idx]\n",
    "    img = img.reshape((28, 28))\n",
    "    show(img)\n",
    "    print(\"Index: {}\".format(idx))\n",
    "    print(\"Predicted: {0}, Actual: {1}\".format(y_pred[idx], y_actual[idx]))\n",
    "    print(\"y_prob: {0}\".format(np.array_str(y_prob[idx], precision = 2, suppress_small = True)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Look at some randomly selected errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAD01JREFUeJzt3XuMVGWax/Hfo6OgYATsFgmL4hiNTtaImwpi1igbsxPW\nmAAayWqiGCdh/hgTL5O4BGPEQIwxgxNjjBHQDArrFW9RsuslQ3T+wSm1lVZ2V8Q2gFyaeAGDuAs8\n+0cdTY92vaesOlWn6Of7STpdfZ46XY+n/XGqznvOe8zdBSCeo8puAEA5CD8QFOEHgiL8QFCEHwiK\n8ANBlRJ+M5tlZv9tZpvNbGEZPdRjZgNmttHM+sysWnIvj5rZbjPrH7Jsgpm9ZmYfZ9/Hd1Fvi81s\ne7bt+szsspJ6m2Jmfzazj8zsQzO7KVte6rZL9FXKdrNOj/Ob2dGS/kfSP0vaJumvkq5294862kgd\nZjYgqeLue7qgl4slfSPpMXf/+2zZvZK+cPd7sn84x7v7v3VJb4slfePuf+h0Pz/qbZKkSe7+rpmd\nIOkdSXMkXa8St12ir3kqYbuVseefLmmzu29x9/+V9KSk2SX00fXc/U1JX/xo8WxJq7LHq1T7n6fj\n6vTWFdx9h7u/mz3eJ2mTpMkqedsl+ipFGeGfLGnrkJ+3qcQNMAyX9KqZvWNmC8puZhgT3X1H9nin\npIllNjOMG83sg+xjQSkfSYYys6mSzpe0QV207X7Ul1TCduOA309d5O7/IOlfJP0ue3vblbz2ma2b\nzs9+SNIZkqZJ2iFpWZnNmNlYSWsl3ezue4fWytx2w/RVynYrI/zbJU0Z8vPfZcu6grtvz77vlvS8\nah9Tusmu7LPj958hd5fczw/cfZe7H3L3w5JWqMRtZ2bHqBawNe7+XLa49G03XF9lbbcywv9XSWea\n2elmdqykf5X0Ugl9/ISZjckOxMjMxkj6taT+9Fod95Kk+dnj+ZJeLLGXv/F9sDJzVdK2MzOT9Iik\nTe5+35BSqduuXl+lbTd37/iXpMtUO+L/iaTby+ihTl+/lPR+9vVh2b1JekK1t4H/p9qxkd9IOknS\nG5I+lvS6pAld1NvjkjZK+kC1oE0qqbeLVHtL/4GkvuzrsrK3XaKvUrZbx4f6AHQHDvgBQRF+ICjC\nDwRF+IGgCD8QVKnh79LTZyV1b2/d2pdEb80qq7ey9/xd+wdR9/bWrX1J9NaskOEHUJKWTvIxs1mS\n7pd0tKSV7n5P6vk9PT0+derUH34eHBxUb29v06/fTt3aW7f2JdFbs4rsbWBgQHv27LFGnvuLZl8k\nm5TjQQ2ZlMPMXvLEpBxTp05VtVrq5DjAiFapVBp+bitv+5mUAziCtRL+bp+UA0BC2w/4mdkCM6ua\nWXVwcLDdLwegQa2Ev6FJOdx9ubtX3L3SrQdcgIhaCX/XTsoBIF/TR/vd/aCZ3SjpP1Ub6nvU3T8s\nrDMAbdV0+CXJ3ddJWldQLwA6iDP8gKAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBTh\nB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiqpdl7UbN169Zk/cknn0zW8+6U\nvGHDhmS9v7+/bm3z5s3JdfMcf/zxyfqDDz6YrM+bN69ubfTo0U31hGKw5weCIvxAUIQfCIrwA0ER\nfiAowg8ERfiBoBjnb9Drr79et/bAAw8k133llVeS9bxxfjNL1tu1riTt378/Wb/hhhuS9TvvvLNu\nbd269A2ezzrrrGT96KOPTtaR1lL4zWxA0j5JhyQddPdKEU0BaL8i9vz/5O57Cvg9ADqIz/xAUK2G\n3yW9ambvmNmCIhoC0Bmtvu2/yN23m9nJkl4zs/9y9zeHPiH7R2GBJJ166qktvhyAorS053f37dn3\n3ZKelzR9mOcsd/eKu1d6e3tbeTkABWo6/GY2xsxO+P6xpF9Lqn9tKYCu0srb/omSns/GkX8h6d/d\n/T8K6aoLnXLKKXVreeP4c+fOTdbHjRuXrF977bXJemosft++fcl1d+3alayvX78+WX/hhReS9dRc\nB+eee25y3Tlz5iTrzz77bLKOtKbD7+5bJJ1XYC8AOoihPiAowg8ERfiBoAg/EBThB4KyvMtJi1Sp\nVLxarXbs9Yr03Xff1a3df//9yXVvuummZH3UqFFN9dQJhw4dStZXr16drC9evLhuLW/K8zx5Q6jP\nPPNMS7//SFSpVFStVhu6jps9PxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ExTg/2mr79u11a9On/2Tu\nl7+Rd7lx3rTkb731Vt3ajBkzkuseqRjnB5CL8ANBEX4gKMIPBEX4gaAIPxAU4QeC4hbdaKvJkyfX\nreVNvX3xxRcn6wcPHkzWr7zyyrq1999/P7luT09Psj4SsOcHgiL8QFCEHwiK8ANBEX4gKMIPBEX4\ngaAY50dpLrzwwmT9lltuSdaXLVuWrKfmA1i5cmVy3YULFybrI0Hunt/MHjWz3WbWP2TZBDN7zcw+\nzr6Pb2+bAIrWyNv+P0ma9aNlCyW94e5nSnoj+xnAESQ3/O7+pqQvfrR4tqRV2eNVkuYU3BeANmv2\ngN9Ed9+RPd4paWK9J5rZAjOrmll1cHCwyZcDULSWj/Z7bQbQurOAuvtyd6+4e6W3t7fVlwNQkGbD\nv8vMJklS9n13cS0B6IRmw/+SpPnZ4/mSXiymHQCdkjvOb2ZPSJopqcfMtkm6U9I9kp42s99I+kzS\nvHY2iZgWLVqUrOeN86ds3Lix6XVHitzwu/vVdUqXFtwLgA7i9F4gKMIPBEX4gaAIPxAU4QeC4pLe\nLrB169Zk/ZNPPulQJ8WrVCp1a2PHjk2uO27cuKLb+cHAwECyfuDAgWR99OjRBXZTDvb8QFCEHwiK\n8ANBEX4gKMIPBEX4gaAIPxAU4/wF2LlzZ7K+ZMmSZP3xxx9P1vfv3/+zeypKbaKm+swsWT/ppJPq\n1i655JLkukuXLk3WW7Fhw4Zk/csvv0zWJ02aVGQ7pWDPDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANB\nMc7foLfffrtuLe9W03kOHz6crB91VPP/Rp988snJ+p49e5L1Q4cOJet5vaV+/9q1a5Pr5tVnzpyZ\nrK9fvz5ZT8k7v2EkYM8PBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0Exzp/5+uuvk/V58+rfhTzvmvY8\np512WrJ+6623JuvnnXde3doFF1yQXLevry9ZzzsP4N57703W+/v769b27t2bXDc157+Uf45BK3+X\nVv+mR4LcPb+ZPWpmu82sf8iyxWa23cz6sq/L2tsmgKI18rb/T5JmDbP8j+4+LftaV2xbANotN/zu\n/qakLzrQC4AOauWA341m9kH2sWB8vSeZ2QIzq5pZdXBwsIWXA1CkZsP/kKQzJE2TtEPSsnpPdPfl\n7l5x90pvb2+TLwegaE2F3913ufshdz8saYWk6cW2BaDdmgq/mQ2dt3iupPrjOQC6Uu44v5k9IWmm\npB4z2ybpTkkzzWyaJJc0IOm3beyxI/Lmzt+2bVvTv3vu3LnJ+ooVK5L1dt6nfsaMGS2tf/nllyfr\nW7durVvLG+c/++yzk/Vjjz02WU+N1efdM2DChAnJ+kiQG353v3qYxY+0oRcAHcTpvUBQhB8IivAD\nQRF+ICjCDwTFJb2ZLVu2NL3uOeeck6yvXr06WR81alTTr93tpkyZUnYLw3rvvfeS9W+++SZZHwl/\nM/b8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4/wFyLsNdoRpoJuxc+fOZH3JkiVte+1LL700WR8/\nvu7MdCMGe34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIpx/syYMWOSdXevW1u/fn1y3bvvvjtZX7Ro\nUbKeN0V1N0tNeZ73371mzZpk/fDhw8l6arvlTaeed/vvkWDk/xcCGBbhB4Ii/EBQhB8IivADQRF+\nICjCDwTVyC26p0h6TNJE1W7Jvdzd7zezCZKekjRVtdt0z3P3L9vXanvdfvvtyXrq+u7bbrstue7S\npUuT9b6+vmR95cqVyXpPT0+y3orPP/88WX/11VeT9cWLF9et5d32PG8ehLzzH1Lb7ZprrkmuG0Ej\ne/6Dkn7v7r+SNEPS78zsV5IWSnrD3c+U9Eb2M4AjRG743X2Hu7+bPd4naZOkyZJmS1qVPW2VpDnt\nahJA8X7WZ34zmyrpfEkbJE109x1ZaadqHwsAHCEaDr+ZjZW0VtLN7r53aM1rJ74Pe/K7mS0ws6qZ\nVQcHB1tqFkBxGgq/mR2jWvDXuPtz2eJdZjYpq0+StHu4dd19ubtX3L3S29tbRM8ACpAbfqsdcn1E\n0iZ3v29I6SVJ87PH8yW9WHx7ANrFUpeqSpKZXSTpLUkbJX1/DeUi1T73Py3pVEmfqTbU90Xqd1Uq\nFa9Wq632XIpvv/22bu3ll19Ornv99dcn6wcOHEjWTzzxxGR91qxZyXornnrqqWS9zGnJH3vssWQ9\n4nBepVJRtVpt6I+SO87v7n+RVO+XpSc/B9C1OMMPCIrwA0ERfiAowg8ERfiBoAg/EBRTdzfouOOO\nq1u76qqrkutu2rQpWb/rrruS9b179ybrTz/9dLLera677rpk/Y477kjWTz/99CLbCYc9PxAU4QeC\nIvxAUIQfCIrwA0ERfiAowg8ElXs9f5GO5Ov52+mrr75K1vOm7v7000/r1h5++OGmevrenDnpeVln\nz56drF9xxRV1a6lzJ6QYt8ku2s+5np+tCwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBMc4PjCCM8wPI\nRfiBoAg/EBThB4Ii/EBQhB8IivADQeWG38ymmNmfzewjM/vQzG7Kli82s+1m1pd9Xdb+dgEUpZGb\ndhyU9Ht3f9fMTpD0jpm9ltX+6O5/aF97ANolN/zuvkPSjuzxPjPbJGlyuxsD0F4/6zO/mU2VdL6k\nDdmiG83sAzN71MzGF9wbgDZqOPxmNlbSWkk3u/teSQ9JOkPSNNXeGSyrs94CM6uaWXVwcLCAlgEU\noaHwm9kxqgV/jbs/J0nuvsvdD7n7YUkrJE0fbl13X+7uFXev9Pb2FtU3gBY1crTfJD0iaZO73zdk\n+aQhT5srqb/49gC0SyNH+/9R0rWSNppZX7ZskaSrzWyaJJc0IOm3bekQQFs0crT/L5KGuz54XfHt\nAOgUzvADgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8E1dFb\ndJvZoKTPOvaCQDynuXtDU2Z1NPwAugdv+4GgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaD+\nH278uPsLj5IoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x108bdacc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index: 2780\n",
      "Predicted: 3, Actual: 2\n",
      "y_prob: [ 0.03  0.    0.12  0.78  0.04  0.    0.    0.    0.02  0.  ]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADipJREFUeJzt3V9sXGV6x/Hfk2wqYeAiENtE1NQBRYIViKQaJUWLCtWq\nK5oLCDf8EVqlkkU20oJKtBdFqcQifBOhwEKkCuGEaENx067Ev1xE7dKIgJaLZAfkhrBpm3RlFIxj\nT0RFiHxBwU8v5rByg+c9kzkzc8Y8349keXyeOT4PJ/x8Zs573jPm7gIQz7KyGwBQDsIPBEX4gaAI\nPxAU4QeCIvxAUKWE38zuMrP/NLPTZvZ4GT00YmaTZvahmU2YWbXkXvaZ2ayZnViw7Coze8vMTmXf\nV/ZQb0+a2VS27ybMbFNJvQ2Z2dtm9jsz+8jM/iZbXuq+S/RVyn6zbo/zm9lySf8l6S8lfSLpt5Ie\ndPffdbWRBsxsUlLF3c/1QC9/LumCpJfd/eZs2dOSPnP3ndkfzpXu/rc90tuTki64+65u93NRb6sl\nrXb3D8zsSknvS9os6a9V4r5L9HWfSthvZRz5N0g67e6/d/cvJf2TpHtK6KPnufu7kj67aPE9kvZn\nj/er/j9P1zXorSe4+7S7f5A9/kLSSUnXquR9l+irFGWE/1pJZxb8/IlK3AGLcEm/NrP3zWxr2c0s\nYtDdp7PHZyUNltnMIh4xs+PZ24JS3pIsZGbDktZLOqoe2ncX9SWVsN844fdtt7v7n0r6K0k/zV7e\n9iSvv2frpeuzX5B0g6R1kqYlPVNmM2Z2haRXJT3m7ucX1srcd4v0Vcp+KyP8U5KGFvz8x9mynuDu\nU9n3WUmvq/42pZfMZO8dv3kPOVtyP3/g7jPu/rW7z0vaoxL3nZmtUD1g4+7+Wra49H23WF9l7bcy\nwv9bSWvNbI2Z/ZGkByQdLKGPbzGzy7MTMTKzyyX9SNKJ9Fpdd1DSluzxFklvltjL//NNsDL3qqR9\nZ2Ym6SVJJ9392QWlUvddo75K22/u3vUvSZtUP+P/35L+roweGvR1vaR/z74+Krs3SQdUfxn4v6qf\nGxmRdLWkw5JOSfo3SVf1UG//IOlDScdVD9rqknq7XfWX9MclTWRfm8red4m+StlvXR/qA9AbOOEH\nBEX4gaAIPxAU4QeCIvxAUKWGv0cvn5XUu731al8SvbWqrN7KPvL37D+Iere3Xu1LordWhQw/gJIU\nusjHzO6S9Lyk5ZL2uvvO1PNXrVrlw8PDf/i5Vqupv7+/5e13Uq/21qt9SfTWqnb2Njk5qXPnzlkz\nz/1eqxvJbsrx91pwUw4zO+iJm3IMDw+rWi315jjAd1qlUmn6uUVe9nNTDmAJKxL+Xr8pB4CEjp/w\nM7OtZlY1s2qtVuv05gA0qUj4m7oph7uPuXvF3Su9esIFiKhI+Hv2phwA8rV8tt/dvzKzRyT9q+pD\nffvc/aO2dQago1oOvyS5+yFJh9rUC4Au4go/ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4g\nKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+\nICjCDwRF+IGgCD8QFOEHgir0Kb3oji+//DJZn5mZaVh78cUXC2079bslycyS9YGBgYa1+++/P7nu\nLbfckqyjmELhN7NJSV9I+lrSV+5eaUdTADqvHUf+v3D3c234PQC6iPf8QFBFw++Sfm1m75vZ1nY0\nBKA7ir7sv93dp8xsQNJbZvYf7v7uwidkfxS2StJ1111XcHMA2qXQkd/dp7Lvs5Jel7RhkeeMuXvF\n3Sv9/f1FNgegjVoOv5ldbmZXfvNY0o8knWhXYwA6q8jL/kFJr2fjvN+T9I/u/i9t6eo7Zm5uLlk/\ncuRIsv7EE08k6xMTEw1r7p5cN2+cvpPr79y5M7nue++9l6xv3LgxWUday+F3999LurWNvQDoIob6\ngKAIPxAU4QeCIvxAUIQfCIopvV2wd+/eZH379u3JepHhtrx185S5/ubNm5P1Y8eOJetDQ0MtbzsC\njvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBTj/F2QN46fNy02T5H1e3nbs7OzyfrY2FiyPjo6esk9\nRcKRHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCYpy/C+bn55P1ZcvSf4Pz5sQPDg42rN14443JdYuq\nVqvJ+oULF1r+3UXvJYA0jvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBTj/G1w9OjRZD1vHD9vXvvd\nd9+drO/evbthrdP3rs+7V0Gqt6L3EkAxuUd+M9tnZrNmdmLBsqvM7C0zO5V9X9nZNgG0WzMv+38p\n6a6Llj0u6bC7r5V0OPsZwBKSG353f1fSZxctvkfS/uzxfknpz1UC0HNaPeE36O7T2eOzkhpeXG5m\nW82sambVWq3W4uYAtFvhs/1en33RcAaGu4+5e8XdK/39/UU3B6BNWg3/jJmtlqTse/o2qwB6Tqvh\nPyhpS/Z4i6Q329MOgG7JHec3swOS7pS0ysw+kfRzSTsl/crMRiR9LOm+TjbZ69asWZOsP/DAA8n6\nnj17kvW+vr5L7qlZc3NzyfrevXuT9dQ4vlRsTn7eugMDAy3/bjQRfnd/sEHph23uBUAXcXkvEBTh\nB4Ii/EBQhB8IivADQTGltw3yhpzGx8e71Mm3HTp0KFnftWtXsv7OO+8k6538iO6RkZFkfdu2bS1v\nGxz5gbAIPxAU4QeCIvxAUIQfCIrwA0ERfiAoxvmXgDNnziTrGzdubFg7e/Zsct28sfaiH5NdZP3n\nnnsuWV+xYkXLvxsc+YGwCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMb5l4DUOL4kzc42/syUvHH8oh+T\n3cn5/EeOHEnWN23a1PK2wZEfCIvwA0ERfiAowg8ERfiBoAg/EBThB4JinL8LPv/882T9pptuStaL\nzMkvcz5+0fXz7tt/7NixZH1oaKjlbUeQe+Q3s31mNmtmJxYse9LMpsxsIvviagtgiWnmZf8vJd21\nyPJfuPu67Cv9sTAAek5u+N39XUmfdaEXAF1U5ITfI2Z2PHtbsLLRk8xsq5lVzaxaq9UKbA5AO7Ua\n/hck3SBpnaRpSc80eqK7j7l7xd0r/f39LW4OQLu1FH53n3H3r919XtIeSRva2xaATmsp/Ga2esGP\n90o60ei5AHpT7ji/mR2QdKekVWb2iaSfS7rTzNZJckmTkn7SwR6XvPPnzyfrqfn4Umfn5PfyfP68\n/TI2Npasj46OXnJPkeSG390fXGTxSx3oBUAXcXkvEBThB4Ii/EBQhB8IivADQTGltwvyprUWra9f\nv75h7amnnkquu3bt2mT91KlTyfr4+HiyfuDAgWQ9peh0YqRx5AeCIvxAUIQfCIrwA0ERfiAowg8E\nRfiBoKybY6mVSsWr1WrXttcrzpw5k6yvWbMmWc/7N5qcnGxYK/v21cuXL29Yy5vSm/fffc011yTr\nU1NTyfp3UaVSUbVabWqeNUd+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK+fxdcPXVVyfr119/fbJ+\n+vTpZP2NN95oWHv00UeT6xY1NzeXrBe5jqTofQ6QxpEfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Jq\n5iO6hyS9LGlQ9Y/kHnP3583sKkn/LGlY9Y/pvs/d/6dzrS5dfX19yforr7ySrN92223J+vbt2xvW\nLly4kFz31ltvTdY//fTTZP3pp59O1lNz9ot+PPjIyEih9aNr5sj/laSfufv3Jf2ZpJ+a2fclPS7p\nsLuvlXQ4+xnAEpEbfnefdvcPssdfSDop6VpJ90janz1tv6TNnWoSQPtd0nt+MxuWtF7SUUmD7j6d\nlc6q/rYAwBLRdPjN7ApJr0p6zN3PL6x5/SLrRS+0NrOtZlY1s2qtVivULID2aSr8ZrZC9eCPu/tr\n2eIZM1ud1VdLml1sXXcfc/eKu1f6+/vb0TOANsgNv9VPyb4k6aS7P7ugdFDSluzxFklvtr89AJ3S\nzJTeH0j6saQPzWwiW7ZD0k5JvzKzEUkfS7qvMy1+923YsCFZn5+fT9aXLWv8N3zHjh0tr1t023nr\nF932wMBAso603PC7+28kNRqQ/WF72wHQLVzhBwRF+IGgCD8QFOEHgiL8QFCEHwiKW3cvAbt3707W\nU1N688bS86bVdnL9vHUffvjhZH3btm3JOtI48gNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIzzLwF5\nH7N92WWXNazljZV3ej7/HXfc0bD20EMPJdfl1tydxZEfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ji\nnH8JmJubS9bffvvthrWi8/E3b05//uro6GiyfvPNNyfrKA9HfiAowg8ERfiBoAg/EBThB4Ii/EBQ\nhB8IKnec38yGJL0saVCSSxpz9+fN7ElJD0uqZU/d4e6HOtVoZH19fcn6+Ph4SzXE1sxFPl9J+pm7\nf2BmV0p638zeymq/cPddnWsPQKfkht/dpyVNZ4+/MLOTkq7tdGMAOuuS3vOb2bCk9ZKOZoseMbPj\nZrbPzFa2uTcAHdR0+M3sCkmvSnrM3c9LekHSDZLWqf7K4JkG6201s6qZVWu12mJPAVCCpsJvZitU\nD/64u78mSe4+4+5fu/u8pD2SNiy2rruPuXvF3Sv9/f3t6htAQbnht/q0r5cknXT3ZxcsX73gafdK\nOtH+9gB0SjNn+38g6ceSPjSziWzZDkkPmtk61Yf/JiX9pCMdAuiIZs72/0bSYpO+GdMHljCu8AOC\nIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRl7t69jZnVJH3c\ntQ0C8fyJuzd1y6yuhh9A7+BlPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/ENT/AX3+dRhZ\ntI2SAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x109e7ee10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index: 1114\n",
      "Predicted: 8, Actual: 3\n",
      "y_prob: [ 0.    0.    0.04  0.12  0.    0.01  0.    0.    0.82  0.01]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADWtJREFUeJzt3W+oXAV+xvHnyTZVo9HE5jZGq8aKIEE0KaMoKzVlddG8\n8R9IFZYUheyLFaJEqKbgRhCUsiYuokLUsFmxtpEoBgzbtWFV8kb2KjEmJq3beKWJ13tvsBKXaDTx\n1xf3ZLmNd87czJyZM/H3/cDlzpzfzJyHo0/OzDlzZxwRApDPtLoDAKgH5QeSovxAUpQfSIryA0lR\nfiCpWspv+3rb/2n7D7bvryNDM7aHbL9ve5vtwZqzrLM9anvHhGVn2n7d9ofF79l9lG2V7X3Ftttm\ne0lN2c61/TvbH9jeaXt5sbzWbVeSq5bt5l6f57f9A0n/Jek6SXsl/V7S7RHxQU+DNGF7SFIjIvb3\nQZa/lfRHSb+OiEuKZf8s6bOIeLT4h3N2RPxjn2RbJemPEfGLXuc5Jts8SfMi4l3bMyW9I+kmSf+g\nGrddSa7bVMN2q2PPf4WkP0TEnoj4WtK/Srqxhhx9LyLekvTZMYtvlLS+uLxe4//z9FyTbH0hIoYj\n4t3i8heSdkk6RzVvu5Jctaij/OdI+p8J1/eqxg0wiZD0W9vv2F5Wd5hJzI2I4eLyp5Lm1hlmEnfb\n3l68LKjlJclEtudLWiTpbfXRtjsml1TDduOA33ddHRF/I+kGST8rnt72pRh/zdZP789+WtKFkhZK\nGpb0WJ1hbJ8maaOkeyLiwMRZndtukly1bLc6yr9P0rkTrv9VsawvRMS+4veopFc0/jKln4wUrx2P\nvoYcrTnPn0TESEQciYhvJT2jGred7ekaL9gLEfFysbj2bTdZrrq2Wx3l/72ki2xfYPvPJf29pE01\n5PgO26cWB2Jk+1RJP5a0o/xePbdJ0tLi8lJJr9aY5f85WqzCzapp29m2pOck7YqI1RNGtW67Zrlq\n224R0fMfSUs0fsT/vyX9Ux0ZmuT6a0nvFT87684m6UWNPw38RuPHRu6S9BeStkj6UNJ/SDqzj7I9\nL+l9Sds1XrR5NWW7WuNP6bdL2lb8LKl725XkqmW79fxUH4D+wAE/ICnKDyRF+YGkKD+QFOUHkqq1\n/H369llJ/ZutX3NJZGtXXdnq3vP37X8Q9W+2fs0lka1dKcsPoCYdvcnH9vWSfinpB5KejYhHy24/\nZ86cmD9//p+uj42NaWBgoO31d1O/ZuvXXBLZ2lVltqGhIe3fv99Tue2ftbuS4kM5ntSED+WwvSlK\nPpRj/vz5Ghys9cNxgO+1RqMx5dt28rSfD+UATmCdlL/fP5QDQImuH/Czvcz2oO3BsbGxbq8OwBR1\nUv4pfShHRKyNiEZENPr1gAuQUSfl79sP5QDQWttH+yPisO27Jf27xk/1rYuInZUlA9BVbZdfkiJi\ns6TNFWUB0EO8ww9IivIDSVF+ICnKDyRF+YGkKD+QFOUHkqL8QFKUH0iK8gNJUX4gKcoPJEX5gaQo\nP5AU5QeSovxAUpQfSIryA0lRfiApyg8kRfmBpCg/kBTlB5Ki/EBSlB9IivIDSVF+ICnKDyRF+YGk\nKD+QVEdf0W17SNIXko5IOhwRjSpCAei+jspf+LuI2F/B4wDoIZ72A0l1Wv6Q9Fvb79heVkUgAL3R\n6dP+qyNin+2/lPS67d0R8dbEGxT/KCyTpPPOO6/D1QGoSkd7/ojYV/welfSKpCsmuc3aiGhERGNg\nYKCT1QGoUNvlt32q7ZlHL0v6saQdVQUD0F2dPO2fK+kV20cf518i4jeVpALQdW2XPyL2SLqswiwA\neohTfUBSlB9IivIDSVF+ICnKDyRF+YGkKD+QFOUHkqL8QFKUH0iK8gNJUX4gKcoPJFXFB3iiQ4cO\nHSqdr1mzpnR+zTXXNJ1dcMEFbWU6EXz11Vel82effbbtxz799NNL58uXLy+dn3TSSW2vu1fY8wNJ\nUX4gKcoPJEX5gaQoP5AU5QeSovxAUpzn7wMbNmwona9cubJHSTBVO3fuLJ0/8sgjpfOzzz67yjht\nYc8PJEX5gaQoP5AU5QeSovxAUpQfSIryA0lxnr8HDh48WDpvdU4Y/ef5558vnY+NjZXON2/eXGWc\ntrTc89teZ3vU9o4Jy860/brtD4vfs7sbE0DVpvK0/1eSrj9m2f2StkTERZK2FNcBnEBalj8i3pL0\n2TGLb5S0vri8XtJNFecC0GXtHvCbGxHDxeVPJc1tdkPby2wP2h5s9ToIQO90fLQ/IkJSlMzXRkQj\nIhoDAwOdrg5ARdot/4jteZJU/B6tLhKAXmi3/JskLS0uL5X0ajVxAPRKy/P8tl+UtFjSHNt7Jf1c\n0qOSNti+S9LHkm7rZsgT3WuvvVY63717d4+SoCpnnHFG6XzBggU9StK+luWPiNubjH5UcRYAPcTb\ne4GkKD+QFOUHkqL8QFKUH0iKP+mtwJEjR0rn9957b4+S5DJjxozS+dKlS0vnZa677rrS+ZVXXlk6\nP+uss9ped6+w5weSovxAUpQfSIryA0lRfiApyg8kRfmBpDjP3wOffPJJ3RHatmLFitL5Qw891KMk\n32W7dH7KKaf0KMmJiT0/kBTlB5Ki/EBSlB9IivIDSVF+ICnKDyTFef4KTJtW/m/orbfeWjrfuHFj\nlXEq1eqrqL/55pvS+apVq5rOZs2a1U4kVIQ9P5AU5QeSovxAUpQfSIryA0lRfiApyg8k5Yjo2coa\njUYMDg72bH3H49ChQ6Xzr7/+uuls5syZpff9/PPPS+c7duwona9cubJ0vnXr1tJ5nc4///yms/vu\nu6/0vnfeeWfpnL/X/65Go6HBwcHyDzootNzz215ne9T2jgnLVtneZ3tb8bOkk8AAem8qT/t/Jen6\nSZaviYiFxc/mamMB6LaW5Y+ItyR91oMsAHqokwN+d9veXrwsmN3sRraX2R60PTg2NtbB6gBUqd3y\nPy3pQkkLJQ1LeqzZDSNibUQ0IqIxMDDQ5uoAVK2t8kfESEQciYhvJT0j6YpqYwHotrbKb3vehKs3\nSyo/VwWg77Q8z2/7RUmLJc2RNCLp58X1hZJC0pCkn0bEcKuV9fN5/gcffLB0/vjjjzedPfDAA6X3\nbXU+e/r06aXzw4cPtz1/8803S+/b6jvsR0dHS+fddO2115bOV69eXTq/5JJLqoxzQjie8/wtP8wj\nIm6fZPFzx50KQF/h7b1AUpQfSIryA0lRfiApyg8kxZ/0Flp93XOreZnFixeXzi+++OK2H7tTrU4z\nPvHEEz1Kcvzmzp1bOt+zZ0/T2ff1z4Er/ZNeAN9PlB9IivIDSVF+ICnKDyRF+YGkKD+QFF/RXVi0\naFHpfNu2bW0/9htvvNHRHJMbGRkpna9YsaLp7Kmnnqo6zgmHPT+QFOUHkqL8QFKUH0iK8gNJUX4g\nKcoPJMV5/sKTTz5ZOr/jjjuazvbu3Vt63yNHjrSVCeWmTSvfdw0NDTWdffnll6X3/b7+vf9E7PmB\npCg/kBTlB5Ki/EBSlB9IivIDSVF+IKmW5/ltnyvp15LmavwruddGxC9tnynp3yTN1/jXdN8WEf/b\nvajdddVVV5XOP/roo6az9957r/S+Dz/8cFuZjtq5c2fpfPfu3R09fl0uv/zy0vkNN9xQOr/llltK\n55deeulxZ8pkKnv+w5JWRMQCSVdK+pntBZLul7QlIi6StKW4DuAE0bL8ETEcEe8Wl7+QtEvSOZJu\nlLS+uNl6STd1KySA6h3Xa37b8yUtkvS2pLkRMVyMPtX4ywIAJ4gpl9/2aZI2SronIg5MnMX4F/5N\n+qV/tpfZHrQ9ODY21lFYANWZUvltT9d48V+IiJeLxSO25xXzeZJGJ7tvRKyNiEZENAYGBqrIDKAC\nLcvv8a+nfU7SrohYPWG0SdLS4vJSSa9WHw9At0zlT3p/KOknkt63ffTzq1dKelTSBtt3SfpY0m3d\nidj/LrvsstL5Sy+91NHjHzx4sHR+4MCB0nm/mjVrVun85JNP7lGSnFqWPyK2Smr2fd8/qjYOgF7h\nHX5AUpQfSIryA0lRfiApyg8kRfmBpPjo7hPAjBkzOpoDk2HPDyRF+YGkKD+QFOUHkqL8QFKUH0iK\n8gNJUX4gKcoPJEX5gaQoP5AU5QeSovxAUpQfSIryA0lRfiApyg8kRfmBpCg/kBTlB5Ki/EBSlB9I\nivIDSbUsv+1zbf/O9ge2d9peXixfZXuf7W3Fz5LuxwVQlal8acdhSSsi4l3bMyW9Y/v1YrYmIn7R\nvXgAuqVl+SNiWNJwcfkL27skndPtYAC667he89ueL2mRpLeLRXfb3m57ne3ZFWcD0EVTLr/t0yRt\nlHRPRByQ9LSkCyUt1Pgzg8ea3G+Z7UHbg2NjYxVEBlCFKZXf9nSNF/+FiHhZkiJiJCKORMS3kp6R\ndMVk942ItRHRiIjGwMBAVbkBdGgqR/st6TlJuyJi9YTl8ybc7GZJO6qPB6BbpnK0/4eSfiLpfdvb\nimUrJd1ue6GkkDQk6addSQigK6ZytH+rJE8y2lx9HAC9wjv8gKQoP5AU5QeSovxAUpQfSIryA0lR\nfiApyg8kRfmBpCg/kBTlB5Ki/EBSlB9IivIDSTkiercye0zSxz1bIZDP+RExpY/M6mn5AfQPnvYD\nSVF+ICnKDyRF+YGkKD+QFOUHkqL8QFKUH0iK8gNJ/R9EvhUn4GgYcwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x120f6b160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index: 7850\n",
      "Predicted: 8, Actual: 5\n",
      "y_prob: [ 0.04  0.    0.04  0.    0.08  0.22  0.16  0.    0.46  0.01]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADkZJREFUeJzt3V+MVHWaxvHnlcWY2CaCtJ2Oq/RKuCGTLG4quGbwzzqB\nICEiXBgNENBmGRR0MSRi1GS4MTGbETAqJu1KBlcXnAQMxH+LSzbBudBMi4gI6zpOMAu2UMgmwoUu\nwrsXdZj0MH1+Vda/U837/SSdrjpPHerNgYdTVafqlLm7AMRzSdEDACgG5QeCovxAUJQfCIryA0FR\nfiCoQspvZrPM7HMz+4OZPVbEDHnM7LCZfWpm+8xssOBZNpnZcTM7MGzZeDN7z8y+yH6P66DZ1prZ\n0Wzb7TOz2QXNdq2Z/aeZHTSzz8zsn7LlhW67xFyFbDdr93F+Mxsj6b8lzZB0RNLvJd3r7gfbOkgO\nMzssqeTuJzpgllsknZb0irv/LFv2z5JOuvvT2X+c49x9TYfMtlbSaXf/dbvnuWC2Xkm97r7XzK6Q\n9JGkuyQtUYHbLjHX3SpguxWx558m6Q/u/kd3/z9JWyXNLWCOjufueySdvGDxXEmbs8ubVfnH03Y5\ns3UEdx9y973Z5VOSDkm6RgVvu8RchSii/NdI+p9h14+owA0wApe0y8w+MrNlRQ8zgh53H8oufyOp\np8hhRrDSzPZnTwsKeUoynJn1SbpB0ofqoG13wVxSAduNF/z+0nR3/ztJd0hakT287Uheec7WSe/P\nflHSJElTJQ1JeqbIYcysS9I2Savc/bvhWZHbboS5CtluRZT/qKRrh13/62xZR3D3o9nv45LeUOVp\nSic5lj13PP8c8njB8/yJux9z97Pufk7SSypw25nZWFUK9pq7b88WF77tRpqrqO1WRPl/L2mymf2N\nmV0q6R5JOwuY4y+Y2eXZCzEys8slzZR0IL1W2+2UtDi7vFjSjgJn+TPni5WZp4K2nZmZpJclHXL3\ndcOiQrdd3lyFbTd3b/uPpNmqvOL/paQnipghZ67rJX2S/XxW9GyStqjyMPCMKq+N9Eu6StJuSV9I\n+g9J4ztotn+V9Kmk/aoUrbeg2aar8pB+v6R92c/sorddYq5CtlvbD/UB6Ay84AcERfmBoCg/EBTl\nB4Ki/EBQhZa/Q98+K6lzZ+vUuSRmq1dRsxW95+/YvxB17mydOpfEbPUKWX4ABWnoTT5mNkvSs5LG\nSPoXd386dfsJEyZ4X1/fn66Xy2V1d3fXff+t1KmzdepcErPVq5mzHT58WCdOnLBabvtX9d5JdlKO\nFzTspBxmttMTJ+Xo6+vT4GChJ8cBLmqlUqnm2zbysJ+TcgCjWCPl7/STcgBIaPkLfma2zMwGzWyw\nXC63+u4A1KiR8td0Ug53H3D3kruXOvUFFyCiRsrfsSflAFBd3a/2u/uPZrZS0r+rcqhvk7t/1rTJ\nALRU3eWXJHd/W9LbTZoFQBvxDj8gKMoPBEX5gaAoPxAU5QeCovxAUJQfCIryA0FRfiAoyg8ERfmB\noCg/EBTlB4Ki/EBQlB8IivIDQVF+ICjKDwRF+YGgKD8QFOUHgqL8QFCUHwiK8gNBUX4gKMoPBEX5\ngaAoPxAU5QeCovxAUA19RbeZHZZ0StJZST+6e6kZQwFovYbKn/kHdz/RhD8HQBvxsB8IqtHyu6Rd\nZvaRmS1rxkAA2qPRh/3T3f2omV0t6T0z+y933zP8Btl/Cssk6brrrmvw7gA0S0N7fnc/mv0+LukN\nSdNGuM2Au5fcvdTd3d3I3QFoorrLb2aXm9kV5y9LminpQLMGA9BajTzs75H0hpmd/3P+zd3fbcpU\n+DPbtm1L5m+99VZudvr06eS6H3zwQTI/cuRIMq/2VG7OnDm52U033ZRc94477kjm48ePT+ZIq7v8\n7v5HSX/bxFkAtBGH+oCgKD8QFOUHgqL8QFCUHwiqGR/sQYMWLlyYzLdu3ZrMJ02alJuVSukPWi5Z\nsiSZ9/b2JvP3338/mR88eDA327hxY3LdsWPHJvPNmzcn83vuuSeZR8eeHwiK8gNBUX4gKMoPBEX5\ngaAoPxAU5QeC4jh/B/j444+T+blz55J5f39/bvboo4/WNVOtHnjggWR+9uzZ3OzkyZPJdXfs2JHM\nly9fnsxT71G49dZbk+tGwJ4fCIryA0FRfiAoyg8ERfmBoCg/EBTlB4LiOH8HmDdvXjI/dOhQQ+sX\nacyYMblZtS9xWbp0aTJ//fXXk/maNWtys2qnLI+APT8QFOUHgqL8QFCUHwiK8gNBUX4gKMoPBMVx\n/g7wyCOPJPP169cn8zNnzjRznFHjzjvvTOYDAwNtmmR0qrrnN7NNZnbczA4MWzbezN4zsy+y3+Na\nOyaAZqvlYf9vJM26YNljkna7+2RJu7PrAEaRquV39z2SLjzf0lxJ578rabOku5o8F4AWq/cFvx53\nH8oufyOpJ++GZrbMzAbNbLBcLtd5dwCareFX+93dJXkiH3D3kruXqn2QA0D71Fv+Y2bWK0nZ7+PN\nGwlAO9Rb/p2SFmeXF0tKn2MZQMepepzfzLZIuk3SBDM7IulXkp6W9Fsz65f0laS7Wznkxe6qq65K\n5rfccksyf/PNN3OzKVOm1DXTaPDJJ58UPcKoVrX87n5vTvSLJs8CoI14ey8QFOUHgqL8QFCUHwiK\n8gNB8ZHeUWD+/PnJfN26dblZ6uu7peqHGYt0+vTpZJ46xClV/wrv6NjzA0FRfiAoyg8ERfmBoCg/\nEBTlB4Ki/EBQHOcfBWbOnJnMV6xYkZs999xzyXXXrl1bz0ht8e233zaUL1iwoJnjXHTY8wNBUX4g\nKMoPBEX5gaAoPxAU5QeCovxAUBznHwUmTpyYzJ966qncrNrprc+dO5fML7mkdfuHap/XX716dTK/\n8cYbk/nkyZN/8kyRsOcHgqL8QFCUHwiK8gNBUX4gKMoPBEX5gaA4zn8RWLhwYW62fv365LobN25M\n5itXrqxrplps3bo1mW/fvj2Zf/75580cJ5yqe34z22Rmx83swLBla83sqJnty35mt3ZMAM1Wy8P+\n30iaNcLy9e4+Nft5u7ljAWi1quV39z2STrZhFgBt1MgLfivNbH/2tGBc3o3MbJmZDZrZYLlcbuDu\nADRTveV/UdIkSVMlDUl6Ju+G7j7g7iV3L3V3d9d5dwCara7yu/sxdz/r7uckvSRpWnPHAtBqdZXf\nzHqHXZ0n6UDebQF0pqrH+c1si6TbJE0wsyOSfiXpNjObKsklHZb0yxbOiCp6e3tzs0bP279kyZJk\n3tXVlcy//vrr3Cz1fQOSdP/99ydzPq/fmKrld/d7R1j8cgtmAdBGvL0XCIryA0FRfiAoyg8ERfmB\noPhI70Vu/vz5yfzo0aPJ/Mknn0zmqdOGS9LcuXNzs2qH8l544YVkjsaw5weCovxAUJQfCIryA0FR\nfiAoyg8ERfmBoDjOf5Ezs2Q+ZcqUZL5mzZpkvmnTpmQ+Y8aM3GzDhg3JdVv59eBgzw+ERfmBoCg/\nEBTlB4Ki/EBQlB8IivIDQXGcvwOcOXMmmX/55ZfJfMeOHXXf9/79+5P5Dz/8kMxvvvnmZL5ly5bc\n7NJLL02ui9Zizw8ERfmBoCg/EBTlB4Ki/EBQlB8IivIDQdXyFd3XSnpFUo8qX8k94O7Pmtl4Sa9L\n6lPla7rvdvf/bd2oF6/Vq1cn8+effz6Z9/T05GYTJ05Mrlvt8/wPPvhgMq92bn0+k9+5avmb+VHS\nanefIunvJa0wsymSHpO0290nS9qdXQcwSlQtv7sPufve7PIpSYckXSNprqTN2c02S7qrVUMCaL6f\n9JjMzPok3SDpQ0k97j6URd+o8rQAwChRc/nNrEvSNkmr3P274Zm7uyqvB4y03jIzGzSzwXK53NCw\nAJqnpvKb2VhViv+au2/PFh8zs94s75V0fKR13X3A3UvuXuru7m7GzACaoGr5rXL615clHXL3dcOi\nnZIWZ5cXS6r/o2UA2q6Wj/T+XNIiSZ+a2b5s2eOSnpb0WzPrl/SVpLtbM+LF79SpU8n8oYceSuap\nr8nu6uqqa6bzvv/++2S+cePGZL53797cbNq0aXXNhOaoWn53/52kvJO//6K54wBoF96BAQRF+YGg\nKD8QFOUHgqL8QFCUHwiKU3d3gP7+/mQ+Z86cZL5gwYLcrNFj6Zdddlkyv++++5L5q6++mptxnL9Y\n7PmBoCg/EBTlB4Ki/EBQlB8IivIDQVF+ICiO83eA6dOnJ/MnnngimS9dujQ327NnT3LdK6+8MplX\ns3z58mS+YcOGhv58tA57fiAoyg8ERfmBoCg/EBTlB4Ki/EBQlB8IiuP8o8CqVauS+dVXX52b3X77\n7cl1t2zZksyvv/76ZP7OO+8k8xkzZiRzFIc9PxAU5QeCovxAUJQfCIryA0FRfiAoyg8EVfU4v5ld\nK+kVST2SXNKAuz9rZmsl/aOkcnbTx9397VYNGtnYsWOT+eLFi3Oz2bNnJ9ddtGhRMt+1a1cynzVr\nVjJ/+OGHkzmKU8ubfH6UtNrd95rZFZI+MrP3smy9u/+6deMBaJWq5Xf3IUlD2eVTZnZI0jWtHgxA\na/2k5/xm1ifpBkkfZotWmtl+M9tkZuOaPBuAFqq5/GbWJWmbpFXu/p2kFyVNkjRVlUcGz+Sst8zM\nBs1ssFwuj3QTAAWoqfxmNlaV4r/m7tslyd2PuftZdz8n6SVJI37rorsPuHvJ3Uvd3d3NmhtAg6qW\n38xM0suSDrn7umHLe4fdbJ6kA80fD0Cr1PJq/88lLZL0qZnty5Y9LuleM5uqyuG/w5J+2ZIJ0ZBq\nj7befffdNk2CTlPLq/2/k2QjRBzTB0Yx3uEHBEX5gaAoPxAU5QeCovxAUJQfCIryA0FRfiAoyg8E\nRfmBoCg/EBTlB4Ki/EBQlB8Iyty9fXdmVpb0VdvuEIhnorvXdMqstpYfQOfgYT8QFOUHgqL8QFCU\nHwiK8gNBUX4gKMoPBEX5gaAoPxDU/wOfQzl1NV03wAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x121041828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index: 3951\n",
      "Predicted: 7, Actual: 8\n",
      "y_prob: [ 0.04  0.    0.15  0.09  0.    0.26  0.    0.33  0.09  0.04]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADHxJREFUeJzt3V+oHPUZxvHnqbU36kVstiHY2NOKnFAKjWWRglIsUklz\noyYg9UJSECKYgIIXlfRCr0SKf/AiCrEGY7G2wjliLqTVBkG8EVdJYzRJtRKp4ZhssGC8atW3FzuR\n03jOzrozs7Mn7/cDy87OzO68/JLnzJ/f7G8dEQKQzzfaLgBAOwg/kBThB5Ii/EBShB9IivADSbUS\nftsbbR+1/Z7tu9uoYTm2j9l+y/YB272Wa9lj+6TtQ4vmXWz7JdvvFs+rpqi2e20fL9rugO1NLdW2\nzvbLtt+x/bbtO4r5rbbdkLpaaTdPup/f9nmS/iHpF5I+lPS6pJsj4p2JFrIM28ckdSPi1BTU8jNJ\nn0p6KiJ+VMz7naSPI+L+4g/nqoj4zZTUdq+kTyPigUnXc1ZtayWtjYg3bV8k6Q1JN0j6tVpsuyF1\n3aQW2q2NPf+Vkt6LiPcj4j+S/iTp+hbqmHoR8Yqkj8+afb2kvcX0Xg3+80zcMrVNhYhYiIg3i+nT\nkg5LukQtt92QulrRRvgvkfSvRa8/VIsNsISQ9KLtN2xva7uYJayJiIVi+iNJa9osZgk7bB8sTgta\nOSVZzPaMpCskvaYparuz6pJaaDcu+H3V1RHxE0m/lLS9OLydSjE4Z5um+7Mfk3SZpA2SFiQ92GYx\nti+UNCfpzoj4ZPGyNttuibpaabc2wn9c0rpFr79bzJsKEXG8eD4p6TkNTlOmyYni3PHMOeTJluv5\nUkSciIjPI+ILSY+rxbazfb4GAXs6IuaL2a233VJ1tdVubYT/dUmX2/6+7W9J+pWkfS3U8RW2Lygu\nxMj2BZKuk3Ro+Lsmbp+krcX0VknPt1jL/zkTrMKNaqntbFvSE5IOR8RDixa12nbL1dVau0XExB+S\nNmlwxf+fkn7bRg3L1PUDSX8vHm+3XZukZzQ4DPyvBtdGbpX0bUn7Jb0r6W+SLp6i2v4g6S1JBzUI\n2tqWartag0P6g5IOFI9NbbfdkLpaabeJd/UBmA5c8AOSIvxAUoQfSIrwA0kRfiCpVsM/pbfPSpre\n2qa1LonaxtVWbW3v+af2H0TTW9u01iVR27hShh9ASyrd5GN7o6RHJJ0n6fcRcf+w9VevXh0zMzNf\nvu73++p0OmNvv0nTWtu01iVR27jqrO3YsWM6deqUR1n3m+NupBiUY5cWDcphe18MGZRjZmZGvV6r\ng+MA57RutzvyulUO+xmUA1jBqoR/2gflADBE4xf8bG+z3bPd6/f7TW8OwIiqhH+kQTkiYndEdCOi\nO60XXICMqoR/agflAFBu7Kv9EfGZ7R2S/qpBV9+eiHi7tsoANGrs8EtSRLwg6YWaagEwQdzhByRF\n+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUpW+1Qds2bJl6PL5+fll\nlx05cmToe2dnZ8eqCaNhzw8kRfiBpAg/kBThB5Ii/EBShB9IivADSdHPj6Gq9OOXoR+/Xez5gaQI\nP5AU4QeSIvxAUoQfSIrwA0kRfiAp+vkxVJV+/DJl9xDMzc1V+vyjR48uu2znzp2VPrtqbdOgUvht\nH5N0WtLnkj6LiG4dRQFoXh17/p9HxKkaPgfABHHODyRVNfwh6UXbb9jeVkdBACaj6mH/1RFx3PZ3\nJL1k+0hEvLJ4heKPwjZJuvTSSytuDkBdKu35I+J48XxS0nOSrlxind0R0Y2IbqfTqbI5ADUaO/y2\nL7B90ZlpSddJOlRXYQCaVeWwf42k52yf+Zw/RsRfaqkKEzOsL7xpZfcQlN0HUPXzsxs7/BHxvqQf\n11gLgAmiqw9IivADSRF+ICnCDyRF+IGk+EpvclW/2tqkNrvqNm/e3Nq2J4U9P5AU4QeSIvxAUoQf\nSIrwA0kRfiApwg8kRT//Oa7sK7tV+9J37do19nu3b99eadtVlNV9++23T6iS9rDnB5Ii/EBShB9I\nivADSRF+ICnCDyRF+IGk6Oc/x61fv77S+8u+116lP7xqP39ZbefCz2g3iT0/kBThB5Ii/EBShB9I\nivADSRF+ICnCDyRFP/85oOpPWQ/TZF95RDT22ShXuue3vcf2SduHFs272PZLtt8tnlc1WyaAuo1y\n2P+kpI1nzbtb0v6IuFzS/uI1gBWkNPwR8Yqkj8+afb2kvcX0Xkk31FwXgIaNe8FvTUQsFNMfSVqz\n3Iq2t9nu2e71+/0xNwegbpWv9sfgqs2yV24iYndEdCOi2+l0qm4OQE3GDf8J22slqXg+WV9JACZh\n3PDvk7S1mN4q6fl6ygEwKaX9/LafkXSNpNW2P5R0j6T7JT1r+1ZJH0i6qckis2t67P0q256dnW1s\n22hWafgj4uZlFl1bcy0AJojbe4GkCD+QFOEHkiL8QFKEH0iKr/SuAPv3729t21WH/q6iys9/Szl+\nZrsK9vxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBT9/CtAm/38bar6E97D3l92D0GGewTY8wNJEX4g\nKcIPJEX4gaQIP5AU4QeSIvxAUvTzrwDXXjv+QMlV3itV72ufVmX3TtDPD+CcRfiBpAg/kBThB5Ii\n/EBShB9IivADSdHPvwKU9Tk32SfdZn/3o48+OnR5lXsQqt7/cC4o3fPb3mP7pO1Di+bda/u47QPF\nY1OzZQKo2yiH/U9K2rjE/IcjYkPxeKHesgA0rTT8EfGKpI8nUAuACapywW+H7YPFacGq5Vayvc12\nz3av3+9X2ByAOo0b/sckXSZpg6QFSQ8ut2JE7I6IbkR0O53OmJsDULexwh8RJyLi84j4QtLjkq6s\ntywATRsr/LbXLnp5o6RDy60LYDqV9vPbfkbSNZJW2/5Q0j2SrrG9QVJIOibptgZrnArD+pwzfPe7\nDWXteq6ONTAppeGPiJuXmP1EA7UAmCBu7wWSIvxAUoQfSIrwA0kRfiApvtJbOHr06NDlw7qV6Opr\nxpYtWxr7bL7Sy54fSIvwA0kRfiApwg8kRfiBpAg/kBThB5Kin7+wc+fOsd9bdo/A7Ozs2J+90g1r\nm/Xr1ze67SNHjiy7LPO/yRns+YGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKfr5a1DWX71r165Kn19l\nvICyn7muapqHz6Yvfzj2/EBShB9IivADSRF+ICnCDyRF+IGkCD+QlCNi+Ar2OklPSVqjwU9y746I\nR2xfLOnPkmY0+JnumyLi38M+q9vtRq/Xq6Hs+tluuwScZfPmzUOXz83NTaiSlaPb7arX6430n3mU\nPf9nku6KiB9K+qmk7bZ/KOluSfsj4nJJ+4vXAFaI0vBHxEJEvFlMn5Z0WNIlkq6XtLdYba+kG5oq\nEkD9vtY5v+0ZSVdIek3SmohYKBZ9pMFpAYAVYuTw275Q0pykOyPik8XLYnDhYMmLB7a32e7Z7vX7\n/UrFAqjPSOG3fb4GwX86IuaL2Sdsry2Wr5V0cqn3RsTuiOhGRLfT6dRRM4AalIbfg8vgT0g6HBEP\nLVq0T9LWYnqrpOfrLw9AU0b5Su9Vkm6R9JbtA8W8nZLul/Ss7VslfSDppmZKnIxhwzxLzQ8znVFZ\nV9599903oUpyKg1/RLwqabl+Q37kHFihuMMPSIrwA0kRfiApwg8kRfiBpAg/kBRDdxfKhnkedh9A\n2c97z8/PD10+zar2xTN89vRizw8kRfiBpAg/kBThB5Ii/EBShB9IivADSdHPP6Jh/dUMIY2ViD0/\nkBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJFUaftvr\nbL9s+x3bb9u+o5h/r+3jtg8Uj03NlwugLqMM5vGZpLsi4k3bF0l6w/ZLxbKHI+KB5soD0JTS8EfE\ngqSFYvq07cOSLmm6MADN+lrn/LZnJF0h6bVi1g7bB23vsb2q5toANGjk8Nu+UNKcpDsj4hNJj0m6\nTNIGDY4MHlzmfdts92z3+v1+DSUDqMNI4bd9vgbBfzoi5iUpIk5ExOcR8YWkxyVdudR7I2J3RHQj\notvpdOqqG0BFo1ztt6QnJB2OiIcWzV+7aLUbJR2qvzwATRnlav9Vkm6R9JbtA8W8nZJutr1BUkg6\nJum2RioE0IhRrva/KslLLHqh/nIATAp3+AFJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kR\nfiApwg8kRfiBpAg/kBThB5JyRExuY3Zf0gcT2yCQz/ciYqQhsyYafgDTg8N+ICnCDyRF+IGkCD+Q\nFOEHkiL8QFKEH0iK8ANJEX4gqf8BTGfuydc60JgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x121114470>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index: 9427\n",
      "Predicted: 3, Actual: 5\n",
      "y_prob: [ 0.    0.    0.    0.61  0.    0.38  0.    0.    0.    0.  ]\n"
     ]
    }
   ],
   "source": [
    "from random import sample\n",
    "for i in sample(list(errors[0]), 5):\n",
    "    review(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}