{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: GeForce GTX TITAN X (CNMeM is enabled with initial size: 90.0% of memory, cuDNN 4007)\n"
     ]
    }
   ],
   "source": [
    "from theano.sandbox import cuda\n",
    "cuda.use('gpu0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import utils; reload(utils)\n",
    "from utils import *\n",
    "from __future__ import division, print_function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = \"data/dogscats/\"\n",
    "model_path = path + 'models/'\n",
    "if not os.path.exists(model_path): os.mkdir(model_path)\n",
    "\n",
    "batch_size=64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 23000 images belonging to 2 classes.\n",
      "Found 2000 images belonging to 2 classes.\n"
     ]
    }
   ],
   "source": [
    "batches = get_batches(path+'train', shuffle=False, batch_size=batch_size)\n",
    "val_batches = get_batches(path+'valid', shuffle=False, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 23000 images belonging to 2 classes.\n",
      "Found 2000 images belonging to 2 classes.\n",
      "Found 0 images belonging to 0 classes.\n"
     ]
    }
   ],
   "source": [
    "(val_classes, trn_classes, val_labels, trn_labels, \n",
    "    val_filenames, filenames, test_filenames) = get_classes(path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook we're going to create an ensemble of models and use their average as our predictions. For each ensemble, we're going to follow our usual fine-tuning steps:\n",
    "\n",
    "1) Create a model that retrains just the last layer\n",
    "2) Add this to a model containing all VGG layers except the last layer\n",
    "3) Fine-tune just the dense layers of this model (pre-computing the convolutional layers)\n",
    "4) Add data augmentation, fine-tuning the dense layers without pre-computation.\n",
    "\n",
    "So first, we need to create our VGG model and pre-compute the output of the conv layers:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Vgg16().model\n",
    "conv_layers,fc_layers = split_at(model, Convolution2D)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "conv_model = Sequential(conv_layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_features = conv_model.predict_generator(val_batches, val_batches.nb_sample)\n",
    "trn_features = conv_model.predict_generator(batches, batches.nb_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "save_array(model_path + 'train_convlayer_features.bc', trn_features)\n",
    "save_array(model_path + 'valid_convlayer_features.bc', val_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the future we can just load these precomputed features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trn_features = load_array(model_path+'train_convlayer_features.bc')\n",
    "val_features = load_array(model_path+'valid_convlayer_features.bc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also save some time by pre-computing the training and validation arrays with the image decoding and resizing already done:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 23000 images belonging to 2 classes.\n",
      "Found 2000 images belonging to 2 classes.\n"
     ]
    }
   ],
   "source": [
    "trn = get_data(path+'train')\n",
    "val = get_data(path+'valid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "save_array(model_path+'train_data.bc', trn)\n",
    "save_array(model_path+'valid_data.bc', val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the future we can just load these resized images:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trn = load_array(model_path+'train_data.bc')\n",
    "val = load_array(model_path+'valid_data.bc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we can precompute the output of all but the last dropout and dense layers, for creating the first stage of the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.pop()\n",
    "model.pop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ll_val_feat = model.predict_generator(val_batches, val_batches.nb_sample)\n",
    "ll_feat = model.predict_generator(batches, batches.nb_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "save_array(model_path + 'train_ll_feat.bc', ll_feat)\n",
    "save_array(model_path + 'valid_ll_feat.bc', ll_val_feat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ll_feat = load_array(model_path+ 'train_ll_feat.bc')\n",
    "ll_val_feat = load_array(model_path + 'valid_ll_feat.bc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "...and let's also grab the test data, for when we need to submit:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 12500 images belonging to 1 classes.\n"
     ]
    }
   ],
   "source": [
    "test = get_data(path+'test')\n",
    "save_array(model_path+'test_data.bc', test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test = load_array(model_path+'test_data.bc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Last layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The functions automate creating a model that trains the last layer from scratch, and then adds those new layers on to the main model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_ll_layers():\n",
    "    return [ \n",
    "        BatchNormalization(input_shape=(4096,)),\n",
    "        Dropout(0.5),\n",
    "        Dense(2, activation='softmax') \n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def train_last_layer(i):\n",
    "    ll_layers = get_ll_layers()\n",
    "    ll_model = Sequential(ll_layers)\n",
    "    ll_model.compile(optimizer=Adam(), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "    ll_model.optimizer.lr=1e-5\n",
    "    ll_model.fit(ll_feat, trn_labels, validation_data=(ll_val_feat, val_labels), nb_epoch=12)\n",
    "    ll_model.optimizer.lr=1e-7\n",
    "    ll_model.fit(ll_feat, trn_labels, validation_data=(ll_val_feat, val_labels), nb_epoch=1)\n",
    "    ll_model.save_weights(model_path+'ll_bn' + i + '.h5')\n",
    "\n",
    "    vgg = Vgg16()\n",
    "    model = vgg.model\n",
    "    model.pop(); model.pop(); model.pop()\n",
    "    for layer in model.layers: layer.trainable=False\n",
    "    model.compile(optimizer=Adam(), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "    ll_layers = get_ll_layers()\n",
    "    for layer in ll_layers: model.add(layer)\n",
    "    for l1,l2 in zip(ll_model.layers, model.layers[-3:]):\n",
    "        l2.set_weights(l1.get_weights())\n",
    "    model.compile(optimizer=Adam(), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "    model.save_weights(model_path+'bn' + i + '.h5')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dense model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_conv_model(model):\n",
    "    layers = model.layers\n",
    "    last_conv_idx = [index for index,layer in enumerate(layers) \n",
    "                         if type(layer) is Convolution2D][-1]\n",
    "\n",
    "    conv_layers = layers[:last_conv_idx+1]\n",
    "    conv_model = Sequential(conv_layers)\n",
    "    fc_layers = layers[last_conv_idx+1:]\n",
    "    return conv_model, fc_layers, last_conv_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_fc_layers(p, in_shape):\n",
    "    return [\n",
    "        MaxPooling2D(input_shape=in_shape),\n",
    "        Flatten(),\n",
    "        Dense(4096, activation='relu'),\n",
    "        BatchNormalization(),\n",
    "        Dropout(p),\n",
    "        Dense(4096, activation='relu'),\n",
    "        BatchNormalization(),\n",
    "        Dropout(p),\n",
    "        Dense(2, activation='softmax')\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_dense_layers(i, model):\n",
    "    conv_model, fc_layers, last_conv_idx = get_conv_model(model)\n",
    "    conv_shape = conv_model.output_shape[1:]\n",
    "    fc_model = Sequential(get_fc_layers(0.5, conv_shape))\n",
    "    for l1,l2 in zip(fc_model.layers, fc_layers): \n",
    "        weights = l2.get_weights()\n",
    "        l1.set_weights(weights)\n",
    "    fc_model.compile(optimizer=Adam(1e-5), loss='categorical_crossentropy', \n",
    "                     metrics=['accuracy'])\n",
    "    fc_model.fit(trn_features, trn_labels, nb_epoch=2, \n",
    "         batch_size=batch_size, validation_data=(val_features, val_labels))\n",
    "\n",
    "    gen = image.ImageDataGenerator(rotation_range=10, width_shift_range=0.05, \n",
    "       width_zoom_range=0.05, zoom_range=0.05,\n",
    "       channel_shift_range=10, height_shift_range=0.05, shear_range=0.05, horizontal_flip=True)\n",
    "    batches = gen.flow(trn, trn_labels, batch_size=batch_size)\n",
    "    val_batches = image.ImageDataGenerator().flow(val, val_labels, \n",
    "                      shuffle=False, batch_size=batch_size)\n",
    "\n",
    "    for layer in conv_model.layers: layer.trainable = False\n",
    "    for layer in get_fc_layers(0.5, conv_shape): conv_model.add(layer)\n",
    "    for l1,l2 in zip(conv_model.layers[last_conv_idx+1:], fc_model.layers): \n",
    "        l1.set_weights(l2.get_weights())\n",
    "\n",
    "    conv_model.compile(optimizer=Adam(1e-5), loss='categorical_crossentropy', \n",
    "                       metrics=['accuracy'])\n",
    "    conv_model.save_weights(model_path+'no_dropout_bn' + i + '.h5')\n",
    "    conv_model.fit_generator(batches, samples_per_epoch=batches.N, nb_epoch=1, \n",
    "                            validation_data=val_batches, nb_val_samples=val_batches.N)\n",
    "    for layer in conv_model.layers[16:]: layer.trainable = True\n",
    "    conv_model.fit_generator(batches, samples_per_epoch=batches.N, nb_epoch=8, \n",
    "                            validation_data=val_batches, nb_val_samples=val_batches.N)\n",
    "\n",
    "    conv_model.optimizer.lr = 1e-7\n",
    "    conv_model.fit_generator(batches, samples_per_epoch=batches.N, nb_epoch=10, \n",
    "                            validation_data=val_batches, nb_val_samples=val_batches.N)\n",
    "    conv_model.save_weights(model_path + 'aug' + i + '.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Build ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false,
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.5184 - acc: 0.7895 - val_loss: 0.1549 - val_acc: 0.9440\n",
      "Epoch 2/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1984 - acc: 0.9237 - val_loss: 0.0941 - val_acc: 0.9670\n",
      "Epoch 3/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1524 - acc: 0.9426 - val_loss: 0.0762 - val_acc: 0.9735\n",
      "Epoch 4/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1247 - acc: 0.9542 - val_loss: 0.0662 - val_acc: 0.9740\n",
      "Epoch 5/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1128 - acc: 0.9567 - val_loss: 0.0609 - val_acc: 0.9760\n",
      "Epoch 6/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1043 - acc: 0.9635 - val_loss: 0.0560 - val_acc: 0.9775\n",
      "Epoch 7/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1010 - acc: 0.9640 - val_loss: 0.0548 - val_acc: 0.9790\n",
      "Epoch 8/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0980 - acc: 0.9650 - val_loss: 0.0526 - val_acc: 0.9780\n",
      "Epoch 9/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0926 - acc: 0.9656 - val_loss: 0.0513 - val_acc: 0.9785\n",
      "Epoch 10/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0881 - acc: 0.9680 - val_loss: 0.0500 - val_acc: 0.9795\n",
      "Epoch 11/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0933 - acc: 0.9666 - val_loss: 0.0497 - val_acc: 0.9800\n",
      "Epoch 12/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0842 - acc: 0.9693 - val_loss: 0.0484 - val_acc: 0.9805\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "23000/23000 [==============================] - 0s - loss: 0.0824 - acc: 0.9696 - val_loss: 0.0486 - val_acc: 0.9805\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/2\n",
      "23000/23000 [==============================] - 21s - loss: 0.0798 - acc: 0.9719 - val_loss: 0.0500 - val_acc: 0.9830\n",
      "Epoch 2/2\n",
      "23000/23000 [==============================] - 21s - loss: 0.0415 - acc: 0.9853 - val_loss: 0.0551 - val_acc: 0.9840\n",
      "Epoch 1/1\n",
      "23000/23000 [==============================] - 271s - loss: 0.0559 - acc: 0.9814 - val_loss: 0.0578 - val_acc: 0.9825\n",
      "Epoch 1/8\n",
      "23000/23000 [==============================] - 271s - loss: 0.0515 - acc: 0.9834 - val_loss: 0.0645 - val_acc: 0.9860\n",
      "Epoch 2/8\n",
      "23000/23000 [==============================] - 271s - loss: 0.0385 - acc: 0.9875 - val_loss: 0.0670 - val_acc: 0.9850\n",
      "Epoch 3/8\n",
      "23000/23000 [==============================] - 271s - loss: 0.0313 - acc: 0.9890 - val_loss: 0.0715 - val_acc: 0.9850\n",
      "Epoch 4/8\n",
      "23000/23000 [==============================] - 271s - loss: 0.0287 - acc: 0.9903 - val_loss: 0.0733 - val_acc: 0.9840\n",
      "Epoch 5/8\n",
      "23000/23000 [==============================] - 271s - loss: 0.0244 - acc: 0.9924 - val_loss: 0.0773 - val_acc: 0.9840\n",
      "Epoch 6/8\n",
      "23000/23000 [==============================] - 271s - loss: 0.0205 - acc: 0.9927 - val_loss: 0.0900 - val_acc: 0.9845\n",
      "Epoch 7/8\n",
      "23000/23000 [==============================] - 271s - loss: 0.0209 - acc: 0.9929 - val_loss: 0.0860 - val_acc: 0.9865\n",
      "Epoch 8/8\n",
      "23000/23000 [==============================] - 420s - loss: 0.0186 - acc: 0.9930 - val_loss: 0.0923 - val_acc: 0.9845\n",
      "Epoch 1/10\n",
      "23000/23000 [==============================] - 315s - loss: 0.0196 - acc: 0.9930 - val_loss: 0.0909 - val_acc: 0.9845\n",
      "Epoch 2/10\n",
      "23000/23000 [==============================] - 362s - loss: 0.0165 - acc: 0.9945 - val_loss: 0.1023 - val_acc: 0.9830\n",
      "Epoch 3/10\n",
      "23000/23000 [==============================] - 447s - loss: 0.0179 - acc: 0.9940 - val_loss: 0.0871 - val_acc: 0.9845\n",
      "Epoch 4/10\n",
      "23000/23000 [==============================] - 601s - loss: 0.0112 - acc: 0.9960 - val_loss: 0.1030 - val_acc: 0.9830\n",
      "Epoch 5/10\n",
      "23000/23000 [==============================] - 528s - loss: 0.0130 - acc: 0.9956 - val_loss: 0.0946 - val_acc: 0.9830\n",
      "Epoch 6/10\n",
      "23000/23000 [==============================] - 657s - loss: 0.0110 - acc: 0.9961 - val_loss: 0.0904 - val_acc: 0.9850\n",
      "Epoch 7/10\n",
      "23000/23000 [==============================] - 621s - loss: 0.0116 - acc: 0.9963 - val_loss: 0.0872 - val_acc: 0.9865\n",
      "Epoch 8/10\n",
      "23000/23000 [==============================] - 603s - loss: 0.0118 - acc: 0.9960 - val_loss: 0.0813 - val_acc: 0.9870\n",
      "Epoch 9/10\n",
      "23000/23000 [==============================] - 616s - loss: 0.0100 - acc: 0.9967 - val_loss: 0.1053 - val_acc: 0.9835\n",
      "Epoch 10/10\n",
      "23000/23000 [==============================] - 661s - loss: 0.0098 - acc: 0.9968 - val_loss: 0.0970 - val_acc: 0.9840\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.5106 - acc: 0.7935 - val_loss: 0.1504 - val_acc: 0.9455\n",
      "Epoch 2/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.2005 - acc: 0.9241 - val_loss: 0.0890 - val_acc: 0.9680\n",
      "Epoch 3/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1465 - acc: 0.9444 - val_loss: 0.0714 - val_acc: 0.9745\n",
      "Epoch 4/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1280 - acc: 0.9540 - val_loss: 0.0614 - val_acc: 0.9765\n",
      "Epoch 5/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1131 - acc: 0.9586 - val_loss: 0.0561 - val_acc: 0.9795\n",
      "Epoch 6/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1079 - acc: 0.9620 - val_loss: 0.0515 - val_acc: 0.9795\n",
      "Epoch 7/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0998 - acc: 0.9631 - val_loss: 0.0484 - val_acc: 0.9825\n",
      "Epoch 8/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0947 - acc: 0.9673 - val_loss: 0.0457 - val_acc: 0.9845\n",
      "Epoch 9/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0913 - acc: 0.9676 - val_loss: 0.0449 - val_acc: 0.9855\n",
      "Epoch 10/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0921 - acc: 0.9670 - val_loss: 0.0451 - val_acc: 0.9845\n",
      "Epoch 11/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0893 - acc: 0.9681 - val_loss: 0.0441 - val_acc: 0.9840\n",
      "Epoch 12/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0836 - acc: 0.9691 - val_loss: 0.0428 - val_acc: 0.9850\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "23000/23000 [==============================] - 0s - loss: 0.0833 - acc: 0.9718 - val_loss: 0.0434 - val_acc: 0.9850\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/2\n",
      "23000/23000 [==============================] - 21s - loss: 0.0814 - acc: 0.9736 - val_loss: 0.0463 - val_acc: 0.9850\n",
      "Epoch 2/2\n",
      "23000/23000 [==============================] - 21s - loss: 0.0389 - acc: 0.9859 - val_loss: 0.0493 - val_acc: 0.9850\n",
      "Epoch 1/1\n",
      "23000/23000 [==============================] - 271s - loss: 0.0613 - acc: 0.9807 - val_loss: 0.0563 - val_acc: 0.9855\n",
      "Epoch 1/8\n",
      "23000/23000 [==============================] - 325s - loss: 0.0450 - acc: 0.9860 - val_loss: 0.0685 - val_acc: 0.9840\n",
      "Epoch 2/8\n",
      "23000/23000 [==============================] - 766s - loss: 0.0364 - acc: 0.9877 - val_loss: 0.0616 - val_acc: 0.9845\n",
      "Epoch 3/8\n",
      "23000/23000 [==============================] - 600s - loss: 0.0338 - acc: 0.9891 - val_loss: 0.0585 - val_acc: 0.9845\n",
      "Epoch 4/8\n",
      "23000/23000 [==============================] - 634s - loss: 0.0288 - acc: 0.9903 - val_loss: 0.0740 - val_acc: 0.9845\n",
      "Epoch 5/8\n",
      "23000/23000 [==============================] - 791s - loss: 0.0265 - acc: 0.9904 - val_loss: 0.0789 - val_acc: 0.9840\n",
      "Epoch 6/8\n",
      "23000/23000 [==============================] - 780s - loss: 0.0254 - acc: 0.9909 - val_loss: 0.0853 - val_acc: 0.9855\n",
      "Epoch 7/8\n",
      "23000/23000 [==============================] - 680s - loss: 0.0180 - acc: 0.9937 - val_loss: 0.0747 - val_acc: 0.9870\n",
      "Epoch 8/8\n",
      "23000/23000 [==============================] - 776s - loss: 0.0191 - acc: 0.9939 - val_loss: 0.0871 - val_acc: 0.9845\n",
      "Epoch 1/10\n",
      "23000/23000 [==============================] - 712s - loss: 0.0191 - acc: 0.9929 - val_loss: 0.0943 - val_acc: 0.9855\n",
      "Epoch 2/10\n",
      "23000/23000 [==============================] - 679s - loss: 0.0175 - acc: 0.9946 - val_loss: 0.0723 - val_acc: 0.9850\n",
      "Epoch 3/10\n",
      "23000/23000 [==============================] - 640s - loss: 0.0148 - acc: 0.9949 - val_loss: 0.0756 - val_acc: 0.9845\n",
      "Epoch 4/10\n",
      "23000/23000 [==============================] - 761s - loss: 0.0147 - acc: 0.9953 - val_loss: 0.0772 - val_acc: 0.9850\n",
      "Epoch 5/10\n",
      "23000/23000 [==============================] - 733s - loss: 0.0163 - acc: 0.9946 - val_loss: 0.0931 - val_acc: 0.9830\n",
      "Epoch 6/10\n",
      "23000/23000 [==============================] - 574s - loss: 0.0107 - acc: 0.9967 - val_loss: 0.0874 - val_acc: 0.9845\n",
      "Epoch 7/10\n",
      "23000/23000 [==============================] - 611s - loss: 0.0123 - acc: 0.9958 - val_loss: 0.0918 - val_acc: 0.9855\n",
      "Epoch 8/10\n",
      "23000/23000 [==============================] - 668s - loss: 0.0098 - acc: 0.9965 - val_loss: 0.0896 - val_acc: 0.9855\n",
      "Epoch 9/10\n",
      "23000/23000 [==============================] - 624s - loss: 0.0096 - acc: 0.9964 - val_loss: 0.1012 - val_acc: 0.9850\n",
      "Epoch 10/10\n",
      "23000/23000 [==============================] - 747s - loss: 0.0113 - acc: 0.9960 - val_loss: 0.0961 - val_acc: 0.9835\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/12\n",
      "23000/23000 [==============================] - 1s - loss: 0.5167 - acc: 0.7867 - val_loss: 0.1299 - val_acc: 0.9555\n",
      "Epoch 2/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1922 - acc: 0.9265 - val_loss: 0.0803 - val_acc: 0.9695\n",
      "Epoch 3/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1461 - acc: 0.9454 - val_loss: 0.0646 - val_acc: 0.9745\n",
      "Epoch 4/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1255 - acc: 0.9536 - val_loss: 0.0543 - val_acc: 0.9790\n",
      "Epoch 5/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1113 - acc: 0.9608 - val_loss: 0.0505 - val_acc: 0.9820\n",
      "Epoch 6/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1058 - acc: 0.9607 - val_loss: 0.0464 - val_acc: 0.9825\n",
      "Epoch 7/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0957 - acc: 0.9654 - val_loss: 0.0448 - val_acc: 0.9840\n",
      "Epoch 8/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0964 - acc: 0.9657 - val_loss: 0.0427 - val_acc: 0.9850\n",
      "Epoch 9/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0996 - acc: 0.9662 - val_loss: 0.0420 - val_acc: 0.9860\n",
      "Epoch 10/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0931 - acc: 0.9670 - val_loss: 0.0408 - val_acc: 0.9855\n",
      "Epoch 11/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0899 - acc: 0.9680 - val_loss: 0.0395 - val_acc: 0.9860\n",
      "Epoch 12/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0837 - acc: 0.9717 - val_loss: 0.0390 - val_acc: 0.9860\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "23000/23000 [==============================] - 0s - loss: 0.0861 - acc: 0.9703 - val_loss: 0.0391 - val_acc: 0.9865\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/2\n",
      "23000/23000 [==============================] - 21s - loss: 0.0796 - acc: 0.9735 - val_loss: 0.0382 - val_acc: 0.9855\n",
      "Epoch 2/2\n",
      "23000/23000 [==============================] - 21s - loss: 0.0353 - acc: 0.9874 - val_loss: 0.0364 - val_acc: 0.9880\n",
      "Epoch 1/1\n",
      "23000/23000 [==============================] - 271s - loss: 0.0622 - acc: 0.9802 - val_loss: 0.0490 - val_acc: 0.9870\n",
      "Epoch 1/8\n",
      "23000/23000 [==============================] - 773s - loss: 0.0426 - acc: 0.9856 - val_loss: 0.0442 - val_acc: 0.9885\n",
      "Epoch 2/8\n",
      "23000/23000 [==============================] - 774s - loss: 0.0394 - acc: 0.9864 - val_loss: 0.0501 - val_acc: 0.9885\n",
      "Epoch 3/8\n",
      "23000/23000 [==============================] - 687s - loss: 0.0329 - acc: 0.9881 - val_loss: 0.0500 - val_acc: 0.9875\n",
      "Epoch 4/8\n",
      "23000/23000 [==============================] - 655s - loss: 0.0292 - acc: 0.9900 - val_loss: 0.0535 - val_acc: 0.9870\n",
      "Epoch 5/8\n",
      "23000/23000 [==============================] - 791s - loss: 0.0268 - acc: 0.9914 - val_loss: 0.0605 - val_acc: 0.9855\n",
      "Epoch 6/8\n",
      "23000/23000 [==============================] - 789s - loss: 0.0208 - acc: 0.9926 - val_loss: 0.0591 - val_acc: 0.9850\n",
      "Epoch 7/8\n",
      "23000/23000 [==============================] - 798s - loss: 0.0191 - acc: 0.9931 - val_loss: 0.0638 - val_acc: 0.9860\n",
      "Epoch 8/8\n",
      "23000/23000 [==============================] - 708s - loss: 0.0192 - acc: 0.9932 - val_loss: 0.0597 - val_acc: 0.9850\n",
      "Epoch 1/10\n",
      "23000/23000 [==============================] - 606s - loss: 0.0178 - acc: 0.9942 - val_loss: 0.0620 - val_acc: 0.9860\n",
      "Epoch 2/10\n",
      "23000/23000 [==============================] - 756s - loss: 0.0158 - acc: 0.9941 - val_loss: 0.0694 - val_acc: 0.9850\n",
      "Epoch 3/10\n",
      "23000/23000 [==============================] - 418s - loss: 0.0176 - acc: 0.9939 - val_loss: 0.0641 - val_acc: 0.9855\n",
      "Epoch 4/10\n",
      "23000/23000 [==============================] - 271s - loss: 0.0118 - acc: 0.9958 - val_loss: 0.0623 - val_acc: 0.9840\n",
      "Epoch 5/10\n",
      "23000/23000 [==============================] - 271s - loss: 0.0150 - acc: 0.9947 - val_loss: 0.0649 - val_acc: 0.9865\n",
      "Epoch 6/10\n",
      "23000/23000 [==============================] - 271s - loss: 0.0119 - acc: 0.9961 - val_loss: 0.0595 - val_acc: 0.9880\n",
      "Epoch 7/10\n",
      "23000/23000 [==============================] - 304s - loss: 0.0121 - acc: 0.9957 - val_loss: 0.0668 - val_acc: 0.9885\n",
      "Epoch 8/10\n",
      "23000/23000 [==============================] - 273s - loss: 0.0124 - acc: 0.9960 - val_loss: 0.0619 - val_acc: 0.9885\n",
      "Epoch 9/10\n",
      "23000/23000 [==============================] - 271s - loss: 0.0099 - acc: 0.9963 - val_loss: 0.0649 - val_acc: 0.9865\n",
      "Epoch 10/10\n",
      "23000/23000 [==============================] - 273s - loss: 0.0091 - acc: 0.9970 - val_loss: 0.0628 - val_acc: 0.9890\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.4585 - acc: 0.8130 - val_loss: 0.1306 - val_acc: 0.9515\n",
      "Epoch 2/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1920 - acc: 0.9270 - val_loss: 0.0863 - val_acc: 0.9655\n",
      "Epoch 3/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1504 - acc: 0.9450 - val_loss: 0.0705 - val_acc: 0.9740\n",
      "Epoch 4/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1275 - acc: 0.9529 - val_loss: 0.0592 - val_acc: 0.9795\n",
      "Epoch 5/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1190 - acc: 0.9555 - val_loss: 0.0555 - val_acc: 0.9815\n",
      "Epoch 6/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1068 - acc: 0.9609 - val_loss: 0.0536 - val_acc: 0.9805\n",
      "Epoch 7/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1003 - acc: 0.9624 - val_loss: 0.0496 - val_acc: 0.9830\n",
      "Epoch 8/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0979 - acc: 0.9660 - val_loss: 0.0482 - val_acc: 0.9825\n",
      "Epoch 9/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0913 - acc: 0.9678 - val_loss: 0.0475 - val_acc: 0.9830\n",
      "Epoch 10/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0917 - acc: 0.9666 - val_loss: 0.0458 - val_acc: 0.9825\n",
      "Epoch 11/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0980 - acc: 0.9665 - val_loss: 0.0454 - val_acc: 0.9840\n",
      "Epoch 12/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0919 - acc: 0.9675 - val_loss: 0.0443 - val_acc: 0.9840\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "23000/23000 [==============================] - 0s - loss: 0.0883 - acc: 0.9685 - val_loss: 0.0440 - val_acc: 0.9850\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/2\n",
      "23000/23000 [==============================] - 21s - loss: 0.0825 - acc: 0.9720 - val_loss: 0.0437 - val_acc: 0.9850\n",
      "Epoch 2/2\n",
      "23000/23000 [==============================] - 21s - loss: 0.0359 - acc: 0.9874 - val_loss: 0.0474 - val_acc: 0.9850\n",
      "Epoch 1/1\n",
      "23000/23000 [==============================] - 272s - loss: 0.0581 - acc: 0.9817 - val_loss: 0.0562 - val_acc: 0.9850\n",
      "Epoch 1/8\n",
      "23000/23000 [==============================] - 520s - loss: 0.0486 - acc: 0.9833 - val_loss: 0.0590 - val_acc: 0.9830\n",
      "Epoch 2/8\n",
      "23000/23000 [==============================] - 745s - loss: 0.0379 - acc: 0.9867 - val_loss: 0.0595 - val_acc: 0.9840\n",
      "Epoch 3/8\n",
      "23000/23000 [==============================] - 736s - loss: 0.0329 - acc: 0.9881 - val_loss: 0.0628 - val_acc: 0.9840\n",
      "Epoch 4/8\n",
      "23000/23000 [==============================] - 708s - loss: 0.0260 - acc: 0.9903 - val_loss: 0.0722 - val_acc: 0.9855\n",
      "Epoch 5/8\n",
      "23000/23000 [==============================] - 700s - loss: 0.0250 - acc: 0.9921 - val_loss: 0.0734 - val_acc: 0.9840\n",
      "Epoch 6/8\n",
      "23000/23000 [==============================] - 802s - loss: 0.0212 - acc: 0.9923 - val_loss: 0.0721 - val_acc: 0.9845\n",
      "Epoch 7/8\n",
      "23000/23000 [==============================] - 765s - loss: 0.0211 - acc: 0.9928 - val_loss: 0.0772 - val_acc: 0.9835\n",
      "Epoch 8/8\n",
      "23000/23000 [==============================] - 743s - loss: 0.0185 - acc: 0.9933 - val_loss: 0.0756 - val_acc: 0.9835\n",
      "Epoch 1/10\n",
      "23000/23000 [==============================] - 782s - loss: 0.0168 - acc: 0.9941 - val_loss: 0.0815 - val_acc: 0.9860\n",
      "Epoch 2/10\n",
      "23000/23000 [==============================] - 580s - loss: 0.0155 - acc: 0.9942 - val_loss: 0.0771 - val_acc: 0.9840\n",
      "Epoch 3/10\n",
      "23000/23000 [==============================] - 654s - loss: 0.0142 - acc: 0.9954 - val_loss: 0.0789 - val_acc: 0.9850\n",
      "Epoch 4/10\n",
      "23000/23000 [==============================] - 692s - loss: 0.0141 - acc: 0.9955 - val_loss: 0.0716 - val_acc: 0.9870\n",
      "Epoch 5/10\n",
      "23000/23000 [==============================] - 607s - loss: 0.0120 - acc: 0.9959 - val_loss: 0.0757 - val_acc: 0.9850\n",
      "Epoch 6/10\n",
      "23000/23000 [==============================] - 789s - loss: 0.0129 - acc: 0.9956 - val_loss: 0.0741 - val_acc: 0.9860\n",
      "Epoch 7/10\n",
      "23000/23000 [==============================] - 767s - loss: 0.0111 - acc: 0.9960 - val_loss: 0.0747 - val_acc: 0.9865\n",
      "Epoch 8/10\n",
      "23000/23000 [==============================] - 557s - loss: 0.0103 - acc: 0.9967 - val_loss: 0.0774 - val_acc: 0.9870\n",
      "Epoch 9/10\n",
      "23000/23000 [==============================] - 521s - loss: 0.0106 - acc: 0.9962 - val_loss: 0.0855 - val_acc: 0.9855\n",
      "Epoch 10/10\n",
      "23000/23000 [==============================] - 484s - loss: 0.0095 - acc: 0.9970 - val_loss: 0.0780 - val_acc: 0.9850\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.5435 - acc: 0.7783 - val_loss: 0.1669 - val_acc: 0.9440\n",
      "Epoch 2/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.2054 - acc: 0.9227 - val_loss: 0.0999 - val_acc: 0.9675\n",
      "Epoch 3/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1549 - acc: 0.9405 - val_loss: 0.0763 - val_acc: 0.9725\n",
      "Epoch 4/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1327 - acc: 0.9520 - val_loss: 0.0642 - val_acc: 0.9755\n",
      "Epoch 5/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1147 - acc: 0.9573 - val_loss: 0.0590 - val_acc: 0.9790\n",
      "Epoch 6/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1078 - acc: 0.9605 - val_loss: 0.0545 - val_acc: 0.9815\n",
      "Epoch 7/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.1001 - acc: 0.9631 - val_loss: 0.0526 - val_acc: 0.9820\n",
      "Epoch 8/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0977 - acc: 0.9654 - val_loss: 0.0515 - val_acc: 0.9815\n",
      "Epoch 9/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0937 - acc: 0.9660 - val_loss: 0.0497 - val_acc: 0.9825\n",
      "Epoch 10/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0942 - acc: 0.9683 - val_loss: 0.0489 - val_acc: 0.9835\n",
      "Epoch 11/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0904 - acc: 0.9687 - val_loss: 0.0473 - val_acc: 0.9830\n",
      "Epoch 12/12\n",
      "23000/23000 [==============================] - 0s - loss: 0.0855 - acc: 0.9689 - val_loss: 0.0469 - val_acc: 0.9840\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "23000/23000 [==============================] - 0s - loss: 0.0861 - acc: 0.9685 - val_loss: 0.0470 - val_acc: 0.9840\n",
      "Train on 23000 samples, validate on 2000 samples\n",
      "Epoch 1/2\n",
      "23000/23000 [==============================] - 21s - loss: 0.0846 - acc: 0.9719 - val_loss: 0.0510 - val_acc: 0.9845\n",
      "Epoch 2/2\n",
      "23000/23000 [==============================] - 21s - loss: 0.0392 - acc: 0.9866 - val_loss: 0.0548 - val_acc: 0.9860\n",
      "Epoch 1/1\n",
      "23000/23000 [==============================] - 273s - loss: 0.0585 - acc: 0.9800 - val_loss: 0.0608 - val_acc: 0.9875\n",
      "Epoch 1/8\n",
      "23000/23000 [==============================] - 677s - loss: 0.0456 - acc: 0.9845 - val_loss: 0.0690 - val_acc: 0.9840\n",
      "Epoch 2/8\n",
      "23000/23000 [==============================] - 654s - loss: 0.0398 - acc: 0.9859 - val_loss: 0.0763 - val_acc: 0.9835\n",
      "Epoch 3/8\n",
      "23000/23000 [==============================] - 711s - loss: 0.0304 - acc: 0.9894 - val_loss: 0.0662 - val_acc: 0.9840\n",
      "Epoch 4/8\n",
      "23000/23000 [==============================] - 646s - loss: 0.0252 - acc: 0.9913 - val_loss: 0.0747 - val_acc: 0.9845\n",
      "Epoch 5/8\n",
      "23000/23000 [==============================] - 726s - loss: 0.0246 - acc: 0.9909 - val_loss: 0.0809 - val_acc: 0.9850\n",
      "Epoch 6/8\n",
      "23000/23000 [==============================] - 582s - loss: 0.0182 - acc: 0.9933 - val_loss: 0.0715 - val_acc: 0.9850\n",
      "Epoch 7/8\n",
      "23000/23000 [==============================] - 627s - loss: 0.0201 - acc: 0.9928 - val_loss: 0.0789 - val_acc: 0.9850\n",
      "Epoch 8/8\n",
      "23000/23000 [==============================] - 674s - loss: 0.0172 - acc: 0.9944 - val_loss: 0.0717 - val_acc: 0.9855\n",
      "Epoch 1/10\n",
      "23000/23000 [==============================] - 736s - loss: 0.0171 - acc: 0.9939 - val_loss: 0.0820 - val_acc: 0.9850\n",
      "Epoch 2/10\n",
      "23000/23000 [==============================] - 634s - loss: 0.0184 - acc: 0.9941 - val_loss: 0.0829 - val_acc: 0.9860\n",
      "Epoch 3/10\n",
      "23000/23000 [==============================] - 599s - loss: 0.0156 - acc: 0.9946 - val_loss: 0.0863 - val_acc: 0.9865\n",
      "Epoch 4/10\n",
      "23000/23000 [==============================] - 717s - loss: 0.0142 - acc: 0.9952 - val_loss: 0.0903 - val_acc: 0.9850\n",
      "Epoch 5/10\n",
      "23000/23000 [==============================] - 809s - loss: 0.0116 - acc: 0.9960 - val_loss: 0.0883 - val_acc: 0.9860\n",
      "Epoch 6/10\n",
      "23000/23000 [==============================] - 754s - loss: 0.0127 - acc: 0.9953 - val_loss: 0.0887 - val_acc: 0.9855\n",
      "Epoch 7/10\n",
      "23000/23000 [==============================] - 499s - loss: 0.0100 - acc: 0.9964 - val_loss: 0.0835 - val_acc: 0.9850\n",
      "Epoch 8/10\n",
      "23000/23000 [==============================] - 317s - loss: 0.0090 - acc: 0.9971 - val_loss: 0.0804 - val_acc: 0.9870\n",
      "Epoch 9/10\n",
      "23000/23000 [==============================] - 301s - loss: 0.0111 - acc: 0.9963 - val_loss: 0.0869 - val_acc: 0.9865\n",
      "Epoch 10/10\n",
      "23000/23000 [==============================] - 442s - loss: 0.0079 - acc: 0.9971 - val_loss: 0.0805 - val_acc: 0.9870\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    i = str(i)\n",
    "    model = train_last_layer(i)\n",
    "    train_dense_layers(i, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine ensemble and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ens_model = vgg_ft(2)\n",
    "for layer in ens_model.layers: layer.trainable=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_ens_pred(arr, fname):\n",
    "    ens_pred = []\n",
    "    for i in range(5):\n",
    "        i = str(i)\n",
    "        ens_model.load_weights('{}{}{}.h5'.format(model_path, fname, i))\n",
    "        preds = ens_model.predict(arr, batch_size=batch_size)\n",
    "        ens_pred.append(preds)\n",
    "    return ens_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "val_pred2 = get_ens_pred(val, 'aug')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_avg_preds2 = np.stack(val_pred2).mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(0.9894999861717224, dtype=float32)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical_accuracy(val_labels, val_avg_preds2).eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  },
  "nav_menu": {},
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}