{ "cells": [ { "cell_type": "markdown", "metadata": { "toc": "true" }, "source": [ " # Table of Contents\n", "
" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Load libraries" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:32:24.349721Z", "start_time": "2017-09-07T07:32:10.527862Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Using TensorFlow backend.\n" ] } ], "source": [ "import cv2\n", "import numpy as np\n", "import pandas as pd\n", "\n", "from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, TensorBoard\n", "from keras.models import Model\n", "from keras.layers import Input, concatenate, Conv2D, MaxPooling2D, Activation, UpSampling2D, BatchNormalization\n", "from keras.optimizers import RMSprop\n", "from keras.losses import binary_crossentropy\n", "import keras.backend as K\n", "\n", "from sklearn.model_selection import train_test_split" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:32:24.480919Z", "start_time": "2017-09-07T07:32:24.358896Z" }, "collapsed": true }, "outputs": [], "source": [ "import random\n", "import gzip\n", "import pickle\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Define loss functions" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:32:24.497301Z", "start_time": "2017-09-07T07:32:24.489981Z" }, "collapsed": true }, "outputs": [], "source": [ "def dice_coeff(y_true, y_pred):\n", " smooth = 1.\n", " y_true_f = K.flatten(y_true)\n", " y_pred_f = K.flatten(y_pred)\n", " intersection = K.sum(y_true_f * y_pred_f)\n", " score = (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)\n", " return score\n", "\n", "\n", "def dice_loss(y_true, y_pred):\n", " loss = 1 - dice_coeff(y_true, y_pred)\n", " return loss\n", "\n", "def bce_dice_loss(y_true, y_pred):\n", " loss = binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)\n", " return loss" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Define models" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:32:24.554423Z", "start_time": "2017-09-07T07:32:24.506618Z" }, "collapsed": true }, "outputs": [], "source": [ "def unet_down_one_block(inputs, num_filters):\n", " x = Conv2D(num_filters, (3, 3), padding='same')(inputs)\n", " x = BatchNormalization()(x)\n", " x = Activation('relu')(x)\n", " x = Conv2D(num_filters, (3, 3), padding='same')(x)\n", " x = BatchNormalization()(x)\n", " x = Activation('relu')(x)\n", " return x" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:32:24.598503Z", "start_time": "2017-09-07T07:32:24.563838Z" }, "collapsed": true }, "outputs": [], "source": [ "def unet_max_pool(inputs):\n", " x = MaxPooling2D((2, 2), strides=(2, 2))(inputs)\n", " return x" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:32:24.679604Z", "start_time": "2017-09-07T07:32:24.672023Z" }, "collapsed": true }, "outputs": [], "source": [ "def unet_up_one_block(up_input, down_input, num_filters):\n", " x = UpSampling2D((2,2))(up_input)\n", " x = concatenate([down_input, x], axis=3)\n", " x = Conv2D(num_filters, (3,3), padding='same')(x)\n", " x = BatchNormalization()(x)\n", " x = Activation('relu')(x)\n", " x = Conv2D(num_filters, (3,3), padding='same')(x)\n", " x = BatchNormalization()(x)\n", " x = Activation('relu')(x)\n", " x = Conv2D(num_filters, (3,3), padding='same')(x)\n", " x = BatchNormalization()(x)\n", " x = Activation('relu')(x)\n", " return x" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:32:24.724212Z", "start_time": "2017-09-07T07:32:24.690133Z" }, "collapsed": true }, "outputs": [], "source": [ "def get_unet(input_shape = (256, 256, 3),\n", " num_classes = 1,\n", " initial_filters = 32,\n", " central_filters = 1024):\n", " \n", " num_filters = initial_filters\n", " \n", " out_list = [Input(shape=input_shape)]\n", " down_interim_list = []\n", " \n", " while num_filters <= central_filters/2:\n", " x = unet_down_one_block(out_list[-1], num_filters)\n", " down_interim_list.append(x)\n", " num_filters = num_filters * 2\n", " y = unet_max_pool(x)\n", " out_list.append(y)\n", " \n", " x = unet_down_one_block(out_list[-1], num_filters)\n", " out_list.append(x)\n", " num_filters = int(num_filters / 2)\n", " \n", " while num_filters >= initial_filters:\n", " x = unet_up_one_block(out_list[-1], down_interim_list.pop(), num_filters)\n", " out_list.append(x)\n", " num_filters = int(num_filters / 2)\n", " \n", " classify = Conv2D(num_classes, (1,1), activation = 'sigmoid')(out_list[-1])\n", " \n", " model = Model(inputs=out_list[0], outputs=classify)\n", " \n", " model.compile(optimizer=RMSprop(lr=0.0001),\n", " loss=bce_dice_loss,\n", " metrics=[dice_coeff])\n", " \n", " return model" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "# Save predictions of 512 model" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2017-09-06T13:22:39.784295Z", "start_time": "2017-09-06T13:22:38.379307Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "model_512 = get_unet(input_shape=(512,512,3), initial_filters=16)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2017-09-06T13:22:55.553577Z", "start_time": "2017-09-06T13:22:39.789832Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "model_512.load_weights('./weights/best_weights_512_2.hdf5')" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2017-09-06T13:22:55.561627Z", "start_time": "2017-09-06T13:22:55.559581Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "input_size=512" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2017-09-06T13:22:56.021875Z", "start_time": "2017-09-06T13:22:55.566960Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "from os import listdir\n", "ids_list = [x.split('.')[0] for x in listdir('data/train')]" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2017-09-06T13:26:51.547810Z", "start_time": "2017-09-06T13:23:01.352780Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "all_imgs=[]\n", "for id in ids_list:\n", " img = cv2.imread('data/train/{}.jpg'.format(id))\n", " img = cv2.resize(img, (input_size, input_size))\n", " all_imgs.append(img)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2017-09-06T13:33:43.087236Z", "start_time": "2017-09-06T13:27:19.506601Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "preds = model_512.predict(np.array(all_imgs, np.float32)/255, batch_size=16)\n", "preds = np.squeeze(preds, axis=3)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2017-09-06T13:34:18.348603Z", "start_time": "2017-09-06T13:34:18.344023Z" }, "hidden": true }, "outputs": [ { "data": { "text/plain": [ "(5088, 512, 512)" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "preds.shape" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "ExecuteTime": { "end_time": "2017-09-06T13:51:43.878252Z", "start_time": "2017-09-06T13:34:25.313172Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "for i in range(len(ids_list)):\n", " this_id = ids_list[i]\n", " this_pred = np.copy(preds[i,:,:])\n", " f = gzip.open('./data/preds512/' + this_id+ '.pklz','wb')\n", " pickle.dump(this_pred, f)\n", " f.close()" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "# Full res model" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2017-09-06T13:59:23.503502Z", "start_time": "2017-09-06T13:59:22.761703Z" }, "collapsed": true, "hidden": true, "scrolled": true }, "outputs": [], "source": [ "model_fullres = get_unet(input_shape=(1280,1920,4), initial_filters=8, central_filters=64)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2017-09-06T13:59:23.541192Z", "start_time": "2017-09-06T13:59:23.516915Z" }, "hidden": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "____________________________________________________________________________________________________\n", "Layer (type) Output Shape Param # Connected to \n", "====================================================================================================\n", "input_1 (InputLayer) (None, 1280, 1920, 4) 0 \n", "____________________________________________________________________________________________________\n", "conv2d_1 (Conv2D) (None, 1280, 1920, 8) 296 input_1[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_1 (BatchNorm (None, 1280, 1920, 8) 32 conv2d_1[0][0] \n", "____________________________________________________________________________________________________\n", "activation_1 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_1[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_2 (Conv2D) (None, 1280, 1920, 8) 584 activation_1[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_2 (BatchNorm (None, 1280, 1920, 8) 32 conv2d_2[0][0] \n", "____________________________________________________________________________________________________\n", "activation_2 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_2[0][0] \n", "____________________________________________________________________________________________________\n", "max_pooling2d_1 (MaxPooling2D) (None, 640, 960, 8) 0 activation_2[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_3 (Conv2D) (None, 640, 960, 16) 1168 max_pooling2d_1[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_3 (BatchNorm (None, 640, 960, 16) 64 conv2d_3[0][0] \n", "____________________________________________________________________________________________________\n", "activation_3 (Activation) (None, 640, 960, 16) 0 batch_normalization_3[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_4 (Conv2D) (None, 640, 960, 16) 2320 activation_3[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_4 (BatchNorm (None, 640, 960, 16) 64 conv2d_4[0][0] \n", "____________________________________________________________________________________________________\n", "activation_4 (Activation) (None, 640, 960, 16) 0 batch_normalization_4[0][0] \n", "____________________________________________________________________________________________________\n", "max_pooling2d_2 (MaxPooling2D) (None, 320, 480, 16) 0 activation_4[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_5 (Conv2D) (None, 320, 480, 32) 4640 max_pooling2d_2[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_5 (BatchNorm (None, 320, 480, 32) 128 conv2d_5[0][0] \n", "____________________________________________________________________________________________________\n", "activation_5 (Activation) (None, 320, 480, 32) 0 batch_normalization_5[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_6 (Conv2D) (None, 320, 480, 32) 9248 activation_5[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_6 (BatchNorm (None, 320, 480, 32) 128 conv2d_6[0][0] \n", "____________________________________________________________________________________________________\n", "activation_6 (Activation) (None, 320, 480, 32) 0 batch_normalization_6[0][0] \n", "____________________________________________________________________________________________________\n", "max_pooling2d_3 (MaxPooling2D) (None, 160, 240, 32) 0 activation_6[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_7 (Conv2D) (None, 160, 240, 64) 18496 max_pooling2d_3[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_7 (BatchNorm (None, 160, 240, 64) 256 conv2d_7[0][0] \n", "____________________________________________________________________________________________________\n", "activation_7 (Activation) (None, 160, 240, 64) 0 batch_normalization_7[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_8 (Conv2D) (None, 160, 240, 64) 36928 activation_7[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_8 (BatchNorm (None, 160, 240, 64) 256 conv2d_8[0][0] \n", "____________________________________________________________________________________________________\n", "activation_8 (Activation) (None, 160, 240, 64) 0 batch_normalization_8[0][0] \n", "____________________________________________________________________________________________________\n", "up_sampling2d_1 (UpSampling2D) (None, 320, 480, 64) 0 activation_8[0][0] \n", "____________________________________________________________________________________________________\n", "concatenate_1 (Concatenate) (None, 320, 480, 96) 0 activation_6[0][0] \n", " up_sampling2d_1[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_9 (Conv2D) (None, 320, 480, 32) 27680 concatenate_1[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_9 (BatchNorm (None, 320, 480, 32) 128 conv2d_9[0][0] \n", "____________________________________________________________________________________________________\n", "activation_9 (Activation) (None, 320, 480, 32) 0 batch_normalization_9[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_10 (Conv2D) (None, 320, 480, 32) 9248 activation_9[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_10 (BatchNor (None, 320, 480, 32) 128 conv2d_10[0][0] \n", "____________________________________________________________________________________________________\n", "activation_10 (Activation) (None, 320, 480, 32) 0 batch_normalization_10[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_11 (Conv2D) (None, 320, 480, 32) 9248 activation_10[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_11 (BatchNor (None, 320, 480, 32) 128 conv2d_11[0][0] \n", "____________________________________________________________________________________________________\n", "activation_11 (Activation) (None, 320, 480, 32) 0 batch_normalization_11[0][0] \n", "____________________________________________________________________________________________________\n", "up_sampling2d_2 (UpSampling2D) (None, 640, 960, 32) 0 activation_11[0][0] \n", "____________________________________________________________________________________________________\n", "concatenate_2 (Concatenate) (None, 640, 960, 48) 0 activation_4[0][0] \n", " up_sampling2d_2[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_12 (Conv2D) (None, 640, 960, 16) 6928 concatenate_2[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_12 (BatchNor (None, 640, 960, 16) 64 conv2d_12[0][0] \n", "____________________________________________________________________________________________________\n", "activation_12 (Activation) (None, 640, 960, 16) 0 batch_normalization_12[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_13 (Conv2D) (None, 640, 960, 16) 2320 activation_12[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_13 (BatchNor (None, 640, 960, 16) 64 conv2d_13[0][0] \n", "____________________________________________________________________________________________________\n", "activation_13 (Activation) (None, 640, 960, 16) 0 batch_normalization_13[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_14 (Conv2D) (None, 640, 960, 16) 2320 activation_13[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_14 (BatchNor (None, 640, 960, 16) 64 conv2d_14[0][0] \n", "____________________________________________________________________________________________________\n", "activation_14 (Activation) (None, 640, 960, 16) 0 batch_normalization_14[0][0] \n", "____________________________________________________________________________________________________\n", "up_sampling2d_3 (UpSampling2D) (None, 1280, 1920, 16 0 activation_14[0][0] \n", "____________________________________________________________________________________________________\n", "concatenate_3 (Concatenate) (None, 1280, 1920, 24 0 activation_2[0][0] \n", " up_sampling2d_3[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_15 (Conv2D) (None, 1280, 1920, 8) 1736 concatenate_3[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_15 (BatchNor (None, 1280, 1920, 8) 32 conv2d_15[0][0] \n", "____________________________________________________________________________________________________\n", "activation_15 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_15[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_16 (Conv2D) (None, 1280, 1920, 8) 584 activation_15[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_16 (BatchNor (None, 1280, 1920, 8) 32 conv2d_16[0][0] \n", "____________________________________________________________________________________________________\n", "activation_16 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_16[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_17 (Conv2D) (None, 1280, 1920, 8) 584 activation_16[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_17 (BatchNor (None, 1280, 1920, 8) 32 conv2d_17[0][0] \n", "____________________________________________________________________________________________________\n", "activation_17 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_17[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_18 (Conv2D) (None, 1280, 1920, 1) 9 activation_17[0][0] \n", "====================================================================================================\n", "Total params: 135,969\n", "Trainable params: 135,153\n", "Non-trainable params: 816\n", "____________________________________________________________________________________________________\n" ] } ], "source": [ "model_fullres.summary()" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2017-09-06T13:59:24.534242Z", "start_time": "2017-09-06T13:59:23.554377Z" }, "hidden": true }, "outputs": [ { "data": { "text/html": [ "model-fullres-v2.png
" ], "text/plain": [ "/home/sainath/extvol/kaggle-carvana-image-masking-challenge/model-fullres-v2.png" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from keras.utils import plot_model \n", "plot_model(model_fullres, to_file='model-fullres-v2.png', show_shapes=True, show_layer_names=True) \n", "from IPython.display import FileLink\n", "FileLink('model-fullres-v2.png')" ] }, { "cell_type": "markdown", "metadata": { "heading_collapsed": true }, "source": [ "# Training" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:33:06.210115Z", "start_time": "2017-09-07T07:33:06.207688Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "threshold = 0.5" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:33:08.011812Z", "start_time": "2017-09-07T07:33:06.543313Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "df_train = pd.read_csv('data/train_masks.csv')" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:33:08.053843Z", "start_time": "2017-09-07T07:33:08.046673Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "ids_train = df_train['img'].map(lambda s: s.split('.')[0])" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:33:08.111105Z", "start_time": "2017-09-07T07:33:08.090424Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "ids_train_split, ids_valid_split = train_test_split(ids_train, test_size=0.2, random_state=42)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:33:08.157209Z", "start_time": "2017-09-07T07:33:08.146094Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "def randomHueSaturationValue(image, hue_shift_limit=(-180, 180),\n", " sat_shift_limit=(-255, 255),\n", " val_shift_limit=(-255, 255), u=0.5):\n", " if np.random.random() < u:\n", " image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)\n", " h, s, v = cv2.split(image)\n", " hue_shift = np.random.uniform(hue_shift_limit[0], hue_shift_limit[1])\n", " h = cv2.add(h, hue_shift)\n", " sat_shift = np.random.uniform(sat_shift_limit[0], sat_shift_limit[1])\n", " s = cv2.add(s, sat_shift)\n", " val_shift = np.random.uniform(val_shift_limit[0], val_shift_limit[1])\n", " v = cv2.add(v, val_shift)\n", " image = cv2.merge((h, s, v))\n", " image = cv2.cvtColor(image, cv2.COLOR_HSV2BGR)\n", "\n", " return image" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:33:08.465491Z", "start_time": "2017-09-07T07:33:08.438040Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "def randomShiftScaleRotate(image, mask,\n", " shift_limit=(-0.0625, 0.0625),\n", " scale_limit=(-0.1, 0.1),\n", " rotate_limit=(-45, 45), aspect_limit=(0, 0),\n", " borderMode=cv2.BORDER_CONSTANT, u=0.5):\n", " if np.random.random() < u:\n", " height, width, channel = image.shape\n", "\n", " angle = np.random.uniform(rotate_limit[0], rotate_limit[1]) # degree\n", " scale = np.random.uniform(1 + scale_limit[0], 1 + scale_limit[1])\n", " aspect = np.random.uniform(1 + aspect_limit[0], 1 + aspect_limit[1])\n", " sx = scale * aspect / (aspect ** 0.5)\n", " sy = scale / (aspect ** 0.5)\n", " dx = round(np.random.uniform(shift_limit[0], shift_limit[1]) * width)\n", " dy = round(np.random.uniform(shift_limit[0], shift_limit[1]) * height)\n", "\n", " cc = np.math.cos(angle / 180 * np.math.pi) * sx\n", " ss = np.math.sin(angle / 180 * np.math.pi) * sy\n", " rotate_matrix = np.array([[cc, -ss], [ss, cc]])\n", "\n", " box0 = np.array([[0, 0], [width, 0], [width, height], [0, height], ])\n", " box1 = box0 - np.array([width / 2, height / 2])\n", " box1 = np.dot(box1, rotate_matrix.T) + np.array([width / 2 + dx, height / 2 + dy])\n", "\n", " box0 = box0.astype(np.float32)\n", " box1 = box1.astype(np.float32)\n", " mat = cv2.getPerspectiveTransform(box0, box1)\n", " image = cv2.warpPerspective(image, mat, (width, height), flags=cv2.INTER_LINEAR, borderMode=borderMode,\n", " borderValue=(\n", " 0, 0,\n", " 0,))\n", " mask = cv2.warpPerspective(mask, mat, (width, height), flags=cv2.INTER_LINEAR, borderMode=borderMode,\n", " borderValue=(\n", " 0, 0,\n", " 0,))\n", "\n", " return image, mask" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:33:09.021220Z", "start_time": "2017-09-07T07:33:09.017598Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "def randomHorizontalFlip(image, mask, u=0.5):\n", " if np.random.random() < u:\n", " image = cv2.flip(image, 1)\n", " mask = cv2.flip(mask, 1)\n", "\n", " return image, mask" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:33:09.517756Z", "start_time": "2017-09-07T07:33:09.511808Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "def m512_stuff(ids_list):\n", " final_512_preds=[]\n", " for this_id in ids_list:\n", " \n", " f = gzip.open('./data/preds512/' + this_id + '.pklz','rb')\n", " mask = pickle.load(f)\n", " f.close()\n", " \n", " resized_mask = cv2.resize(mask, (1920, 1280))\n", " resized_mask = np.expand_dims(resized_mask, axis=3)*255\n", " final_512_preds.append(resized_mask)\n", " \n", " return final_512_preds" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:33:10.125172Z", "start_time": "2017-09-07T07:33:10.096712Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "def train_generator(train_batch_size):\n", " while True:\n", " this_ids_train_split = random.sample(list(ids_train_split), len(list(ids_train_split)))\n", " for start in range(0, len(ids_train_split), train_batch_size):\n", " x_batch = []\n", " y_batch = []\n", " \n", " start=0\n", " end = min(start + train_batch_size, len(ids_train_split))\n", " ids_train_batch = this_ids_train_split[start:end]\n", " \n", " all_imgs = []\n", " for id in ids_train_batch:\n", " img = cv2.imread('data/train/{}.jpg'.format(id))\n", " all_imgs.append(img)\n", " \n", " final_512_preds = m512_stuff(ids_train_batch)\n", " \n", " for i in range(len(all_imgs)):\n", " img = cv2.resize(all_imgs[i], (1920,1280))\n", " img = randomHueSaturationValue(img,\n", " hue_shift_limit=(-50, 50),\n", " sat_shift_limit=(-5, 5),\n", " val_shift_limit=(-15, 15))\n", " img = np.concatenate([img, final_512_preds[i]], axis=2)\n", " \n", " mask = cv2.imread('data/train_masks/{}_mask.png'.format(ids_train_batch[i]), cv2.IMREAD_GRAYSCALE)\n", " mask = cv2.resize(mask, (1920,1280))\n", " \n", " img, mask = randomShiftScaleRotate(img, mask,\n", " shift_limit=(-0.0625, 0.0625),\n", " scale_limit=(-0.1, 0.1),\n", " rotate_limit=(-0, 0))\n", " img, mask = randomHorizontalFlip(img, mask)\n", " mask = np.expand_dims(mask, axis=2)\n", " x_batch.append(img)\n", " y_batch.append(mask)\n", " \n", " x_batch = np.array(x_batch, np.float32) / 255\n", " y_batch = np.array(y_batch, np.float32) / 255\n", " yield x_batch, y_batch" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:33:10.952169Z", "start_time": "2017-09-07T07:33:10.934413Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "def valid_generator(val_batch_size):\n", " while True:\n", " for start in range(0, len(ids_valid_split), val_batch_size):\n", " \n", " x_batch = []\n", " y_batch = []\n", " \n", " end = min(start + val_batch_size, len(ids_valid_split))\n", " ids_valid_batch = ids_valid_split[start:end]\n", " \n", " all_imgs = []\n", " for id in ids_valid_batch.values:\n", " img = cv2.imread('data/train/{}.jpg'.format(id))\n", " all_imgs.append(img)\n", " \n", " final_512_preds = m512_stuff(ids_valid_batch.values)\n", " \n", " for i in range(len(all_imgs)):\n", " img = cv2.resize(all_imgs[i], (1920,1280))\n", " img = np.concatenate([img, final_512_preds[i]], axis=2)\n", " mask = cv2.imread('data/train_masks/{}_mask.png'.format(ids_valid_batch.values[i]), cv2.IMREAD_GRAYSCALE)\n", " mask = cv2.resize(mask, (1920,1280))\n", " mask = np.expand_dims(mask, axis=2)\n", " x_batch.append(img)\n", " y_batch.append(mask)\n", " \n", " x_batch = np.array(x_batch, np.float32) / 255\n", " y_batch = np.array(y_batch, np.float32) / 255\n", " yield x_batch, y_batch" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:33:15.965706Z", "start_time": "2017-09-07T07:33:15.953898Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "max_epochs = 50" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:33:16.200426Z", "start_time": "2017-09-07T07:33:16.198126Z" }, "collapsed": true, "hidden": true }, "outputs": [], "source": [ "train_batch_size = 3\n", "val_batch_size = 8" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "start_time": "2017-09-06T14:00:22.313Z" }, "collapsed": true, "hidden": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 1/50\n", "4466s - loss: 0.7622 - dice_coeff: 0.5814 - val_loss: 0.6156 - val_dice_coeff: 0.6557\n", "Epoch 2/50\n", "4459s - loss: 0.4551 - dice_coeff: 0.7366 - val_loss: 0.3884 - val_dice_coeff: 0.7606\n", "Epoch 3/50\n", "4459s - loss: 0.2361 - dice_coeff: 0.8573 - val_loss: 0.2059 - val_dice_coeff: 0.8662\n", "Epoch 4/50\n", "4460s - loss: 0.2182 - dice_coeff: 0.8479 - val_loss: 0.1380 - val_dice_coeff: 0.9107\n", "Epoch 5/50\n", "4460s - loss: 0.0832 - dice_coeff: 0.9425 - val_loss: 0.0594 - val_dice_coeff: 0.9633\n", "Epoch 6/50\n", "4460s - loss: 0.0290 - dice_coeff: 0.9808 - val_loss: 0.0294 - val_dice_coeff: 0.9848\n", "Epoch 7/50\n", "4459s - loss: 0.0129 - dice_coeff: 0.9913 - val_loss: 0.0186 - val_dice_coeff: 0.9920\n", "Epoch 8/50\n", "4458s - loss: 0.0083 - dice_coeff: 0.9943 - val_loss: 0.0174 - val_dice_coeff: 0.9937\n", "Epoch 9/50\n", "4460s - loss: 0.0049 - dice_coeff: 0.9973 - val_loss: 0.0141 - val_dice_coeff: 0.9953\n", "Epoch 10/50\n", "4461s - loss: 0.0045 - dice_coeff: 0.9975 - val_loss: 0.0148 - val_dice_coeff: 0.9953\n", "Epoch 11/50\n", "4461s - loss: 0.0031 - dice_coeff: 0.9984 - val_loss: 0.0141 - val_dice_coeff: 0.9957\n", "Epoch 12/50\n", "4462s - loss: 0.0029 - dice_coeff: 0.9985 - val_loss: 0.0152 - val_dice_coeff: 0.9955\n", "Epoch 13/50\n", "4461s - loss: 0.0028 - dice_coeff: 0.9985 - val_loss: 0.0144 - val_dice_coeff: 0.9956\n", "Epoch 14/50\n", "\n", "Epoch 00013: reducing learning rate to 9.999999747378752e-06.\n", "4461s - loss: 0.0032 - dice_coeff: 0.9983 - val_loss: 0.0155 - val_dice_coeff: 0.9954\n", "Epoch 15/50\n" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalid_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_batch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))\n\u001b[0m", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 85\u001b[0m warnings.warn('Update your `' + object_name +\n\u001b[1;32m 86\u001b[0m '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 87\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 88\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 2040\u001b[0m outs = self.train_on_batch(x, y,\n\u001b[1;32m 2041\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2042\u001b[0;31m class_weight=class_weight)\n\u001b[0m\u001b[1;32m 2043\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2044\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m 1760\u001b[0m \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1761\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1762\u001b[0;31m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1763\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1764\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2271\u001b[0m updated = session.run(self.outputs + [self.updates_op],\n\u001b[1;32m 2272\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2273\u001b[0;31m **self.session_kwargs)\n\u001b[0m\u001b[1;32m 2274\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 893\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 894\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 896\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 897\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1122\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1123\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1124\u001b[0;31m feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m 1125\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1126\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1319\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1320\u001b[0m return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1321\u001b[0;31m options, run_metadata)\n\u001b[0m\u001b[1;32m 1322\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1323\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1325\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1326\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1327\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1328\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1329\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m 1304\u001b[0m return tf_session.TF_Run(session, options,\n\u001b[1;32m 1305\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1306\u001b[0;31m status, run_metadata)\n\u001b[0m\u001b[1;32m 1307\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1308\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "callbacks = [EarlyStopping(monitor='val_loss',\n", " patience=8,\n", " verbose=1,\n", " min_delta=1e-4),\n", " ReduceLROnPlateau(monitor='val_loss',\n", " factor=0.1,\n", " patience=4,\n", " verbose=1,\n", " epsilon=1e-4),\n", " ModelCheckpoint(monitor='val_loss',\n", " filepath='weights/best_weights_fullresv2.hdf5',\n", " save_best_only=True,\n", " save_weights_only=True)]\n", "\n", "model_fullres.fit_generator(generator=train_generator(train_batch_size),\n", " steps_per_epoch=np.ceil(float(len(ids_train_split)) / float(train_batch_size)),\n", " epochs=max_epochs,\n", " verbose=2,\n", " callbacks=callbacks,\n", " validation_data=valid_generator(val_batch_size),\n", " validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Full res model 2" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:32:43.966341Z", "start_time": "2017-09-07T07:32:42.342318Z" }, "collapsed": true }, "outputs": [], "source": [ "model_fullres = get_unet(input_shape=(1280,1920,4), initial_filters=8, central_filters=1024)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2017-09-07T07:32:53.560629Z", "start_time": "2017-09-07T07:32:53.524081Z" }, "collapsed": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "____________________________________________________________________________________________________\n", "Layer (type) Output Shape Param # Connected to \n", "====================================================================================================\n", "input_1 (InputLayer) (None, 1280, 1920, 4) 0 \n", "____________________________________________________________________________________________________\n", "conv2d_1 (Conv2D) (None, 1280, 1920, 8) 296 input_1[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_1 (BatchNorm (None, 1280, 1920, 8) 32 conv2d_1[0][0] \n", "____________________________________________________________________________________________________\n", "activation_1 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_1[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_2 (Conv2D) (None, 1280, 1920, 8) 584 activation_1[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_2 (BatchNorm (None, 1280, 1920, 8) 32 conv2d_2[0][0] \n", "____________________________________________________________________________________________________\n", "activation_2 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_2[0][0] \n", "____________________________________________________________________________________________________\n", "max_pooling2d_1 (MaxPooling2D) (None, 640, 960, 8) 0 activation_2[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_3 (Conv2D) (None, 640, 960, 16) 1168 max_pooling2d_1[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_3 (BatchNorm (None, 640, 960, 16) 64 conv2d_3[0][0] \n", "____________________________________________________________________________________________________\n", "activation_3 (Activation) (None, 640, 960, 16) 0 batch_normalization_3[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_4 (Conv2D) (None, 640, 960, 16) 2320 activation_3[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_4 (BatchNorm (None, 640, 960, 16) 64 conv2d_4[0][0] \n", "____________________________________________________________________________________________________\n", "activation_4 (Activation) (None, 640, 960, 16) 0 batch_normalization_4[0][0] \n", "____________________________________________________________________________________________________\n", "max_pooling2d_2 (MaxPooling2D) (None, 320, 480, 16) 0 activation_4[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_5 (Conv2D) (None, 320, 480, 32) 4640 max_pooling2d_2[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_5 (BatchNorm (None, 320, 480, 32) 128 conv2d_5[0][0] \n", "____________________________________________________________________________________________________\n", "activation_5 (Activation) (None, 320, 480, 32) 0 batch_normalization_5[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_6 (Conv2D) (None, 320, 480, 32) 9248 activation_5[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_6 (BatchNorm (None, 320, 480, 32) 128 conv2d_6[0][0] \n", "____________________________________________________________________________________________________\n", "activation_6 (Activation) (None, 320, 480, 32) 0 batch_normalization_6[0][0] \n", "____________________________________________________________________________________________________\n", "max_pooling2d_3 (MaxPooling2D) (None, 160, 240, 32) 0 activation_6[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_7 (Conv2D) (None, 160, 240, 64) 18496 max_pooling2d_3[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_7 (BatchNorm (None, 160, 240, 64) 256 conv2d_7[0][0] \n", "____________________________________________________________________________________________________\n", "activation_7 (Activation) (None, 160, 240, 64) 0 batch_normalization_7[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_8 (Conv2D) (None, 160, 240, 64) 36928 activation_7[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_8 (BatchNorm (None, 160, 240, 64) 256 conv2d_8[0][0] \n", "____________________________________________________________________________________________________\n", "activation_8 (Activation) (None, 160, 240, 64) 0 batch_normalization_8[0][0] \n", "____________________________________________________________________________________________________\n", "max_pooling2d_4 (MaxPooling2D) (None, 80, 120, 64) 0 activation_8[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_9 (Conv2D) (None, 80, 120, 128) 73856 max_pooling2d_4[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_9 (BatchNorm (None, 80, 120, 128) 512 conv2d_9[0][0] \n", "____________________________________________________________________________________________________\n", "activation_9 (Activation) (None, 80, 120, 128) 0 batch_normalization_9[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_10 (Conv2D) (None, 80, 120, 128) 147584 activation_9[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_10 (BatchNor (None, 80, 120, 128) 512 conv2d_10[0][0] \n", "____________________________________________________________________________________________________\n", "activation_10 (Activation) (None, 80, 120, 128) 0 batch_normalization_10[0][0] \n", "____________________________________________________________________________________________________\n", "max_pooling2d_5 (MaxPooling2D) (None, 40, 60, 128) 0 activation_10[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_11 (Conv2D) (None, 40, 60, 256) 295168 max_pooling2d_5[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_11 (BatchNor (None, 40, 60, 256) 1024 conv2d_11[0][0] \n", "____________________________________________________________________________________________________\n", "activation_11 (Activation) (None, 40, 60, 256) 0 batch_normalization_11[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_12 (Conv2D) (None, 40, 60, 256) 590080 activation_11[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_12 (BatchNor (None, 40, 60, 256) 1024 conv2d_12[0][0] \n", "____________________________________________________________________________________________________\n", "activation_12 (Activation) (None, 40, 60, 256) 0 batch_normalization_12[0][0] \n", "____________________________________________________________________________________________________\n", "max_pooling2d_6 (MaxPooling2D) (None, 20, 30, 256) 0 activation_12[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_13 (Conv2D) (None, 20, 30, 512) 1180160 max_pooling2d_6[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_13 (BatchNor (None, 20, 30, 512) 2048 conv2d_13[0][0] \n", "____________________________________________________________________________________________________\n", "activation_13 (Activation) (None, 20, 30, 512) 0 batch_normalization_13[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_14 (Conv2D) (None, 20, 30, 512) 2359808 activation_13[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_14 (BatchNor (None, 20, 30, 512) 2048 conv2d_14[0][0] \n", "____________________________________________________________________________________________________\n", "activation_14 (Activation) (None, 20, 30, 512) 0 batch_normalization_14[0][0] \n", "____________________________________________________________________________________________________\n", "max_pooling2d_7 (MaxPooling2D) (None, 10, 15, 512) 0 activation_14[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_15 (Conv2D) (None, 10, 15, 1024) 4719616 max_pooling2d_7[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_15 (BatchNor (None, 10, 15, 1024) 4096 conv2d_15[0][0] \n", "____________________________________________________________________________________________________\n", "activation_15 (Activation) (None, 10, 15, 1024) 0 batch_normalization_15[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_16 (Conv2D) (None, 10, 15, 1024) 9438208 activation_15[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_16 (BatchNor (None, 10, 15, 1024) 4096 conv2d_16[0][0] \n", "____________________________________________________________________________________________________\n", "activation_16 (Activation) (None, 10, 15, 1024) 0 batch_normalization_16[0][0] \n", "____________________________________________________________________________________________________\n", "up_sampling2d_1 (UpSampling2D) (None, 20, 30, 1024) 0 activation_16[0][0] \n", "____________________________________________________________________________________________________\n", "concatenate_1 (Concatenate) (None, 20, 30, 1536) 0 activation_14[0][0] \n", " up_sampling2d_1[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_17 (Conv2D) (None, 20, 30, 512) 7078400 concatenate_1[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_17 (BatchNor (None, 20, 30, 512) 2048 conv2d_17[0][0] \n", "____________________________________________________________________________________________________\n", "activation_17 (Activation) (None, 20, 30, 512) 0 batch_normalization_17[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_18 (Conv2D) (None, 20, 30, 512) 2359808 activation_17[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_18 (BatchNor (None, 20, 30, 512) 2048 conv2d_18[0][0] \n", "____________________________________________________________________________________________________\n", "activation_18 (Activation) (None, 20, 30, 512) 0 batch_normalization_18[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_19 (Conv2D) (None, 20, 30, 512) 2359808 activation_18[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_19 (BatchNor (None, 20, 30, 512) 2048 conv2d_19[0][0] \n", "____________________________________________________________________________________________________\n", "activation_19 (Activation) (None, 20, 30, 512) 0 batch_normalization_19[0][0] \n", "____________________________________________________________________________________________________\n", "up_sampling2d_2 (UpSampling2D) (None, 40, 60, 512) 0 activation_19[0][0] \n", "____________________________________________________________________________________________________\n", "concatenate_2 (Concatenate) (None, 40, 60, 768) 0 activation_12[0][0] \n", " up_sampling2d_2[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_20 (Conv2D) (None, 40, 60, 256) 1769728 concatenate_2[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_20 (BatchNor (None, 40, 60, 256) 1024 conv2d_20[0][0] \n", "____________________________________________________________________________________________________\n", "activation_20 (Activation) (None, 40, 60, 256) 0 batch_normalization_20[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_21 (Conv2D) (None, 40, 60, 256) 590080 activation_20[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_21 (BatchNor (None, 40, 60, 256) 1024 conv2d_21[0][0] \n", "____________________________________________________________________________________________________\n", "activation_21 (Activation) (None, 40, 60, 256) 0 batch_normalization_21[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_22 (Conv2D) (None, 40, 60, 256) 590080 activation_21[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_22 (BatchNor (None, 40, 60, 256) 1024 conv2d_22[0][0] \n", "____________________________________________________________________________________________________\n", "activation_22 (Activation) (None, 40, 60, 256) 0 batch_normalization_22[0][0] \n", "____________________________________________________________________________________________________\n", "up_sampling2d_3 (UpSampling2D) (None, 80, 120, 256) 0 activation_22[0][0] \n", "____________________________________________________________________________________________________\n", "concatenate_3 (Concatenate) (None, 80, 120, 384) 0 activation_10[0][0] \n", " up_sampling2d_3[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_23 (Conv2D) (None, 80, 120, 128) 442496 concatenate_3[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_23 (BatchNor (None, 80, 120, 128) 512 conv2d_23[0][0] \n", "____________________________________________________________________________________________________\n", "activation_23 (Activation) (None, 80, 120, 128) 0 batch_normalization_23[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_24 (Conv2D) (None, 80, 120, 128) 147584 activation_23[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_24 (BatchNor (None, 80, 120, 128) 512 conv2d_24[0][0] \n", "____________________________________________________________________________________________________\n", "activation_24 (Activation) (None, 80, 120, 128) 0 batch_normalization_24[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_25 (Conv2D) (None, 80, 120, 128) 147584 activation_24[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_25 (BatchNor (None, 80, 120, 128) 512 conv2d_25[0][0] \n", "____________________________________________________________________________________________________\n", "activation_25 (Activation) (None, 80, 120, 128) 0 batch_normalization_25[0][0] \n", "____________________________________________________________________________________________________\n", "up_sampling2d_4 (UpSampling2D) (None, 160, 240, 128) 0 activation_25[0][0] \n", "____________________________________________________________________________________________________\n", "concatenate_4 (Concatenate) (None, 160, 240, 192) 0 activation_8[0][0] \n", " up_sampling2d_4[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_26 (Conv2D) (None, 160, 240, 64) 110656 concatenate_4[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_26 (BatchNor (None, 160, 240, 64) 256 conv2d_26[0][0] \n", "____________________________________________________________________________________________________\n", "activation_26 (Activation) (None, 160, 240, 64) 0 batch_normalization_26[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_27 (Conv2D) (None, 160, 240, 64) 36928 activation_26[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_27 (BatchNor (None, 160, 240, 64) 256 conv2d_27[0][0] \n", "____________________________________________________________________________________________________\n", "activation_27 (Activation) (None, 160, 240, 64) 0 batch_normalization_27[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_28 (Conv2D) (None, 160, 240, 64) 36928 activation_27[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_28 (BatchNor (None, 160, 240, 64) 256 conv2d_28[0][0] \n", "____________________________________________________________________________________________________\n", "activation_28 (Activation) (None, 160, 240, 64) 0 batch_normalization_28[0][0] \n", "____________________________________________________________________________________________________\n", "up_sampling2d_5 (UpSampling2D) (None, 320, 480, 64) 0 activation_28[0][0] \n", "____________________________________________________________________________________________________\n", "concatenate_5 (Concatenate) (None, 320, 480, 96) 0 activation_6[0][0] \n", " up_sampling2d_5[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_29 (Conv2D) (None, 320, 480, 32) 27680 concatenate_5[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_29 (BatchNor (None, 320, 480, 32) 128 conv2d_29[0][0] \n", "____________________________________________________________________________________________________\n", "activation_29 (Activation) (None, 320, 480, 32) 0 batch_normalization_29[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_30 (Conv2D) (None, 320, 480, 32) 9248 activation_29[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_30 (BatchNor (None, 320, 480, 32) 128 conv2d_30[0][0] \n", "____________________________________________________________________________________________________\n", "activation_30 (Activation) (None, 320, 480, 32) 0 batch_normalization_30[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_31 (Conv2D) (None, 320, 480, 32) 9248 activation_30[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_31 (BatchNor (None, 320, 480, 32) 128 conv2d_31[0][0] \n", "____________________________________________________________________________________________________\n", "activation_31 (Activation) (None, 320, 480, 32) 0 batch_normalization_31[0][0] \n", "____________________________________________________________________________________________________\n", "up_sampling2d_6 (UpSampling2D) (None, 640, 960, 32) 0 activation_31[0][0] \n", "____________________________________________________________________________________________________\n", "concatenate_6 (Concatenate) (None, 640, 960, 48) 0 activation_4[0][0] \n", " up_sampling2d_6[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_32 (Conv2D) (None, 640, 960, 16) 6928 concatenate_6[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_32 (BatchNor (None, 640, 960, 16) 64 conv2d_32[0][0] \n", "____________________________________________________________________________________________________\n", "activation_32 (Activation) (None, 640, 960, 16) 0 batch_normalization_32[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_33 (Conv2D) (None, 640, 960, 16) 2320 activation_32[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_33 (BatchNor (None, 640, 960, 16) 64 conv2d_33[0][0] \n", "____________________________________________________________________________________________________\n", "activation_33 (Activation) (None, 640, 960, 16) 0 batch_normalization_33[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_34 (Conv2D) (None, 640, 960, 16) 2320 activation_33[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_34 (BatchNor (None, 640, 960, 16) 64 conv2d_34[0][0] \n", "____________________________________________________________________________________________________\n", "activation_34 (Activation) (None, 640, 960, 16) 0 batch_normalization_34[0][0] \n", "____________________________________________________________________________________________________\n", "up_sampling2d_7 (UpSampling2D) (None, 1280, 1920, 16 0 activation_34[0][0] \n", "____________________________________________________________________________________________________\n", "concatenate_7 (Concatenate) (None, 1280, 1920, 24 0 activation_2[0][0] \n", " up_sampling2d_7[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_35 (Conv2D) (None, 1280, 1920, 8) 1736 concatenate_7[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_35 (BatchNor (None, 1280, 1920, 8) 32 conv2d_35[0][0] \n", "____________________________________________________________________________________________________\n", "activation_35 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_35[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_36 (Conv2D) (None, 1280, 1920, 8) 584 activation_35[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_36 (BatchNor (None, 1280, 1920, 8) 32 conv2d_36[0][0] \n", "____________________________________________________________________________________________________\n", "activation_36 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_36[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_37 (Conv2D) (None, 1280, 1920, 8) 584 activation_36[0][0] \n", "____________________________________________________________________________________________________\n", "batch_normalization_37 (BatchNor (None, 1280, 1920, 8) 32 conv2d_37[0][0] \n", "____________________________________________________________________________________________________\n", "activation_37 (Activation) (None, 1280, 1920, 8) 0 batch_normalization_37[0][0] \n", "____________________________________________________________________________________________________\n", "conv2d_38 (Conv2D) (None, 1280, 1920, 1) 9 activation_37[0][0] \n", "====================================================================================================\n", "Total params: 34,637,409\n", "Trainable params: 34,623,153\n", "Non-trainable params: 14,256\n", "____________________________________________________________________________________________________\n" ] } ], "source": [ "model_fullres.summary()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "start_time": "2017-09-07T07:33:27.015Z" }, "collapsed": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 1/50\n", "5437s - loss: 0.6051 - dice_coeff: 0.6865 - val_loss: 0.5555 - val_dice_coeff: 0.6788\n", "Epoch 2/50\n", "5413s - loss: 0.5022 - dice_coeff: 0.6988 - val_loss: 0.3965 - val_dice_coeff: 0.7570\n", "Epoch 3/50\n", "5411s - loss: 0.1670 - dice_coeff: 0.8868 - val_loss: 0.1190 - val_dice_coeff: 0.9239\n", "Epoch 5/50\n", "5414s - loss: 0.0846 - dice_coeff: 0.9406 - val_loss: 0.0596 - val_dice_coeff: 0.9635\n", "Epoch 6/50\n", "5414s - loss: 0.0291 - dice_coeff: 0.9803 - val_loss: 0.0271 - val_dice_coeff: 0.9856\n", "Epoch 7/50\n", "5415s - loss: 0.0101 - dice_coeff: 0.9936 - val_loss: 0.0176 - val_dice_coeff: 0.9929\n", "Epoch 8/50\n", "5416s - loss: 0.0048 - dice_coeff: 0.9971 - val_loss: 0.0148 - val_dice_coeff: 0.9950\n", "Epoch 9/50\n", "5415s - loss: 0.0036 - dice_coeff: 0.9978 - val_loss: 0.0150 - val_dice_coeff: 0.9954\n", "Epoch 10/50\n", "5416s - loss: 0.0030 - dice_coeff: 0.9986 - val_loss: 0.0139 - val_dice_coeff: 0.9958\n", "Epoch 11/50\n", "5418s - loss: 0.0026 - dice_coeff: 0.9986 - val_loss: 0.0145 - val_dice_coeff: 0.9958\n", "Epoch 12/50\n", "5417s - loss: 0.0023 - dice_coeff: 0.9988 - val_loss: 0.0146 - val_dice_coeff: 0.9958\n", "Epoch 13/50\n", "5417s - loss: 0.0025 - dice_coeff: 0.9986 - val_loss: 0.0148 - val_dice_coeff: 0.9958\n", "Epoch 14/50\n", "5417s - loss: 0.0027 - dice_coeff: 0.9986 - val_loss: 0.0147 - val_dice_coeff: 0.9958\n", "Epoch 15/50\n" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalid_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_batch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))\n\u001b[0m", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 85\u001b[0m warnings.warn('Update your `' + object_name +\n\u001b[1;32m 86\u001b[0m '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 87\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 88\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 2040\u001b[0m outs = self.train_on_batch(x, y,\n\u001b[1;32m 2041\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2042\u001b[0;31m class_weight=class_weight)\n\u001b[0m\u001b[1;32m 2043\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2044\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m 1760\u001b[0m \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1761\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1762\u001b[0;31m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1763\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1764\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2271\u001b[0m updated = session.run(self.outputs + [self.updates_op],\n\u001b[1;32m 2272\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2273\u001b[0;31m **self.session_kwargs)\n\u001b[0m\u001b[1;32m 2274\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 893\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 894\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 896\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 897\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1122\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1123\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1124\u001b[0;31m feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m 1125\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1126\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1319\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1320\u001b[0m return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1321\u001b[0;31m options, run_metadata)\n\u001b[0m\u001b[1;32m 1322\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1323\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1325\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1326\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1327\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1328\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1329\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m 1304\u001b[0m return tf_session.TF_Run(session, options,\n\u001b[1;32m 1305\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1306\u001b[0;31m status, run_metadata)\n\u001b[0m\u001b[1;32m 1307\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1308\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "callbacks = [EarlyStopping(monitor='val_loss',\n", " patience=8,\n", " verbose=1,\n", " min_delta=1e-4),\n", " ReduceLROnPlateau(monitor='val_loss',\n", " factor=0.1,\n", " patience=4,\n", " verbose=1,\n", " epsilon=1e-4),\n", " ModelCheckpoint(monitor='val_loss',\n", " filepath='weights/best_weights_fullresv3.hdf5',\n", " save_best_only=True,\n", " save_weights_only=True)]\n", "\n", "model_fullres.fit_generator(generator=train_generator(train_batch_size),\n", " steps_per_epoch=np.ceil(float(len(ids_train_split)) / float(train_batch_size)),\n", " epochs=max_epochs,\n", " verbose=2,\n", " callbacks=callbacks,\n", " validation_data=valid_generator(val_batch_size),\n", " validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "ExecuteTime": { "end_time": "2017-09-08T04:59:41.874111Z", "start_time": "2017-09-08T04:59:41.687937Z" }, "collapsed": true }, "outputs": [], "source": [ "model_fullres.load_weights('./weights/best_weights_fullresv3.hdf5')" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "ExecuteTime": { "end_time": "2017-09-08T04:59:42.552399Z", "start_time": "2017-09-08T04:59:42.502751Z" } }, "outputs": [], "source": [ "model_fullres.compile(optimizer=RMSprop(lr=0.00001),\n", " loss=bce_dice_loss,\n", " metrics=[dice_coeff])" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "ExecuteTime": { "end_time": "2017-09-08T06:54:43.221837Z", "start_time": "2017-09-08T04:59:43.460981Z" }, "collapsed": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 1/50\n", "5415s - loss: 0.0028 - dice_coeff: 0.9985 - val_loss: 0.0153 - val_dice_coeff: 0.9956\n", "Epoch 2/50\n" ] }, { "ename": "ResourceExhaustedError", "evalue": "OOM when allocating tensor with shape[3,24,1280,1920]\n\t [[Node: training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput = Conv2DBackpropInput[T=DT_FLOAT, _class=[\"loc:@conv2d_35/convolution\"], data_format=\"NHWC\", padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](training_1/RMSprop/gradients/conv2d_35/convolution_grad/Shape, conv2d_35/kernel/read, training_1/RMSprop/gradients/AddN_18)]]\n\nCaused by op 'training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput', defined at:\n File \"/home/sainath/anaconda3/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n \"__main__\", mod_spec)\n File \"/home/sainath/anaconda3/lib/python3.6/runpy.py\", line 85, in _run_code\n exec(code, run_globals)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py\", line 16, in \n app.launch_new_instance()\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n app.start()\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 477, in start\n ioloop.IOLoop.instance().start()\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n super(ZMQIOLoop, self).start()\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py\", line 888, in start\n handler_func(fd_obj, events)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n return fn(*args, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n self._handle_recv()\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n self._run_callback(callback, msg)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n callback(*args, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n return fn(*args, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n return self.dispatch_shell(stream, msg)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 235, in dispatch_shell\n handler(stream, idents, msg)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n user_expressions, allow_stdin)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n res = shell.run_cell(code, store_history=store_history, silent=silent)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 533, in run_cell\n return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2827, in run_ast_nodes\n if self.run_code(code, result):\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"\", line 21, in \n validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\", line 87, in wrapper\n return func(*args, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\", line 1926, in fit_generator\n self._make_train_function()\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\", line 960, in _make_train_function\n loss=self.total_loss)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\", line 87, in wrapper\n return func(*args, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/optimizers.py\", line 225, in get_updates\n grads = self.get_gradients(loss, params)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/optimizers.py\", line 73, in get_gradients\n grads = K.gradients(loss, params)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\", line 2310, in gradients\n return tf.gradients(loss, variables, colocate_gradients_with_ops=True)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py\", line 542, in gradients\n grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py\", line 348, in _MaybeCompile\n return grad_fn() # Exit early\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py\", line 542, in \n grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_grad.py\", line 446, in _Conv2DGrad\n op.get_attr(\"data_format\")),\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py\", line 486, in conv2d_backprop_input\n data_format=data_format, name=name)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 767, in apply_op\n op_def=op_def)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2630, in create_op\n original_op=self._default_original_op, op_def=op_def)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 1204, in __init__\n self._traceback = self._graph._extract_stack() # pylint: disable=protected-access\n\n...which was originally created as op 'conv2d_35/convolution', defined at:\n File \"/home/sainath/anaconda3/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n \"__main__\", mod_spec)\n[elided 16 identical lines from previous traceback]\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n if self.run_code(code, result):\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"\", line 1, in \n model_fullres = get_unet(input_shape=(1280,1920,4), initial_filters=8, central_filters=1024)\n File \"\", line 23, in get_unet\n x = unet_up_one_block(out_list[-1], down_interim_list.pop(), num_filters)\n File \"\", line 4, in unet_up_one_block\n x = Conv2D(num_filters, (3,3), padding='same')(x)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/topology.py\", line 602, in __call__\n output = self.call(inputs, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/layers/convolutional.py\", line 164, in call\n dilation_rate=self.dilation_rate)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\", line 3164, in conv2d\n data_format='NHWC')\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 672, in convolution\n op=op)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 338, in with_space_to_batch\n return op(input, num_spatial_dims, padding)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 664, in op\n name=name)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 131, in _non_atrous_convolution\n name=name)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py\", line 397, in conv2d\n data_format=data_format, name=name)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 767, in apply_op\n op_def=op_def)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2630, in create_op\n original_op=self._default_original_op, op_def=op_def)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 1204, in __init__\n self._traceback = self._graph._extract_stack() # pylint: disable=protected-access\n\nResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[3,24,1280,1920]\n\t [[Node: training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput = Conv2DBackpropInput[T=DT_FLOAT, _class=[\"loc:@conv2d_35/convolution\"], data_format=\"NHWC\", padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](training_1/RMSprop/gradients/conv2d_35/convolution_grad/Shape, conv2d_35/kernel/read, training_1/RMSprop/gradients/AddN_18)]]\n", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mResourceExhaustedError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1326\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1327\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1328\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m 1305\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1306\u001b[0;31m status, run_metadata)\n\u001b[0m\u001b[1;32m 1307\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/contextlib.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, type, value, traceback)\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 90\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py\u001b[0m in \u001b[0;36mraise_exception_on_not_ok_status\u001b[0;34m()\u001b[0m\n\u001b[1;32m 465\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpywrap_tensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 466\u001b[0;31m pywrap_tensorflow.TF_GetCode(status))\n\u001b[0m\u001b[1;32m 467\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[3,24,1280,1920]\n\t [[Node: training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput = Conv2DBackpropInput[T=DT_FLOAT, _class=[\"loc:@conv2d_35/convolution\"], data_format=\"NHWC\", padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](training_1/RMSprop/gradients/conv2d_35/convolution_grad/Shape, conv2d_35/kernel/read, training_1/RMSprop/gradients/AddN_18)]]", "\nDuring handling of the above exception, another exception occurred:\n", "\u001b[0;31mResourceExhaustedError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalid_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_batch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))\n\u001b[0m", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 85\u001b[0m warnings.warn('Update your `' + object_name +\n\u001b[1;32m 86\u001b[0m '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 87\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 88\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 2040\u001b[0m outs = self.train_on_batch(x, y,\n\u001b[1;32m 2041\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2042\u001b[0;31m class_weight=class_weight)\n\u001b[0m\u001b[1;32m 2043\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2044\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m 1760\u001b[0m \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1761\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1762\u001b[0;31m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1763\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1764\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2271\u001b[0m updated = session.run(self.outputs + [self.updates_op],\n\u001b[1;32m 2272\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2273\u001b[0;31m **self.session_kwargs)\n\u001b[0m\u001b[1;32m 2274\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 893\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 894\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 896\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 897\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1122\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1123\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1124\u001b[0;31m feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m 1125\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1126\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1319\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1320\u001b[0m return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1321\u001b[0;31m options, run_metadata)\n\u001b[0m\u001b[1;32m 1322\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1323\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1338\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1339\u001b[0m \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1340\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1342\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[3,24,1280,1920]\n\t [[Node: training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput = Conv2DBackpropInput[T=DT_FLOAT, _class=[\"loc:@conv2d_35/convolution\"], data_format=\"NHWC\", padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](training_1/RMSprop/gradients/conv2d_35/convolution_grad/Shape, conv2d_35/kernel/read, training_1/RMSprop/gradients/AddN_18)]]\n\nCaused by op 'training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput', defined at:\n File \"/home/sainath/anaconda3/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n \"__main__\", mod_spec)\n File \"/home/sainath/anaconda3/lib/python3.6/runpy.py\", line 85, in _run_code\n exec(code, run_globals)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py\", line 16, in \n app.launch_new_instance()\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n app.start()\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 477, in start\n ioloop.IOLoop.instance().start()\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n super(ZMQIOLoop, self).start()\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py\", line 888, in start\n handler_func(fd_obj, events)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n return fn(*args, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n self._handle_recv()\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n self._run_callback(callback, msg)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n callback(*args, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n return fn(*args, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n return self.dispatch_shell(stream, msg)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 235, in dispatch_shell\n handler(stream, idents, msg)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n user_expressions, allow_stdin)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n res = shell.run_cell(code, store_history=store_history, silent=silent)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 533, in run_cell\n return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2827, in run_ast_nodes\n if self.run_code(code, result):\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"\", line 21, in \n validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\", line 87, in wrapper\n return func(*args, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\", line 1926, in fit_generator\n self._make_train_function()\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\", line 960, in _make_train_function\n loss=self.total_loss)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\", line 87, in wrapper\n return func(*args, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/optimizers.py\", line 225, in get_updates\n grads = self.get_gradients(loss, params)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/optimizers.py\", line 73, in get_gradients\n grads = K.gradients(loss, params)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\", line 2310, in gradients\n return tf.gradients(loss, variables, colocate_gradients_with_ops=True)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py\", line 542, in gradients\n grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py\", line 348, in _MaybeCompile\n return grad_fn() # Exit early\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py\", line 542, in \n grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_grad.py\", line 446, in _Conv2DGrad\n op.get_attr(\"data_format\")),\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py\", line 486, in conv2d_backprop_input\n data_format=data_format, name=name)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 767, in apply_op\n op_def=op_def)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2630, in create_op\n original_op=self._default_original_op, op_def=op_def)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 1204, in __init__\n self._traceback = self._graph._extract_stack() # pylint: disable=protected-access\n\n...which was originally created as op 'conv2d_35/convolution', defined at:\n File \"/home/sainath/anaconda3/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n \"__main__\", mod_spec)\n[elided 16 identical lines from previous traceback]\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n if self.run_code(code, result):\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"\", line 1, in \n model_fullres = get_unet(input_shape=(1280,1920,4), initial_filters=8, central_filters=1024)\n File \"\", line 23, in get_unet\n x = unet_up_one_block(out_list[-1], down_interim_list.pop(), num_filters)\n File \"\", line 4, in unet_up_one_block\n x = Conv2D(num_filters, (3,3), padding='same')(x)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/engine/topology.py\", line 602, in __call__\n output = self.call(inputs, **kwargs)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/layers/convolutional.py\", line 164, in call\n dilation_rate=self.dilation_rate)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\", line 3164, in conv2d\n data_format='NHWC')\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 672, in convolution\n op=op)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 338, in with_space_to_batch\n return op(input, num_spatial_dims, padding)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 664, in op\n name=name)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 131, in _non_atrous_convolution\n name=name)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py\", line 397, in conv2d\n data_format=data_format, name=name)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 767, in apply_op\n op_def=op_def)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2630, in create_op\n original_op=self._default_original_op, op_def=op_def)\n File \"/home/sainath/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 1204, in __init__\n self._traceback = self._graph._extract_stack() # pylint: disable=protected-access\n\nResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[3,24,1280,1920]\n\t [[Node: training_1/RMSprop/gradients/conv2d_35/convolution_grad/Conv2DBackpropInput = Conv2DBackpropInput[T=DT_FLOAT, _class=[\"loc:@conv2d_35/convolution\"], data_format=\"NHWC\", padding=\"SAME\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/gpu:0\"](training_1/RMSprop/gradients/conv2d_35/convolution_grad/Shape, conv2d_35/kernel/read, training_1/RMSprop/gradients/AddN_18)]]\n" ] } ], "source": [ "callbacks = [EarlyStopping(monitor='val_loss',\n", " patience=4,\n", " verbose=1,\n", " min_delta=1e-4),\n", " ReduceLROnPlateau(monitor='val_loss',\n", " factor=0.1,\n", " patience=3,\n", " verbose=1,\n", " epsilon=1e-4),\n", " ModelCheckpoint(monitor='val_loss',\n", " filepath='weights/best_weights_fullresv3.hdf5',\n", " save_best_only=True,\n", " save_weights_only=True)]\n", "\n", "model_fullres.fit_generator(generator=train_generator(train_batch_size),\n", " steps_per_epoch=np.ceil(float(len(ids_train_split)) / float(train_batch_size)),\n", " epochs=max_epochs,\n", " verbose=2,\n", " callbacks=callbacks,\n", " validation_data=valid_generator(val_batch_size),\n", " validation_steps=np.ceil(float(len(ids_valid_split)) / float(val_batch_size)))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "anaconda-cloud": {}, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.1" }, "toc": { "nav_menu": { "height": "48px", "width": "252px" }, "number_sections": true, "sideBar": true, "skip_h1_title": false, "toc_cell": true, "toc_position": { "height": "857px", "left": "0px", "right": "1706px", "top": "106px", "width": "212px" }, "toc_section_display": "block", "toc_window_display": true }, "varInspector": { "cols": { "lenName": 16, "lenType": 16, "lenVar": 40 }, "kernels_config": { "python": { "delete_cmd_postfix": "", "delete_cmd_prefix": "del ", "library": "var_list.py", "varRefreshCmd": "print(var_dic_list())" }, "r": { "delete_cmd_postfix": ") ", "delete_cmd_prefix": "rm(", "library": "var_list.r", "varRefreshCmd": "cat(var_dic_list()) " } }, "types_to_exclude": [ "module", "function", "builtin_function_or_method", "instance", "_Feature" ], "window_display": false } }, "nbformat": 4, "nbformat_minor": 2 }