{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2.0.0-alpha0\n" ] } ], "source": [ "# boilerplate code\n", "import tensorflow as tf\n", "print(tf.__version__)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "from tensorflow.keras.models import Model\n", "from tensorflow.keras.layers import Conv2D, Layer, MaxPool2D, \\\n", " Dropout, Dense, Input, concatenate, \\\n", " GlobalAveragePooling2D, AveragePooling2D,\\\n", " Flatten, BatchNormalization\n", "\n", "import cv2 #python -m pip install opencv-python\n", "import numpy as np\n", "from tensorflow.keras.datasets import cifar10\n", "from tensorflow.keras.utils import to_categorical\n", "from tensorflow import keras\n", "\n", "import math\n", "from tensorflow.keras.optimizers import SGD\n", "from tensorflow.keras.callbacks import LearningRateScheduler" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "num_classes = 10" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "def load_cifar10_data(img_rows, img_cols):\n", " # Load cifar10 training and test sets\n", " (X_train, Y_train), (X_test, Y_test) = cifar10.load_data()\n", "\n", " # Resize training images\n", " X_train = np.array([cv2.resize(img, (img_rows, img_cols)) for img in X_train[:, :, :, :]])\n", " X_test = np.array([cv2.resize(img, (img_rows, img_cols)) for img in X_test[:, :, :, :]])\n", "\n", " X_train = X_train.astype('float16') / 255.0\n", " X_test = X_test.astype('float16') / 255.0\n", "\n", " # Transform targets to keras compatible format\n", " Y_train = to_categorical(Y_train, num_classes)\n", " Y_test = to_categorical(Y_test, num_classes)\n", "\n", " print(\"X_train: {0}\".format(X_train.shape))\n", " print(\"Y_train: {0}\".format(Y_train.shape))\n", " print(\"X_test: {0}\".format(X_test.shape))\n", " print(\"Y_test: {0}\".format(Y_test.shape))\n", "\n", " return X_train, Y_train, X_test, Y_test" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "X_train: (50000, 224, 224, 3)\n", "Y_train: (50000, 10)\n", "X_test: (10000, 224, 224, 3)\n", "Y_test: (10000, 10)\n" ] } ], "source": [ "X_train, y_train, X_test, y_test = load_cifar10_data(224, 224)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "kernel_init = tf.keras.initializers.glorot_uniform()\n", "bias_init = tf.keras.initializers.Constant(value=0.2)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "from functools import partial\n", "\n", "DefaultConv2D = partial(\n", " Conv2D, \n", " kernel_size=(3, 3), \n", " strides=(1, 1),\n", " padding=\"SAME\",\n", " kernel_initializer=kernel_init,\n", " bias_initializer=bias_init\n", ")" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "class ResidualUnit(Layer):\n", " def __init__(self, filters, strides=1, activation=\"relu\", **kwargs):\n", " super().__init__(**kwargs)\n", " self.activation = keras.activations.get(activation)\n", " self.main_layers = [\n", " DefaultConv2D(filters, strides=strides),\n", " keras.layers.BatchNormalization(),\n", " self.activation,\n", " DefaultConv2D(filters),\n", " keras.layers.BatchNormalization()\n", " ]\n", "\n", " if strides > 1:\n", " self.skip_layers = [\n", " DefaultConv2D(filters, kernel_size=1, strides=strides),\n", " keras.layers.BatchNormalization()\n", " ]\n", " else:\n", " self.skip_layers = []\n", "\n", " def call(self, inputs):\n", " Z = inputs\n", "\n", " for layer in self.main_layers:\n", " Z = layer(Z)\n", " \n", " skip_Z = inputs \n", " for layer in self.skip_layers:\n", " skip_Z = layer(skip_Z)\n", " \n", " return self.activation(Z + skip_Z)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Model: \"resnet_34\"\n", "_________________________________________________________________\n", "Layer (type) Output Shape Param # \n", "=================================================================\n", "conv2d (Conv2D) (None, 112, 112, 64) 9472 \n", "_________________________________________________________________\n", "batch_normalization_v2 (Batc (None, 112, 112, 64) 256 \n", "_________________________________________________________________\n", "activation (Activation) (None, 112, 112, 64) 0 \n", "_________________________________________________________________\n", "max_pooling2d (MaxPooling2D) (None, 56, 56, 64) 0 \n", "_________________________________________________________________\n", "residual_unit (ResidualUnit) (None, 56, 56, 64) 74368 \n", "_________________________________________________________________\n", "residual_unit_1 (ResidualUni (None, 56, 56, 64) 74368 \n", "_________________________________________________________________\n", "residual_unit_2 (ResidualUni (None, 56, 56, 64) 74368 \n", "_________________________________________________________________\n", "residual_unit_3 (ResidualUni (None, 28, 28, 128) 231296 \n", "_________________________________________________________________\n", "residual_unit_4 (ResidualUni (None, 28, 28, 128) 296192 \n", "_________________________________________________________________\n", "residual_unit_5 (ResidualUni (None, 28, 28, 128) 296192 \n", "_________________________________________________________________\n", "residual_unit_6 (ResidualUni (None, 28, 28, 128) 296192 \n", "_________________________________________________________________\n", "residual_unit_7 (ResidualUni (None, 14, 14, 256) 921344 \n", "_________________________________________________________________\n", "residual_unit_8 (ResidualUni (None, 14, 14, 256) 1182208 \n", "_________________________________________________________________\n", "residual_unit_9 (ResidualUni (None, 14, 14, 256) 1182208 \n", "_________________________________________________________________\n", "residual_unit_10 (ResidualUn (None, 14, 14, 256) 1182208 \n", "_________________________________________________________________\n", "residual_unit_11 (ResidualUn (None, 14, 14, 256) 1182208 \n", "_________________________________________________________________\n", "residual_unit_12 (ResidualUn (None, 14, 14, 256) 1182208 \n", "_________________________________________________________________\n", "residual_unit_13 (ResidualUn (None, 7, 7, 512) 3677696 \n", "_________________________________________________________________\n", "residual_unit_14 (ResidualUn (None, 7, 7, 512) 4723712 \n", "_________________________________________________________________\n", "residual_unit_15 (ResidualUn (None, 7, 7, 512) 4723712 \n", "_________________________________________________________________\n", "global_average_pooling2d (Gl (None, 512) 0 \n", "_________________________________________________________________\n", "flatten (Flatten) (None, 512) 0 \n", "_________________________________________________________________\n", "dense (Dense) (None, 10) 5130 \n", "=================================================================\n", "Total params: 21,315,338\n", "Trainable params: 21,298,314\n", "Non-trainable params: 17,024\n", "_________________________________________________________________\n" ] } ], "source": [ "model = keras.models.Sequential(name='resnet_34')\n", "\n", "model.add(DefaultConv2D(\n", " 64, kernel_size=7, strides=2, input_shape=[224, 224, 3]\n", "))\n", "\n", "model.add(keras.layers.BatchNormalization())\n", "model.add(keras.layers.Activation(\"relu\"))\n", "model.add(keras.layers.MaxPool2D(pool_size=3, strides=2, padding=\"SAME\"))\n", "\n", "prev_filters = 64\n", "for filters in [64] * 3 + [128] * 4 + [256] * 6 + [512] * 3:\n", " strides = 1 if filters == prev_filters else 2\n", " model.add(ResidualUnit(filters, strides=strides))\n", " prev_filters = filters\n", " \n", "model.add(keras.layers.GlobalAvgPool2D())\n", "model.add(keras.layers.Flatten())\n", "model.add(keras.layers.Dense(10, activation=\"softmax\"))\n", "model.summary()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Train on 50000 samples, validate on 10000 samples\n", "\n", "Epoch 00001: LearningRateScheduler reducing learning rate to 0.01.\n", "Epoch 1/35\n", "50000/50000 [==============================] - 205s 4ms/sample - loss: 1.4258 - accuracy: 0.4903 - val_loss: 1.9112 - val_accuracy: 0.3711\n", "\n", "Epoch 00002: LearningRateScheduler reducing learning rate to 0.01.\n", "Epoch 2/35\n", "50000/50000 [==============================] - 193s 4ms/sample - loss: 0.8504 - accuracy: 0.6988 - val_loss: 1.1408 - val_accuracy: 0.6201\n", "\n", "Epoch 00003: LearningRateScheduler reducing learning rate to 0.01.\n", "Epoch 3/35\n", "50000/50000 [==============================] - 192s 4ms/sample - loss: 0.5902 - accuracy: 0.7951 - val_loss: 1.1549 - val_accuracy: 0.6433\n", "\n", "Epoch 00004: LearningRateScheduler reducing learning rate to 0.01.\n", "Epoch 4/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 0.4049 - accuracy: 0.8586 - val_loss: 1.1226 - val_accuracy: 0.6340\n", "\n", "Epoch 00005: LearningRateScheduler reducing learning rate to 0.01.\n", "Epoch 5/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 0.2547 - accuracy: 0.9108 - val_loss: 1.6556 - val_accuracy: 0.6001\n", "\n", "Epoch 00006: LearningRateScheduler reducing learning rate to 0.01.\n", "Epoch 6/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 0.1696 - accuracy: 0.9421 - val_loss: 1.8149 - val_accuracy: 0.6156\n", "\n", "Epoch 00007: LearningRateScheduler reducing learning rate to 0.01.\n", "Epoch 7/35\n", "50000/50000 [==============================] - 189s 4ms/sample - loss: 0.1107 - accuracy: 0.9640 - val_loss: 1.5841 - val_accuracy: 0.6699\n", "\n", "Epoch 00008: LearningRateScheduler reducing learning rate to 0.0096.\n", "Epoch 8/35\n", "50000/50000 [==============================] - 189s 4ms/sample - loss: 0.0691 - accuracy: 0.9771 - val_loss: 2.2737 - val_accuracy: 0.6329\n", "\n", "Epoch 00009: LearningRateScheduler reducing learning rate to 0.0096.\n", "Epoch 9/35\n", "50000/50000 [==============================] - 189s 4ms/sample - loss: 0.0437 - accuracy: 0.9857 - val_loss: 2.2252 - val_accuracy: 0.6192\n", "\n", "Epoch 00010: LearningRateScheduler reducing learning rate to 0.0096.\n", "Epoch 10/35\n", "50000/50000 [==============================] - 189s 4ms/sample - loss: 0.0300 - accuracy: 0.9906 - val_loss: 1.1013 - val_accuracy: 0.7675\n", "\n", "Epoch 00011: LearningRateScheduler reducing learning rate to 0.0096.\n", "Epoch 11/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 0.0163 - accuracy: 0.9953 - val_loss: 1.4369 - val_accuracy: 0.7379\n", "\n", "Epoch 00012: LearningRateScheduler reducing learning rate to 0.0096.\n", "Epoch 12/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 0.0141 - accuracy: 0.9957 - val_loss: 1.4038 - val_accuracy: 0.7509\n", "\n", "Epoch 00013: LearningRateScheduler reducing learning rate to 0.0096.\n", "Epoch 13/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 0.0129 - accuracy: 0.9964 - val_loss: 1.4215 - val_accuracy: 0.7438\n", "\n", "Epoch 00014: LearningRateScheduler reducing learning rate to 0.0096.\n", "Epoch 14/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 0.0067 - accuracy: 0.9983 - val_loss: 1.2960 - val_accuracy: 0.7695\n", "\n", "Epoch 00015: LearningRateScheduler reducing learning rate to 0.0096.\n", "Epoch 15/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 0.0014 - accuracy: 0.9999 - val_loss: 0.9393 - val_accuracy: 0.8182\n", "\n", "Epoch 00016: LearningRateScheduler reducing learning rate to 0.009216.\n", "Epoch 16/35\n", "50000/50000 [==============================] - 191s 4ms/sample - loss: 3.6020e-04 - accuracy: 1.0000 - val_loss: 0.9105 - val_accuracy: 0.8231\n", "\n", "Epoch 00017: LearningRateScheduler reducing learning rate to 0.009216.\n", "Epoch 17/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 2.1701e-04 - accuracy: 1.0000 - val_loss: 0.9101 - val_accuracy: 0.8233\n", "\n", "Epoch 00018: LearningRateScheduler reducing learning rate to 0.009216.\n", "Epoch 18/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 1.7623e-04 - accuracy: 1.0000 - val_loss: 0.9136 - val_accuracy: 0.8238\n", "\n", "Epoch 00019: LearningRateScheduler reducing learning rate to 0.009216.\n", "Epoch 19/35\n", "50000/50000 [==============================] - 191s 4ms/sample - loss: 1.5298e-04 - accuracy: 1.0000 - val_loss: 0.9170 - val_accuracy: 0.8248\n", "\n", "Epoch 00020: LearningRateScheduler reducing learning rate to 0.009216.\n", "Epoch 20/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 1.3303e-04 - accuracy: 1.0000 - val_loss: 0.9178 - val_accuracy: 0.8265\n", "\n", "Epoch 00021: LearningRateScheduler reducing learning rate to 0.009216.\n", "Epoch 21/35\n", "50000/50000 [==============================] - 189s 4ms/sample - loss: 1.2130e-04 - accuracy: 1.0000 - val_loss: 0.9188 - val_accuracy: 0.8253\n", "\n", "Epoch 00022: LearningRateScheduler reducing learning rate to 0.009216.\n", "Epoch 22/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 1.0718e-04 - accuracy: 1.0000 - val_loss: 0.9209 - val_accuracy: 0.8254\n", "\n", "Epoch 00023: LearningRateScheduler reducing learning rate to 0.009216.\n", "Epoch 23/35\n", "50000/50000 [==============================] - 189s 4ms/sample - loss: 9.9417e-05 - accuracy: 1.0000 - val_loss: 0.9238 - val_accuracy: 0.8266\n", "\n", "Epoch 00024: LearningRateScheduler reducing learning rate to 0.008847359999999999.\n", "Epoch 24/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 9.1316e-05 - accuracy: 1.0000 - val_loss: 0.9236 - val_accuracy: 0.8266\n", "\n", "Epoch 00025: LearningRateScheduler reducing learning rate to 0.008847359999999999.\n", "Epoch 25/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 8.8492e-05 - accuracy: 1.0000 - val_loss: 0.9260 - val_accuracy: 0.8270\n", "\n", "Epoch 00026: LearningRateScheduler reducing learning rate to 0.008847359999999999.\n", "Epoch 26/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 8.4004e-05 - accuracy: 1.0000 - val_loss: 0.9270 - val_accuracy: 0.8264\n", "\n", "Epoch 00027: LearningRateScheduler reducing learning rate to 0.008847359999999999.\n", "Epoch 27/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 7.6821e-05 - accuracy: 1.0000 - val_loss: 0.9286 - val_accuracy: 0.8264\n", "\n", "Epoch 00028: LearningRateScheduler reducing learning rate to 0.008847359999999999.\n", "Epoch 28/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 7.1725e-05 - accuracy: 1.0000 - val_loss: 0.9297 - val_accuracy: 0.8265\n", "\n", "Epoch 00029: LearningRateScheduler reducing learning rate to 0.008847359999999999.\n", "Epoch 29/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 6.9910e-05 - accuracy: 1.0000 - val_loss: 0.9298 - val_accuracy: 0.8270\n", "\n", "Epoch 00030: LearningRateScheduler reducing learning rate to 0.008847359999999999.\n", "Epoch 30/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 6.7411e-05 - accuracy: 1.0000 - val_loss: 0.9316 - val_accuracy: 0.8272\n", "\n", "Epoch 00031: LearningRateScheduler reducing learning rate to 0.008847359999999999.\n", "Epoch 31/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 6.2929e-05 - accuracy: 1.0000 - val_loss: 0.9318 - val_accuracy: 0.8275\n", "\n", "Epoch 00032: LearningRateScheduler reducing learning rate to 0.008493465599999998.\n", "Epoch 32/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 5.9789e-05 - accuracy: 1.0000 - val_loss: 0.9338 - val_accuracy: 0.8272\n", "\n", "Epoch 00033: LearningRateScheduler reducing learning rate to 0.008493465599999998.\n", "Epoch 33/35\n", "50000/50000 [==============================] - 189s 4ms/sample - loss: 5.8887e-05 - accuracy: 1.0000 - val_loss: 0.9339 - val_accuracy: 0.8267\n", "\n", "Epoch 00034: LearningRateScheduler reducing learning rate to 0.008493465599999998.\n", "Epoch 34/35\n", "50000/50000 [==============================] - 189s 4ms/sample - loss: 5.7854e-05 - accuracy: 1.0000 - val_loss: 0.9351 - val_accuracy: 0.8264\n", "\n", "Epoch 00035: LearningRateScheduler reducing learning rate to 0.008493465599999998.\n", "Epoch 35/35\n", "50000/50000 [==============================] - 190s 4ms/sample - loss: 5.6941e-05 - accuracy: 1.0000 - val_loss: 0.9361 - val_accuracy: 0.8266\n" ] } ], "source": [ "initial_lrate = 0.01\n", "\n", "def decay(epoch, steps=100):\n", " drop = 0.96\n", " epochs_drop = 8\n", " lrate = initial_lrate * math.pow(drop, math.floor((1 + epoch) / epochs_drop))\n", " return lrate\n", "\n", "lr_sc = LearningRateScheduler(decay, verbose=1)\n", "\n", "sgd = SGD(lr=initial_lrate, momentum=0.9, nesterov=True)\n", "\n", "model.compile(\n", " loss='categorical_crossentropy',\n", " optimizer=sgd,\n", " metrics=['accuracy']\n", ")\n", "\n", "epochs = 35\n", "\n", "history = model.fit(\n", " x=X_train,\n", " y=y_train,\n", " validation_data=(X_test, y_test),\n", " epochs=epochs, batch_size=256, callbacks=[lr_sc]\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 2 }