{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "mnist_with_distributed_training.ipynb", "provenance": [], "collapsed_sections": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "tYXhqJSiIlC3" }, "source": [ "# Distributed Training for MNIST" ] }, { "cell_type": "code", "metadata": { "id": "UN5aC7CYBnGR", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "f02e9260-7bb8-4e86-89ab-d3836d3465f2" }, "source": [ "!pip3 install cloudmesh-installer\n", "!pip3 install cloudmesh-common" ], "execution_count": 1, "outputs": [ { "output_type": "stream", "text": [ "Collecting cloudmesh-installer\n", " Downloading https://files.pythonhosted.org/packages/fa/e8/77c2ea195c97469133aca9931f7600f3622a75274ad53f71a0bb681761d4/cloudmesh_installer-4.4.24-py2.py3-none-any.whl\n", "Requirement already satisfied: pip-tools in /usr/local/lib/python3.7/dist-packages (from cloudmesh-installer) (4.5.1)\n", "Requirement already satisfied: wheel in /usr/local/lib/python3.7/dist-packages (from cloudmesh-installer) (0.36.2)\n", "Collecting python-hostlist\n", " Downloading https://files.pythonhosted.org/packages/2b/4f/f31dd4b4bf1a57a5c29599e1165d0df70dbdddcfa59a7c1d04ee2ff4ccbd/python-hostlist-1.21.tar.gz\n", "Requirement already satisfied: humanize in /usr/local/lib/python3.7/dist-packages (from cloudmesh-installer) (0.5.1)\n", "Collecting ordered-set\n", " Downloading https://files.pythonhosted.org/packages/f5/ab/8252360bfe965bba31ec05112b3067bd129ce4800d89e0b85613bc6044f6/ordered-set-4.0.2.tar.gz\n", "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from cloudmesh-installer) (2.23.0)\n", "Collecting oyaml\n", " Downloading https://files.pythonhosted.org/packages/37/aa/111610d8bf5b1bb7a295a048fc648cec346347a8b0be5881defd2d1b4a52/oyaml-1.0-py2.py3-none-any.whl\n", "Requirement already satisfied: coverage in /usr/local/lib/python3.7/dist-packages (from cloudmesh-installer) (3.7.1)\n", "Collecting flake8\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/d4/ca/3971802ee6251da1abead1a22831d7f4743781e2f743bd266bdd2f46c19b/flake8-3.8.4-py2.py3-none-any.whl (72kB)\n", "\u001b[K |████████████████████████████████| 81kB 5.6MB/s \n", "\u001b[?25hCollecting bump2version==1.0.0\n", " Downloading https://files.pythonhosted.org/packages/61/10/560509d9bfe8300e03d268dadec74fac7ae04a430f62e2d06d11d9e4e704/bump2version-1.0.0-py2.py3-none-any.whl\n", "Collecting pytest-cov\n", " Downloading https://files.pythonhosted.org/packages/e3/1a/6affecd2344efee7f2487fac82242474cbac09f9e04929da5944907baf11/pytest_cov-2.11.1-py2.py3-none-any.whl\n", "Collecting pipdeptree\n", " Downloading https://files.pythonhosted.org/packages/fa/22/8f1350b55e4297670813815142425b58829036197f0b4a0fc8f543928717/pipdeptree-2.0.0-py3-none-any.whl\n", "Collecting colorama\n", " Downloading https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl\n", "Collecting cloudmesh-common\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/7d/b8/828ad8fe9483221c6aeaaf128745d65e73ff395b29771b0633ab91bd1a0f/cloudmesh_common-4.3.57-py2.py3-none-any.whl (80kB)\n", "\u001b[K |████████████████████████████████| 81kB 5.6MB/s \n", "\u001b[?25hRequirement already satisfied: psutil in /usr/local/lib/python3.7/dist-packages (from cloudmesh-installer) (5.4.8)\n", "Requirement already satisfied: pytest in /usr/local/lib/python3.7/dist-packages (from cloudmesh-installer) (3.6.4)\n", "Requirement already satisfied: docopt in /usr/local/lib/python3.7/dist-packages (from cloudmesh-installer) (0.6.2)\n", "Requirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from cloudmesh-installer) (0.8.9)\n", "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from pip-tools->cloudmesh-installer) (1.15.0)\n", "Requirement already satisfied: click>=7 in /usr/local/lib/python3.7/dist-packages (from pip-tools->cloudmesh-installer) (7.1.2)\n", "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->cloudmesh-installer) (3.0.4)\n", "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->cloudmesh-installer) (2.10)\n", "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->cloudmesh-installer) (1.24.3)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->cloudmesh-installer) (2020.12.5)\n", "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from oyaml->cloudmesh-installer) (3.13)\n", "Collecting pycodestyle<2.7.0,>=2.6.0a1\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/10/5b/88879fb861ab79aef45c7e199cae3ef7af487b5603dcb363517a50602dd7/pycodestyle-2.6.0-py2.py3-none-any.whl (41kB)\n", "\u001b[K |████████████████████████████████| 51kB 6.5MB/s \n", "\u001b[?25hCollecting pyflakes<2.3.0,>=2.2.0\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/69/5b/fd01b0c696f2f9a6d2c839883b642493b431f28fa32b29abc465ef675473/pyflakes-2.2.0-py2.py3-none-any.whl (66kB)\n", "\u001b[K |████████████████████████████████| 71kB 5.4MB/s \n", "\u001b[?25hCollecting mccabe<0.7.0,>=0.6.0\n", " Downloading https://files.pythonhosted.org/packages/87/89/479dc97e18549e21354893e4ee4ef36db1d237534982482c3681ee6e7b57/mccabe-0.6.1-py2.py3-none-any.whl\n", "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from flake8->cloudmesh-installer) (3.7.0)\n", "Requirement already satisfied: pip>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from pipdeptree->cloudmesh-installer) (19.3.1)\n", "Collecting simplejson\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/a8/04/377418ac1e530ce2a196b54c6552c018fdf1fe776718053efb1f216bffcd/simplejson-3.17.2-cp37-cp37m-manylinux2010_x86_64.whl (128kB)\n", "\u001b[K |████████████████████████████████| 133kB 8.9MB/s \n", "\u001b[?25hCollecting pyfiglet\n", "\u001b[?25l Downloading https://files.pythonhosted.org/packages/33/07/fcfdd7a2872f5b348953de35acce1544dab0c1e8368dca54279b1cde5c15/pyfiglet-0.8.post1-py2.py3-none-any.whl (865kB)\n", "\u001b[K |████████████████████████████████| 870kB 9.1MB/s \n", "\u001b[?25hRequirement already satisfied: pathlib in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common->cloudmesh-installer) (1.0.1)\n", "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common->cloudmesh-installer) (2.8.1)\n", "Requirement already satisfied: pytz in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common->cloudmesh-installer) (2018.9)\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from pytest->cloudmesh-installer) (54.0.0)\n", "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.7/dist-packages (from pytest->cloudmesh-installer) (20.3.0)\n", "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.7/dist-packages (from pytest->cloudmesh-installer) (0.7.1)\n", "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from pytest->cloudmesh-installer) (8.7.0)\n", "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.7/dist-packages (from pytest->cloudmesh-installer) (1.4.0)\n", "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from pytest->cloudmesh-installer) (1.10.0)\n", "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->flake8->cloudmesh-installer) (3.7.4.3)\n", "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->flake8->cloudmesh-installer) (3.4.0)\n", "Building wheels for collected packages: python-hostlist, ordered-set\n", " Building wheel for python-hostlist (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for python-hostlist: filename=python_hostlist-1.21-cp37-none-any.whl size=38931 sha256=e28832a16c1f2d1e69f04876929d05c883e6e92b00edff275a5ef839117d4cea\n", " Stored in directory: /root/.cache/pip/wheels/0b/5b/55/ddcf52288f0b10f4564ca1b2531594ff7ccc65f487ba8dc437\n", " Building wheel for ordered-set (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for ordered-set: filename=ordered_set-4.0.2-py2.py3-none-any.whl size=8210 sha256=faf02216bc83168425070d828886a437aee42a227fcf7b3d37b2c418c5ce2005\n", " Stored in directory: /root/.cache/pip/wheels/e1/c6/9b/651d8a21d59b51a75ab9c070838f9231b8126421bc0569af47\n", "Successfully built python-hostlist ordered-set\n", "\u001b[31mERROR: pytest-cov 2.11.1 has requirement coverage>=5.2.1, but you'll have coverage 3.7.1 which is incompatible.\u001b[0m\n", "\u001b[31mERROR: pytest-cov 2.11.1 has requirement pytest>=4.6, but you'll have pytest 3.6.4 which is incompatible.\u001b[0m\n", "Installing collected packages: python-hostlist, ordered-set, oyaml, pycodestyle, pyflakes, mccabe, flake8, bump2version, pytest-cov, pipdeptree, colorama, simplejson, pyfiglet, cloudmesh-common, cloudmesh-installer\n", "Successfully installed bump2version-1.0.0 cloudmesh-common-4.3.57 cloudmesh-installer-4.4.24 colorama-0.4.4 flake8-3.8.4 mccabe-0.6.1 ordered-set-4.0.2 oyaml-1.0 pipdeptree-2.0.0 pycodestyle-2.6.0 pyfiglet-0.8.post1 pyflakes-2.2.0 pytest-cov-2.11.1 python-hostlist-1.21 simplejson-3.17.2\n", "Requirement already satisfied: cloudmesh-common in /usr/local/lib/python3.7/dist-packages (4.3.57)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common) (5.4.8)\n", "Requirement already satisfied: pytz in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common) (2018.9)\n", "Requirement already satisfied: pathlib in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common) (1.0.1)\n", "Requirement already satisfied: pyfiglet in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common) (0.8.post1)\n", "Requirement already satisfied: oyaml in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common) (1.0)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common) (2.23.0)\n", "Requirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common) (0.8.9)\n", "Requirement already satisfied: python-hostlist in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common) (1.21)\n", "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common) (2.8.1)\n", "Requirement already satisfied: simplejson in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common) (3.17.2)\n", "Requirement already satisfied: humanize in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common) (0.5.1)\n", "Requirement already satisfied: colorama in /usr/local/lib/python3.7/dist-packages (from cloudmesh-common) (0.4.4)\n", "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from oyaml->cloudmesh-common) (3.13)\n", "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->cloudmesh-common) (2.10)\n", "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->cloudmesh-common) (3.0.4)\n", "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->cloudmesh-common) (1.24.3)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->cloudmesh-common) (2020.12.5)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil->cloudmesh-common) (1.15.0)\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "Qn-x5O_xBiit" }, "source": [ "from __future__ import absolute_import\n", "from __future__ import division\n", "from __future__ import print_function\n", "\n", "import numpy as np\n", "import tensorflow as tf\n", "from tensorflow.keras.models import Sequential\n", "from tensorflow.keras.layers import Dense, Activation, SimpleRNN, InputLayer, LSTM, Dropout\n", "from tensorflow.keras.utils import to_categorical, plot_model\n", "from tensorflow.keras.datasets import mnist\n", "from cloudmesh.common.StopWatch import StopWatch" ], "execution_count": 2, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "m44N0arzvqx5" }, "source": [ "## Data Pre-Process" ] }, { "cell_type": "code", "metadata": { "id": "amkZr0dbvt7M", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "bb843a13-502a-49fa-b0eb-5b1d5f1668e6" }, "source": [ "StopWatch.start(\"data-load\")\n", "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", "StopWatch.stop(\"data-load\")\n", "\n", "\n", "StopWatch.start(\"data-pre-process\")\n", "num_labels = len(np.unique(y_train))\n", "\n", "\n", "y_train = to_categorical(y_train)\n", "y_test = to_categorical(y_test)\n", "\n", "\n", "image_size = x_train.shape[1]\n", "x_train = np.reshape(x_train,[-1, image_size, image_size])\n", "x_test = np.reshape(x_test,[-1, image_size, image_size])\n", "x_train = x_train.astype('float32') / 255\n", "x_test = x_test.astype('float32') / 255\n", "StopWatch.stop(\"data-pre-process\")\n", "\n", "input_shape = (image_size, image_size)\n", "batch_size = 128\n", "units = 256\n", "dropout = 0.2" ], "execution_count": 3, "outputs": [ { "output_type": "stream", "text": [ "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n", "11493376/11490434 [==============================] - 0s 0us/step\n" ], "name": "stdout" } ] }, { "cell_type": "markdown", "metadata": { "id": "Ku3FZI9VvvDt" }, "source": [ "## Define Model" ] }, { "cell_type": "markdown", "metadata": { "id": "pDNT4-APe-Jf" }, "source": [ "Here we use the Tensorflow distributed training components to train the model in multiple CPUs or GPUs. In the Colab instance multiple GPUs are not supported. Hence, the training must be done in the device type 'None' when selecting the 'runtime type' from Runtime menu. To run with multiple-GPUs no code change is required. [Learn more about distributed training](https://www.tensorflow.org/guide/distributed_training)." ] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "3Xr1SPagvxpq", "outputId": "1b026d05-8aa8-4ec0-ea43-7bd75fc16f14" }, "source": [ "StopWatch.start(\"compile\")\n", "strategy = tf.distribute.MirroredStrategy()\n", "with strategy.scope():\n", " model = Sequential()\n", " # LSTM Layers\n", " model.add(LSTM(units=units, \n", " input_shape=input_shape,\n", " return_sequences=True))\n", " model.add(LSTM(units=units, \n", " dropout=dropout, \n", " return_sequences=True))\n", " model.add(LSTM(units=units, \n", " dropout=dropout, \n", " return_sequences=False))\n", " # MLP Layers\n", " model.add(Dense(units))\n", " model.add(Activation('relu'))\n", " model.add(Dropout(dropout))\n", " model.add(Dense(units))\n", " model.add(Activation('relu'))\n", " model.add(Dropout(dropout))\n", " # Softmax_layer\n", " model.add(Dense(num_labels))\n", " model.add(Activation('softmax'))\n", " model.summary()\n", " plot_model(model, to_file='rnn-mnist.png', show_shapes=True)\n", " \n", " print(\"Number of devices: {}\".format(strategy.num_replicas_in_sync))\n", "\n", " model.compile(loss='categorical_crossentropy',\n", " optimizer='sgd',\n", " metrics=['accuracy'])\n", "StopWatch.stop(\"compile\")" ], "execution_count": 10, "outputs": [ { "output_type": "stream", "text": [ "Model: \"sequential_2\"\n", "_________________________________________________________________\n", "Layer (type) Output Shape Param # \n", "=================================================================\n", "lstm_6 (LSTM) (None, 28, 256) 291840 \n", "_________________________________________________________________\n", "lstm_7 (LSTM) (None, 28, 256) 525312 \n", "_________________________________________________________________\n", "lstm_8 (LSTM) (None, 256) 525312 \n", "_________________________________________________________________\n", "dense_6 (Dense) (None, 256) 65792 \n", "_________________________________________________________________\n", "activation_6 (Activation) (None, 256) 0 \n", "_________________________________________________________________\n", "dropout_4 (Dropout) (None, 256) 0 \n", "_________________________________________________________________\n", "dense_7 (Dense) (None, 256) 65792 \n", "_________________________________________________________________\n", "activation_7 (Activation) (None, 256) 0 \n", "_________________________________________________________________\n", "dropout_5 (Dropout) (None, 256) 0 \n", "_________________________________________________________________\n", "dense_8 (Dense) (None, 10) 2570 \n", "_________________________________________________________________\n", "activation_8 (Activation) (None, 10) 0 \n", "=================================================================\n", "Total params: 1,476,618\n", "Trainable params: 1,476,618\n", "Non-trainable params: 0\n", "_________________________________________________________________\n", "INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:GPU:0',)\n", "Number of devices: 1\n", "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n", "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n", "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n", "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n" ], "name": "stdout" } ] }, { "cell_type": "markdown", "metadata": { "id": "0hwr9YOzv4aY" }, "source": [ "## Train" ] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Xe9Mskn7v5cv", "outputId": "cb6d03d1-cb96-450b-f2d8-e84b86474650" }, "source": [ "StopWatch.start(\"train\")\n", "model.fit(x_train, y_train, epochs=30, batch_size=batch_size)\n", "StopWatch.stop(\"train\")" ], "execution_count": 12, "outputs": [ { "output_type": "stream", "text": [ "Epoch 1/30\n", "469/469 [==============================] - 7s 16ms/step - loss: 2.0427 - accuracy: 0.2718\n", "Epoch 2/30\n", "469/469 [==============================] - 7s 16ms/step - loss: 1.6934 - accuracy: 0.4007\n", "Epoch 3/30\n", "469/469 [==============================] - 7s 16ms/step - loss: 1.2997 - accuracy: 0.5497\n", "Epoch 4/30\n", "469/469 [==============================] - 7s 16ms/step - loss: 1.0248 - accuracy: 0.6434\n", "Epoch 5/30\n", "469/469 [==============================] - 7s 16ms/step - loss: 0.8614 - accuracy: 0.6988\n", "Epoch 6/30\n", "469/469 [==============================] - 7s 16ms/step - loss: 0.7409 - accuracy: 0.7443\n", "Epoch 7/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.6460 - accuracy: 0.7804\n", "Epoch 8/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.5657 - accuracy: 0.8119\n", "Epoch 9/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.4978 - accuracy: 0.8365\n", "Epoch 10/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.4327 - accuracy: 0.8595\n", "Epoch 11/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.3771 - accuracy: 0.8793\n", "Epoch 12/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.3314 - accuracy: 0.8942\n", "Epoch 13/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.2944 - accuracy: 0.9075\n", "Epoch 14/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.2661 - accuracy: 0.9174\n", "Epoch 15/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.2405 - accuracy: 0.9247\n", "Epoch 16/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.2218 - accuracy: 0.9316\n", "Epoch 17/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.2084 - accuracy: 0.9359\n", "Epoch 18/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.1915 - accuracy: 0.9408\n", "Epoch 19/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.1796 - accuracy: 0.9444\n", "Epoch 20/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.1700 - accuracy: 0.9480\n", "Epoch 21/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.1607 - accuracy: 0.9513\n", "Epoch 22/30\n", "469/469 [==============================] - 8s 17ms/step - loss: 0.1534 - accuracy: 0.9530\n", "Epoch 23/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.1483 - accuracy: 0.9548\n", "Epoch 24/30\n", "469/469 [==============================] - 8s 17ms/step - loss: 0.1406 - accuracy: 0.9572\n", "Epoch 25/30\n", "469/469 [==============================] - 8s 16ms/step - loss: 0.1333 - accuracy: 0.9597\n", "Epoch 26/30\n", "469/469 [==============================] - 8s 17ms/step - loss: 0.1271 - accuracy: 0.9604\n", "Epoch 27/30\n", "469/469 [==============================] - 8s 17ms/step - loss: 0.1229 - accuracy: 0.9622\n", "Epoch 28/30\n", "469/469 [==============================] - 8s 17ms/step - loss: 0.1175 - accuracy: 0.9640\n", "Epoch 29/30\n", "469/469 [==============================] - 8s 17ms/step - loss: 0.1158 - accuracy: 0.9645\n", "Epoch 30/30\n", "469/469 [==============================] - 8s 17ms/step - loss: 0.1098 - accuracy: 0.9661\n" ], "name": "stdout" } ] }, { "cell_type": "markdown", "metadata": { "id": "ZPGR0hNJv6ln" }, "source": [ "## Test" ] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "-htoQ51sv71v", "outputId": "e2b0da63-759c-4ed4-e692-da6a663f9b34" }, "source": [ "StopWatch.start(\"evaluate\")\n", "loss, acc = model.evaluate(x_test, y_test, batch_size=batch_size)\n", "print(\"\\nTest accuracy: %.1f%%\" % (100.0 * acc))\n", "StopWatch.stop(\"evaluate\")\n", "\n", "StopWatch.benchmark()" ], "execution_count": 13, "outputs": [ { "output_type": "stream", "text": [ "79/79 [==============================] - 3s 9ms/step - loss: 0.0898 - accuracy: 0.9719\n", "\n", "Test accuracy: 97.2%\n", "\n", "+---------------------+------------------------------------------------------------------+\n", "| Attribute | Value |\n", "|---------------------+------------------------------------------------------------------|\n", "| BUG_REPORT_URL | \"https://bugs.launchpad.net/ubuntu/\" |\n", "| DISTRIB_CODENAME | bionic |\n", "| DISTRIB_DESCRIPTION | \"Ubuntu 18.04.5 LTS\" |\n", "| DISTRIB_ID | Ubuntu |\n", "| DISTRIB_RELEASE | 18.04 |\n", "| HOME_URL | \"https://www.ubuntu.com/\" |\n", "| ID | ubuntu |\n", "| ID_LIKE | debian |\n", "| NAME | \"Ubuntu\" |\n", "| PRETTY_NAME | \"Ubuntu 18.04.5 LTS\" |\n", "| PRIVACY_POLICY_URL | \"https://www.ubuntu.com/legal/terms-and-policies/privacy-policy\" |\n", "| SUPPORT_URL | \"https://help.ubuntu.com/\" |\n", "| UBUNTU_CODENAME | bionic |\n", "| VERSION | \"18.04.5 LTS (Bionic Beaver)\" |\n", "| VERSION_CODENAME | bionic |\n", "| VERSION_ID | \"18.04\" |\n", "| cpu_count | 2 |\n", "| mem.active | 2.4 GiB |\n", "| mem.available | 10.3 GiB |\n", "| mem.free | 4.5 GiB |\n", "| mem.inactive | 5.4 GiB |\n", "| mem.percent | 18.6 % |\n", "| mem.total | 12.7 GiB |\n", "| mem.used | 3.3 GiB |\n", "| platform.version | #1 SMP Thu Jul 23 08:00:38 PDT 2020 |\n", "| python | 3.7.10 (default, Feb 20 2021, 21:17:23) |\n", "| | [GCC 7.5.0] |\n", "| python.pip | 19.3.1 |\n", "| python.version | 3.7.10 |\n", "| sys.platform | linux |\n", "| uname.machine | x86_64 |\n", "| uname.node | b39e0899c1f8 |\n", "| uname.processor | x86_64 |\n", "| uname.release | 4.19.112+ |\n", "| uname.system | Linux |\n", "| uname.version | #1 SMP Thu Jul 23 08:00:38 PDT 2020 |\n", "| user | collab |\n", "+---------------------+------------------------------------------------------------------+\n", "\n", "+------------------+----------+---------+---------+---------------------+-------+--------------+--------+-------+-------------------------------------+\n", "| Name | Status | Time | Sum | Start | tag | Node | User | OS | Version |\n", "|------------------+----------+---------+---------+---------------------+-------+--------------+--------+-------+-------------------------------------|\n", "| data-load | failed | 0.473 | 0.473 | 2021-03-07 11:34:03 | | b39e0899c1f8 | collab | Linux | #1 SMP Thu Jul 23 08:00:38 PDT 2020 |\n", "| data-pre-process | failed | 0.073 | 0.073 | 2021-03-07 11:34:03 | | b39e0899c1f8 | collab | Linux | #1 SMP Thu Jul 23 08:00:38 PDT 2020 |\n", "| compile | failed | 0.876 | 7.187 | 2021-03-07 11:38:05 | | b39e0899c1f8 | collab | Linux | #1 SMP Thu Jul 23 08:00:38 PDT 2020 |\n", "| train | failed | 229.341 | 257.023 | 2021-03-07 11:38:44 | | b39e0899c1f8 | collab | Linux | #1 SMP Thu Jul 23 08:00:38 PDT 2020 |\n", "| evaluate | failed | 2.659 | 4.25 | 2021-03-07 11:44:54 | | b39e0899c1f8 | collab | Linux | #1 SMP Thu Jul 23 08:00:38 PDT 2020 |\n", "+------------------+----------+---------+---------+---------------------+-------+--------------+--------+-------+-------------------------------------+\n", "\n", "# csv,timer,status,time,sum,start,tag,uname.node,user,uname.system,platform.version\n", "# csv,data-load,failed,0.473,0.473,2021-03-07 11:34:03,,b39e0899c1f8,collab,Linux,#1 SMP Thu Jul 23 08:00:38 PDT 2020\n", "# csv,data-pre-process,failed,0.073,0.073,2021-03-07 11:34:03,,b39e0899c1f8,collab,Linux,#1 SMP Thu Jul 23 08:00:38 PDT 2020\n", "# csv,compile,failed,0.876,7.187,2021-03-07 11:38:05,,b39e0899c1f8,collab,Linux,#1 SMP Thu Jul 23 08:00:38 PDT 2020\n", "# csv,train,failed,229.341,257.023,2021-03-07 11:38:44,,b39e0899c1f8,collab,Linux,#1 SMP Thu Jul 23 08:00:38 PDT 2020\n", "# csv,evaluate,failed,2.659,4.25,2021-03-07 11:44:54,,b39e0899c1f8,collab,Linux,#1 SMP Thu Jul 23 08:00:38 PDT 2020\n", "\n" ], "name": "stdout" } ] }, { "cell_type": "markdown", "metadata": { "id": "k8vqRCcsDEeu" }, "source": [ "# References\n", "\n", "1. [Advance Deep Learning with Keras](https://github.com/PacktPublishing/Advanced-Deep-Learning-with-Keras)\n", "2. [Distributed With Tensorflow](https://www.tensorflow.org/guide/distributed_training)\n", "3. [Keras with Tensorflow Distributed Training](https://keras.io/guides/distributed_training/)" ] } ] }