{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "84d4608d-4cc3-fcbb-57fb-61f07ad7d020", "_uuid": "6407080d145a62b4803d7f159c00118a056a7b5f" }, "source": [ "# Deep Neural Network training on MNIST\n", "\n", "This notebook is based on this [Kaggle project](https://www.kaggle.com/kernels/scriptcontent/4482867/download), adapted to fit into Cloudera [AI to EDGE demo](https://github.com/paulvid/ai_to_edge)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "654456b6-e648-0379-0d66-1cc97af6d00d", "_uuid": "6b48ce0e361bdb67689dd2f254ecedd9ade1f5ff" }, "source": [ "**Import all required libraries**\n", "===============================" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "_cell_guid": "e5b02688-c589-5a89-e11c-837c6a99eb6e", "_uuid": "f043e48097bfd98e41710142dd8aac41fa88a801" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Using TensorFlow backend.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "test.csv\n", "test.csv.gz\n", "train.csv.gz\n", "\n" ] } ], "source": [ "# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "\n", "from keras.models import Sequential\n", "from keras.layers import Dense , Dropout , Lambda, Flatten\n", "from keras.optimizers import Adam ,RMSprop\n", "from sklearn.model_selection import train_test_split\n", "from keras import backend as K\n", "from keras.preprocessing.image import ImageDataGenerator\n", "\n", "\n", "import gzip\n", "import shutil\n", "\n", "# Input data files are available in the \"./input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"./input\"]).decode(\"utf8\"))\n", "\n", "# adding Open Neural Network Exchange (ONNX) \n", "import onnxruntime" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "22a7fd70-ab61-432d-24cb-93e558414495", "_uuid": "62fbd0fe9c338b7ac0b04e688c8ee7947e6170f7" }, "source": [ "**Load Train and Test data**\n", "============================" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "# unzipping files\n", " \n", "with gzip.open('./input/train.csv.gz', 'rb') as f_in:\n", " with open('./input/train.csv', 'wb') as f_out:\n", " shutil.copyfileobj(f_in, f_out)\n", " \n", "with gzip.open('./input/test.csv.gz', 'rb') as f_in:\n", " with open('./input/test.csv', 'wb') as f_out:\n", " shutil.copyfileobj(f_in, f_out)\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "_cell_guid": "05226b08-226a-1a00-044d-a0e6b2101388", "_uuid": "4eff577bcd43479a3b7e91180393cbad9fcfca33" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(42000, 785)\n" ] }, { "data": { "text/html": [ "
\n", " | label | \n", "pixel0 | \n", "pixel1 | \n", "pixel2 | \n", "pixel3 | \n", "pixel4 | \n", "pixel5 | \n", "pixel6 | \n", "pixel7 | \n", "pixel8 | \n", "... | \n", "pixel774 | \n", "pixel775 | \n", "pixel776 | \n", "pixel777 | \n", "pixel778 | \n", "pixel779 | \n", "pixel780 | \n", "pixel781 | \n", "pixel782 | \n", "pixel783 | \n", "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "1 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "
1 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "
2 | \n", "1 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "
3 | \n", "4 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "
4 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "
5 rows × 785 columns
\n", "\n", " | pixel0 | \n", "pixel1 | \n", "pixel2 | \n", "pixel3 | \n", "pixel4 | \n", "pixel5 | \n", "pixel6 | \n", "pixel7 | \n", "pixel8 | \n", "pixel9 | \n", "... | \n", "pixel774 | \n", "pixel775 | \n", "pixel776 | \n", "pixel777 | \n", "pixel778 | \n", "pixel779 | \n", "pixel780 | \n", "pixel781 | \n", "pixel782 | \n", "pixel783 | \n", "
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "
1 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "
2 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "
3 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "
4 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "... | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "0 | \n", "
5 rows × 784 columns
\n", "