{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Digit Recognizer using Multiclass Clasification\n", "* https://www.kaggle.com/c/digit-recognizer" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import tensorflow as tf" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## prepareing data\n", "Download data from https://www.kaggle.com/c/digit-recognizer/data" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "n_input = 784 # MNIST data input (img shape: 28*28)\n", "n_classes = 10 # MNIST total classes (0-9 digits)\n", "\n", "validation_size = 2000" ] }, { "cell_type": "code", "execution_count": 39, "metadata": { "collapsed": true }, "outputs": [], "source": [ "train = pd.read_csv('../input/digit-recognizer/train.csv')\n", "test = pd.read_csv('../input/digit-recognizer/test.csv')" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(42000, 785)\n", "(28000, 784)\n" ] } ], "source": [ "print(train.shape)\n", "print(test.shape)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
labelpixel0pixel1pixel2pixel3pixel4pixel5pixel6pixel7pixel8...pixel774pixel775pixel776pixel777pixel778pixel779pixel780pixel781pixel782pixel783
01000000000...0000000000
10000000000...0000000000
21000000000...0000000000
34000000000...0000000000
40000000000...0000000000
\n", "

5 rows × 785 columns

\n", "
" ], "text/plain": [ " label pixel0 pixel1 pixel2 pixel3 pixel4 pixel5 pixel6 pixel7 \\\n", "0 1 0 0 0 0 0 0 0 0 \n", "1 0 0 0 0 0 0 0 0 0 \n", "2 1 0 0 0 0 0 0 0 0 \n", "3 4 0 0 0 0 0 0 0 0 \n", "4 0 0 0 0 0 0 0 0 0 \n", "\n", " pixel8 ... pixel774 pixel775 pixel776 pixel777 pixel778 \\\n", "0 0 ... 0 0 0 0 0 \n", "1 0 ... 0 0 0 0 0 \n", "2 0 ... 0 0 0 0 0 \n", "3 0 ... 0 0 0 0 0 \n", "4 0 ... 0 0 0 0 0 \n", "\n", " pixel779 pixel780 pixel781 pixel782 pixel783 \n", "0 0 0 0 0 0 \n", "1 0 0 0 0 0 \n", "2 0 0 0 0 0 \n", "3 0 0 0 0 0 \n", "4 0 0 0 0 0 \n", "\n", "[5 rows x 785 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train.head()" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
pixel0pixel1pixel2pixel3pixel4pixel5pixel6pixel7pixel8pixel9...pixel774pixel775pixel776pixel777pixel778pixel779pixel780pixel781pixel782pixel783
00000000000...0000000000
10000000000...0000000000
20000000000...0000000000
30000000000...0000000000
40000000000...0000000000
\n", "

5 rows × 784 columns

\n", "
" ], "text/plain": [ " pixel0 pixel1 pixel2 pixel3 pixel4 pixel5 pixel6 pixel7 pixel8 \\\n", "0 0 0 0 0 0 0 0 0 0 \n", "1 0 0 0 0 0 0 0 0 0 \n", "2 0 0 0 0 0 0 0 0 0 \n", "3 0 0 0 0 0 0 0 0 0 \n", "4 0 0 0 0 0 0 0 0 0 \n", "\n", " pixel9 ... pixel774 pixel775 pixel776 pixel777 pixel778 \\\n", "0 0 ... 0 0 0 0 0 \n", "1 0 ... 0 0 0 0 0 \n", "2 0 ... 0 0 0 0 0 \n", "3 0 ... 0 0 0 0 0 \n", "4 0 ... 0 0 0 0 0 \n", "\n", " pixel779 pixel780 pixel781 pixel782 pixel783 \n", "0 0 0 0 0 0 \n", "1 0 0 0 0 0 \n", "2 0 0 0 0 0 \n", "3 0 0 0 0 0 \n", "4 0 0 0 0 0 \n", "\n", "[5 rows x 784 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test.head()" ] }, { "cell_type": "markdown", "metadata": { "collapsed": true }, "source": [ "split train data to labels and pixels." ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": true }, "outputs": [], "source": [ "features = (train.ix[:,1:].values).astype('float32')\n", "labels = pd.get_dummies(train.ix[:,0]).astype('float32') # one hot encoding" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(42000, 784)\n", "(42000, 10)\n" ] } ], "source": [ "print(features.shape)\n", "print(labels.shape)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# split data into training & validation\n", "valid_features = features[:validation_size]\n", "valid_labels = labels[:validation_size]\n", "\n", "train_features = features[validation_size:]\n", "train_labels = labels[validation_size:]" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(40000, 784)\n", "(40000, 10)\n", "(2000, 784)\n", "(2000, 10)\n" ] } ], "source": [ "print(train_features.shape)\n", "print(train_labels.shape)\n", "print(valid_features.shape)\n", "print(valid_labels.shape)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "collapsed": true }, "outputs": [], "source": [ "test_features = (test.values).astype('float32')" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(28000, 784)\n" ] } ], "source": [ "print(test_features.shape)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Make a TensorFlow Graph" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Features and Labels\n", "features = tf.placeholder(tf.float32, [None, n_input])\n", "labels = tf.placeholder(tf.float32, [None, n_classes])\n", "\n", "# Weights & bias\n", "weights = tf.Variable(tf.random_normal([n_input, n_classes]))\n", "bias = tf.Variable(tf.random_normal([n_classes]))\n", "\n", "# Logits - xW + b\n", "logits = tf.add(tf.matmul(features, weights), bias)\n", "\n", "# Define loss and optimizer\n", "learning_rate = tf.placeholder(tf.float32)\n", "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels))\n", "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n", "\n", "# Calculate accuracy\n", "predict = tf.argmax(logits, 1)\n", "correct_prediction = tf.equal(predict, tf.argmax(labels, 1))\n", "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n", "\n", "init = tf.global_variables_initializer()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Training" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Define helper functions." ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def print_epoch_stats(epoch_i, sess, last_features, last_labels):\n", " \"\"\"\n", " Print cost and validation accuracy of an epoch\n", " \"\"\"\n", " current_cost = sess.run(\n", " cost,\n", " feed_dict={features: last_features, labels: last_labels})\n", " valid_accuracy = sess.run(\n", " accuracy,\n", " feed_dict={features: valid_features, labels: valid_labels})\n", " print('Epoch: {:<4} - Cost: {:<8.3} Valid Accuracy: {:<5.3}'.format(\n", " epoch_i,\n", " current_cost,\n", " valid_accuracy))" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import math\n", "def batches(batch_size, features, labels):\n", " \"\"\"\n", " Create batches of features and labels\n", " :param batch_size: The batch size\n", " :param features: List of features\n", " :param labels: List of labels\n", " :return: Batches of (Features, Labels)\n", " \"\"\"\n", " assert len(features) == len(labels)\n", " outout_batches = []\n", " \n", " sample_size = len(features)\n", " for start_i in range(0, sample_size, batch_size):\n", " end_i = start_i + batch_size\n", " batch = [features[start_i:end_i], labels[start_i:end_i]]\n", " outout_batches.append(batch)\n", " \n", " return outout_batches" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "HyperParameters" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "collapsed": true }, "outputs": [], "source": [ "batch_size = 128\n", "epochs = 100\n", "learn_rate = 0.0001" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "collapsed": true }, "outputs": [], "source": [ "train_batches = batches(batch_size, train_features, train_labels)" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch: 0 - Cost: 2.76e+03 Valid Accuracy: 0.114\n", "Epoch: 1 - Cost: 2.43e+03 Valid Accuracy: 0.143\n", "Epoch: 2 - Cost: 2.14e+03 Valid Accuracy: 0.174\n", "Epoch: 3 - Cost: 1.87e+03 Valid Accuracy: 0.212\n", "Epoch: 4 - Cost: 1.64e+03 Valid Accuracy: 0.25 \n", "Epoch: 5 - Cost: 1.46e+03 Valid Accuracy: 0.294\n", "Epoch: 6 - Cost: 1.3e+03 Valid Accuracy: 0.331\n", "Epoch: 7 - Cost: 1.17e+03 Valid Accuracy: 0.373\n", "Epoch: 8 - Cost: 1.05e+03 Valid Accuracy: 0.411\n", "Epoch: 9 - Cost: 9.48e+02 Valid Accuracy: 0.439\n", "Epoch: 10 - Cost: 8.55e+02 Valid Accuracy: 0.463\n", "Epoch: 11 - Cost: 7.71e+02 Valid Accuracy: 0.491\n", "Epoch: 12 - Cost: 6.96e+02 Valid Accuracy: 0.516\n", "Epoch: 13 - Cost: 6.27e+02 Valid Accuracy: 0.537\n", "Epoch: 14 - Cost: 5.68e+02 Valid Accuracy: 0.558\n", "Epoch: 15 - Cost: 5.18e+02 Valid Accuracy: 0.582\n", "Epoch: 16 - Cost: 4.72e+02 Valid Accuracy: 0.604\n", "Epoch: 17 - Cost: 4.36e+02 Valid Accuracy: 0.623\n", "Epoch: 18 - Cost: 4.06e+02 Valid Accuracy: 0.64 \n", "Epoch: 19 - Cost: 3.83e+02 Valid Accuracy: 0.658\n", "Epoch: 20 - Cost: 3.62e+02 Valid Accuracy: 0.668\n", "Epoch: 21 - Cost: 3.42e+02 Valid Accuracy: 0.679\n", "Epoch: 22 - Cost: 3.24e+02 Valid Accuracy: 0.689\n", "Epoch: 23 - Cost: 3.11e+02 Valid Accuracy: 0.695\n", "Epoch: 24 - Cost: 3.01e+02 Valid Accuracy: 0.706\n", "Epoch: 25 - Cost: 2.91e+02 Valid Accuracy: 0.714\n", "Epoch: 26 - Cost: 2.81e+02 Valid Accuracy: 0.724\n", "Epoch: 27 - Cost: 2.72e+02 Valid Accuracy: 0.73 \n", "Epoch: 28 - Cost: 2.63e+02 Valid Accuracy: 0.738\n", "Epoch: 29 - Cost: 2.56e+02 Valid Accuracy: 0.743\n", "Epoch: 30 - Cost: 2.5e+02 Valid Accuracy: 0.751\n", "Epoch: 31 - Cost: 2.43e+02 Valid Accuracy: 0.756\n", "Epoch: 32 - Cost: 2.38e+02 Valid Accuracy: 0.759\n", "Epoch: 33 - Cost: 2.33e+02 Valid Accuracy: 0.763\n", "Epoch: 34 - Cost: 2.28e+02 Valid Accuracy: 0.768\n", "Epoch: 35 - Cost: 2.23e+02 Valid Accuracy: 0.772\n", "Epoch: 36 - Cost: 2.18e+02 Valid Accuracy: 0.777\n", "Epoch: 37 - Cost: 2.13e+02 Valid Accuracy: 0.781\n", "Epoch: 38 - Cost: 2.09e+02 Valid Accuracy: 0.784\n", "Epoch: 39 - Cost: 2.06e+02 Valid Accuracy: 0.789\n", "Epoch: 40 - Cost: 2.02e+02 Valid Accuracy: 0.793\n", "Epoch: 41 - Cost: 1.98e+02 Valid Accuracy: 0.798\n", "Epoch: 42 - Cost: 1.95e+02 Valid Accuracy: 0.8 \n", "Epoch: 43 - Cost: 1.91e+02 Valid Accuracy: 0.803\n", "Epoch: 44 - Cost: 1.88e+02 Valid Accuracy: 0.808\n", "Epoch: 45 - Cost: 1.85e+02 Valid Accuracy: 0.812\n", "Epoch: 46 - Cost: 1.82e+02 Valid Accuracy: 0.814\n", "Epoch: 47 - Cost: 1.79e+02 Valid Accuracy: 0.817\n", "Epoch: 48 - Cost: 1.76e+02 Valid Accuracy: 0.819\n", "Epoch: 49 - Cost: 1.73e+02 Valid Accuracy: 0.82 \n", "Epoch: 50 - Cost: 1.71e+02 Valid Accuracy: 0.821\n", "Epoch: 51 - Cost: 1.67e+02 Valid Accuracy: 0.821\n", "Epoch: 52 - Cost: 1.64e+02 Valid Accuracy: 0.823\n", "Epoch: 53 - Cost: 1.61e+02 Valid Accuracy: 0.826\n", "Epoch: 54 - Cost: 1.58e+02 Valid Accuracy: 0.828\n", "Epoch: 55 - Cost: 1.55e+02 Valid Accuracy: 0.829\n", "Epoch: 56 - Cost: 1.52e+02 Valid Accuracy: 0.828\n", "Epoch: 57 - Cost: 1.49e+02 Valid Accuracy: 0.831\n", "Epoch: 58 - Cost: 1.46e+02 Valid Accuracy: 0.831\n", "Epoch: 59 - Cost: 1.44e+02 Valid Accuracy: 0.833\n", "Epoch: 60 - Cost: 1.41e+02 Valid Accuracy: 0.834\n", "Epoch: 61 - Cost: 1.38e+02 Valid Accuracy: 0.835\n", "Epoch: 62 - Cost: 1.36e+02 Valid Accuracy: 0.836\n", "Epoch: 63 - Cost: 1.34e+02 Valid Accuracy: 0.837\n", "Epoch: 64 - Cost: 1.31e+02 Valid Accuracy: 0.84 \n", "Epoch: 65 - Cost: 1.29e+02 Valid Accuracy: 0.84 \n", "Epoch: 66 - Cost: 1.27e+02 Valid Accuracy: 0.84 \n", "Epoch: 67 - Cost: 1.25e+02 Valid Accuracy: 0.841\n", "Epoch: 68 - Cost: 1.23e+02 Valid Accuracy: 0.84 \n", "Epoch: 69 - Cost: 1.21e+02 Valid Accuracy: 0.841\n", "Epoch: 70 - Cost: 1.19e+02 Valid Accuracy: 0.841\n", "Epoch: 71 - Cost: 1.17e+02 Valid Accuracy: 0.842\n", "Epoch: 72 - Cost: 1.15e+02 Valid Accuracy: 0.843\n", "Epoch: 73 - Cost: 1.12e+02 Valid Accuracy: 0.845\n", "Epoch: 74 - Cost: 1.1e+02 Valid Accuracy: 0.845\n", "Epoch: 75 - Cost: 1.08e+02 Valid Accuracy: 0.845\n", "Epoch: 76 - Cost: 1.06e+02 Valid Accuracy: 0.845\n", "Epoch: 77 - Cost: 1.04e+02 Valid Accuracy: 0.846\n", "Epoch: 78 - Cost: 1.02e+02 Valid Accuracy: 0.847\n", "Epoch: 79 - Cost: 99.9 Valid Accuracy: 0.848\n", "Epoch: 80 - Cost: 97.9 Valid Accuracy: 0.848\n", "Epoch: 81 - Cost: 95.9 Valid Accuracy: 0.848\n", "Epoch: 82 - Cost: 93.9 Valid Accuracy: 0.849\n", "Epoch: 83 - Cost: 92.0 Valid Accuracy: 0.849\n", "Epoch: 84 - Cost: 90.3 Valid Accuracy: 0.849\n", "Epoch: 85 - Cost: 88.6 Valid Accuracy: 0.849\n", "Epoch: 86 - Cost: 87.0 Valid Accuracy: 0.849\n", "Epoch: 87 - Cost: 85.3 Valid Accuracy: 0.85 \n", "Epoch: 88 - Cost: 83.6 Valid Accuracy: 0.85 \n", "Epoch: 89 - Cost: 82.1 Valid Accuracy: 0.851\n", "Epoch: 90 - Cost: 80.7 Valid Accuracy: 0.851\n", "Epoch: 91 - Cost: 79.6 Valid Accuracy: 0.853\n", "Epoch: 92 - Cost: 78.4 Valid Accuracy: 0.854\n", "Epoch: 93 - Cost: 77.3 Valid Accuracy: 0.854\n", "Epoch: 94 - Cost: 76.1 Valid Accuracy: 0.854\n", "Epoch: 95 - Cost: 75.3 Valid Accuracy: 0.854\n", "Epoch: 96 - Cost: 74.4 Valid Accuracy: 0.855\n", "Epoch: 97 - Cost: 73.5 Valid Accuracy: 0.856\n", "Epoch: 98 - Cost: 72.6 Valid Accuracy: 0.855\n", "Epoch: 99 - Cost: 72.0 Valid Accuracy: 0.855\n" ] } ], "source": [ "with tf.Session() as sess:\n", " sess.run(init)\n", "\n", " # Training cycle\n", " for epoch_i in range(epochs):\n", "\n", " # Loop over all batches\n", " for batch_features, batch_labels in train_batches:\n", " train_feed_dict = {\n", " features: batch_features,\n", " labels: batch_labels,\n", " learning_rate: learn_rate}\n", " sess.run(optimizer, feed_dict=train_feed_dict)\n", "\n", " # Print cost and validation accuracy of an epoch\n", " print_epoch_stats(epoch_i, sess, batch_features, batch_labels)\n", "\n", " predictions = sess.run(\n", " predict, \n", " feed_dict={features: test_features})" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Write to file" ] }, { "cell_type": "code", "execution_count": 38, "metadata": { "collapsed": true }, "outputs": [], "source": [ "submissions = pd.DataFrame({\"ImageId\": list(range(1, len(predictions)+1)),\n", " \"Label\": predictions})\n", "submissions.to_csv(\"output.csv\", index=False, header=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.2" }, "toc": { "colors": { "hover_highlight": "#DAA520", "navigate_num": "#000000", "navigate_text": "#333333", "running_highlight": "#FF0000", "selected_highlight": "#FFD700", "sidebar_border": "#EEEEEE", "wrapper_background": "#FFFFFF" }, "moveMenuLeft": true, "nav_menu": { "height": "57px", "width": "252px" }, "navigate_menu": true, "number_sections": false, "sideBar": true, "threshold": 4, "toc_cell": false, "toc_section_display": "block", "toc_window_display": false, "widenNotebook": false } }, "nbformat": 4, "nbformat_minor": 2 }