{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "multi-class_classification_of_handwritten_digits.ipynb", "version": "0.3.2", "views": {}, "default_view": {}, "provenance": [], "collapsed_sections": [ "266KQvZoMxMv", "6sfw3LH0Oycm", "copyright-notice" ] } }, "cells": [ { "source": [ "#### Copyright 2017 Google LLC." ], "cell_type": "markdown", "metadata": { "colab_type": "text", "id": "copyright-notice" } }, { "source": [ "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", "#\n", "# https://www.apache.org/licenses/LICENSE-2.0\n", "#\n", "# Unless required by applicable law or agreed to in writing, software\n", "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." ], "execution_count": 0, "cell_type": "code", "outputs": [], "metadata": { "colab": { "autoexec": { "wait_interval": 0, "startup": false } }, "colab_type": "code", "id": "copyright-notice2", "cellView": "both" } }, { "metadata": { "id": "mPa95uXvcpcn", "colab_type": "text" }, "cell_type": "markdown", "source": [ " # \u4f7f\u7528\u795e\u7ecf\u7f51\u7edc\u5bf9\u624b\u5199\u6570\u5b57\u8fdb\u884c\u5206\u7c7b" ] }, { "metadata": { "id": "Fdpn8b90u8Tp", "colab_type": "text" }, "cell_type": "markdown", "source": [ " ![img](https://www.tensorflow.org/versions/r0.11/images/MNIST.png)" ] }, { "metadata": { "id": "c7HLCm66Cs2p", "colab_type": "text" }, "cell_type": "markdown", "source": [ " **\u5b66\u4e60\u76ee\u6807\uff1a**\n", " * \u8bad\u7ec3\u7ebf\u6027\u6a21\u578b\u548c\u795e\u7ecf\u7f51\u7edc\uff0c\u4ee5\u5bf9\u4f20\u7edf [MNIST](http://yann.lecun.com/exdb/mnist/) \u6570\u636e\u96c6\u4e2d\u7684\u624b\u5199\u6570\u5b57\u8fdb\u884c\u5206\u7c7b\n", " * \u6bd4\u8f83\u7ebf\u6027\u5206\u7c7b\u6a21\u578b\u548c\u795e\u7ecf\u7f51\u7edc\u5206\u7c7b\u6a21\u578b\u7684\u6548\u679c\n", " * \u53ef\u89c6\u5316\u795e\u7ecf\u7f51\u7edc\u9690\u85cf\u5c42\u7684\u6743\u91cd" ] }, { "metadata": { "id": "HSEh-gNdu8T0", "colab_type": "text" }, "cell_type": "markdown", "source": [ " \u6211\u4eec\u7684\u76ee\u6807\u662f\u5c06\u6bcf\u4e2a\u8f93\u5165\u56fe\u7247\u4e0e\u6b63\u786e\u7684\u6570\u5b57\u76f8\u5bf9\u5e94\u3002\u6211\u4eec\u4f1a\u521b\u5efa\u4e00\u4e2a\u5305\u542b\u51e0\u4e2a\u9690\u85cf\u5c42\u7684\u795e\u7ecf\u7f51\u7edc\uff0c\u5e76\u5728\u9876\u90e8\u653e\u7f6e\u4e00\u4e2a\u5f52\u4e00\u5316\u6307\u6570\u5c42\uff0c\u4ee5\u9009\u51fa\u6700\u5408\u9002\u7684\u7c7b\u522b\u3002" ] }, { "metadata": { "id": "2NMdE1b-7UIH", "colab_type": "text" }, "cell_type": "markdown", "source": [ " ## \u8bbe\u7f6e\n", "\n", "\u9996\u5148\uff0c\u6211\u4eec\u4e0b\u8f7d\u6570\u636e\u96c6\u3001\u5bfc\u5165 TensorFlow \u548c\u5176\u4ed6\u5b9e\u7528\u5de5\u5177\uff0c\u5e76\u5c06\u6570\u636e\u52a0\u8f7d\u5230 *Pandas* `DataFrame`\u3002\u8bf7\u6ce8\u610f\uff0c\u6b64\u6570\u636e\u662f\u539f\u59cb MNIST \u8bad\u7ec3\u6570\u636e\u7684\u6837\u672c\uff1b\u6211\u4eec\u968f\u673a\u9009\u62e9\u4e86 20000 \u884c\u3002" ] }, { "metadata": { "id": "4LJ4SD8BWHeh", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 }, "test": { "output": "ignore", "timeout": 600 } }, "cellView": "both" }, "source": [ "from __future__ import print_function\n", "\n", "import glob\n", "import math\n", "import os\n", "\n", "from IPython import display\n", "from matplotlib import cm\n", "from matplotlib import gridspec\n", "from matplotlib import pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", "import seaborn as sns\n", "from sklearn import metrics\n", "import tensorflow as tf\n", "from tensorflow.python.data import Dataset\n", "\n", "tf.logging.set_verbosity(tf.logging.ERROR)\n", "pd.options.display.max_rows = 10\n", "pd.options.display.float_format = '{:.1f}'.format\n", "\n", "mnist_dataframe = pd.read_csv(\n", " \"https://storage.googleapis.com/mledu-datasets/mnist_train_small.csv\",\n", " sep=\",\",\n", " header=None)\n", "\n", "# Use just the first 10,000 records for training/validation\n", "mnist_dataframe = mnist_dataframe.head(10000)\n", "\n", "mnist_dataframe = mnist_dataframe.reindex(np.random.permutation(mnist_dataframe.index))\n", "mnist_dataframe.head()" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "kg0-25p2mOi0", "colab_type": "text" }, "cell_type": "markdown", "source": [ " \u7b2c\u4e00\u5217\u4e2d\u5305\u542b\u7c7b\u522b\u6807\u7b7e\u3002\u5176\u4f59\u5217\u4e2d\u5305\u542b\u7279\u5f81\u503c\uff0c\u6bcf\u4e2a\u50cf\u7d20\u5bf9\u5e94\u4e00\u4e2a\u7279\u5f81\u503c\uff0c\u6709 `28\u00d728=784` \u4e2a\u50cf\u7d20\u503c\uff0c\u5176\u4e2d\u5927\u90e8\u5206\u50cf\u7d20\u503c\u90fd\u4e3a\u96f6\uff1b\u60a8\u4e5f\u8bb8\u9700\u8981\u82b1\u4e00\u5206\u949f\u65f6\u95f4\u6765\u786e\u8ba4\u5b83\u4eec\u4e0d*\u5168\u90e8*\u4e3a\u96f6\u3002" ] }, { "metadata": { "id": "PQ7vuOwRCsZ1", "colab_type": "text" }, "cell_type": "markdown", "source": [ " ![img](https://www.tensorflow.org/versions/r0.11/images/MNIST-Matrix.png)" ] }, { "metadata": { "id": "dghlqJPIu8UM", "colab_type": "text" }, "cell_type": "markdown", "source": [ " \u8fd9\u4e9b\u6837\u672c\u90fd\u662f\u5206\u8fa8\u7387\u76f8\u5bf9\u8f83\u4f4e\u3001\u5bf9\u6bd4\u5ea6\u76f8\u5bf9\u8f83\u9ad8\u7684\u624b\u5199\u6570\u5b57\u56fe\u7247\u3002`0-9` \u8fd9\u5341\u4e2a\u6570\u5b57\u4e2d\u7684\u6bcf\u4e2a\u53ef\u80fd\u51fa\u73b0\u7684\u6570\u5b57\u5747\u7531\u552f\u4e00\u7684\u7c7b\u522b\u6807\u7b7e\u8868\u793a\u3002\u56e0\u6b64\uff0c\u8fd9\u662f\u4e00\u4e2a\u5177\u6709 10 \u4e2a\u7c7b\u522b\u7684\u591a\u7c7b\u522b\u5206\u7c7b\u95ee\u9898\u3002\n", "\n", "\u73b0\u5728\uff0c\u6211\u4eec\u89e3\u6790\u4e00\u4e0b\u6807\u7b7e\u548c\u7279\u5f81\uff0c\u5e76\u67e5\u770b\u51e0\u4e2a\u6837\u672c\u3002\u6ce8\u610f `loc` \u7684\u4f7f\u7528\uff0c\u501f\u52a9 `loc`\uff0c\u6211\u4eec\u80fd\u591f\u57fa\u4e8e\u539f\u6765\u7684\u4f4d\u7f6e\u62bd\u51fa\u5404\u5217\uff0c\u56e0\u4e3a\u6b64\u6570\u636e\u96c6\u4e2d\u6ca1\u6709\u6807\u9898\u884c\u3002" ] }, { "metadata": { "id": "JfFWWvMWDFrR", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 }, "test": { "output": "ignore", "timeout": 600 } } }, "source": [ "def parse_labels_and_features(dataset):\n", " \"\"\"Extracts labels and features.\n", " \n", " This is a good place to scale or transform the features if needed.\n", " \n", " Args:\n", " dataset: A Pandas `Dataframe`, containing the label on the first column and\n", " monochrome pixel values on the remaining columns, in row major order.\n", " Returns:\n", " A `tuple` `(labels, features)`:\n", " labels: A Pandas `Series`.\n", " features: A Pandas `DataFrame`.\n", " \"\"\"\n", " labels = dataset[0]\n", "\n", " # DataFrame.loc index ranges are inclusive at both ends.\n", " features = dataset.loc[:,1:784]\n", " # Scale the data to [0, 1] by dividing out the max value, 255.\n", " features = features / 255\n", "\n", " return labels, features" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "mFY_-7vZu8UU", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "training_targets, training_examples = parse_labels_and_features(mnist_dataframe[:7500])\n", "training_examples.describe()" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "4-Vgg-1zu8Ud", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "validation_targets, validation_examples = parse_labels_and_features(mnist_dataframe[7500:10000])\n", "validation_examples.describe()" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "wrnAI1v6u8Uh", "colab_type": "text" }, "cell_type": "markdown", "source": [ " \u663e\u793a\u4e00\u4e2a\u968f\u673a\u6837\u672c\u53ca\u5176\u5bf9\u5e94\u7684\u6807\u7b7e\u3002" ] }, { "metadata": { "id": "s-euVJVtu8Ui", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "rand_example = np.random.choice(training_examples.index)\n", "_, ax = plt.subplots()\n", "ax.matshow(training_examples.loc[rand_example].values.reshape(28, 28))\n", "ax.set_title(\"Label: %i\" % training_targets.loc[rand_example])\n", "ax.grid(False)" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "ScmYX7xdZMXE", "colab_type": "text" }, "cell_type": "markdown", "source": [ " ## \u4efb\u52a1 1\uff1a\u4e3a MNIST \u6784\u5efa\u7ebf\u6027\u6a21\u578b\n", "\n", "\u9996\u5148\uff0c\u6211\u4eec\u521b\u5efa\u4e00\u4e2a\u57fa\u51c6\u6a21\u578b\uff0c\u4f5c\u4e3a\u6bd4\u8f83\u5bf9\u8c61\u3002`LinearClassifier` \u53ef\u63d0\u4f9b\u4e00\u7ec4 *k* \u7c7b\u4e00\u5bf9\u591a\u5206\u7c7b\u5668\uff0c\u6bcf\u4e2a\u7c7b\u522b\uff08\u5171 *k* \u4e2a\uff09\u5bf9\u5e94\u4e00\u4e2a\u5206\u7c7b\u5668\u3002\n", "\n", "\u60a8\u4f1a\u53d1\u73b0\uff0c\u9664\u4e86\u62a5\u544a\u51c6\u786e\u7387\u548c\u7ed8\u5236\u5bf9\u6570\u635f\u5931\u51fd\u6570\u968f\u65f6\u95f4\u53d8\u5316\u60c5\u51b5\u7684\u66f2\u7ebf\u56fe\u4e4b\u5916\uff0c\u6211\u4eec\u8fd8\u5c55\u793a\u4e86\u4e00\u4e2a[**\u6df7\u6dc6\u77e9\u9635**](https://en.wikipedia.org/wiki/Confusion_matrix)\u3002\u6df7\u6dc6\u77e9\u9635\u4f1a\u663e\u793a\u9519\u8bef\u5206\u7c7b\u4e3a\u5176\u4ed6\u7c7b\u522b\u7684\u7c7b\u522b\u3002\u54ea\u4e9b\u6570\u5b57\u76f8\u4e92\u4e4b\u95f4\u5bb9\u6613\u6df7\u6dc6\uff1f\n", "\n", "\u53e6\u8bf7\u6ce8\u610f\uff0c\u6211\u4eec\u4f1a\u4f7f\u7528 `log_loss` \u51fd\u6570\u8ddf\u8e2a\u6a21\u578b\u7684\u9519\u8bef\u3002\u4e0d\u5e94\u5c06\u6b64\u51fd\u6570\u4e0e\u7528\u4e8e\u8bad\u7ec3\u7684 `LinearClassifier` \u5185\u90e8\u635f\u5931\u51fd\u6570\u76f8\u6df7\u6dc6\u3002" ] }, { "metadata": { "id": "cpoVC4TSdw5Z", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "def construct_feature_columns():\n", " \"\"\"Construct the TensorFlow Feature Columns.\n", "\n", " Returns:\n", " A set of feature columns\n", " \"\"\" \n", " \n", " # There are 784 pixels in each image \n", " return set([tf.feature_column.numeric_column('pixels', shape=784)])" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "kMmL89yGeTfz", "colab_type": "text" }, "cell_type": "markdown", "source": [ " \u5728\u672c\u6b21\u7ec3\u4e60\u4e2d\uff0c\u6211\u4eec\u4f1a\u5bf9\u8bad\u7ec3\u548c\u9884\u6d4b\u4f7f\u7528\u5355\u72ec\u7684\u8f93\u5165\u51fd\u6570\uff0c\u5e76\u5c06\u8fd9\u4e9b\u51fd\u6570\u5206\u522b\u5d4c\u5957\u5728 `create_training_input_fn()` \u548c `create_predict_input_fn()` \u4e2d\uff0c\u8fd9\u6837\u4e00\u6765\uff0c\u6211\u4eec\u5c31\u53ef\u4ee5\u8c03\u7528\u8fd9\u4e9b\u51fd\u6570\uff0c\u4ee5\u8fd4\u56de\u76f8\u5e94\u7684 `_input_fn`\uff0c\u5e76\u5c06\u5176\u4f20\u9012\u5230 `.train()` \u548c `.predict()` \u8c03\u7528\u3002" ] }, { "metadata": { "id": "OeS47Bmn5Ms2", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "def create_training_input_fn(features, labels, batch_size, num_epochs=None, shuffle=True):\n", " \"\"\"A custom input_fn for sending MNIST data to the estimator for training.\n", "\n", " Args:\n", " features: The training features.\n", " labels: The training labels.\n", " batch_size: Batch size to use during training.\n", "\n", " Returns:\n", " A function that returns batches of training features and labels during\n", " training.\n", " \"\"\"\n", " def _input_fn(num_epochs=None, shuffle=True):\n", " # Input pipelines are reset with each call to .train(). To ensure model\n", " # gets a good sampling of data, even when steps is small, we \n", " # shuffle all the data before creating the Dataset object\n", " idx = np.random.permutation(features.index)\n", " raw_features = {\"pixels\":features.reindex(idx)}\n", " raw_targets = np.array(labels[idx])\n", " \n", " ds = Dataset.from_tensor_slices((raw_features,raw_targets)) # warning: 2GB limit\n", " ds = ds.batch(batch_size).repeat(num_epochs)\n", " \n", " if shuffle:\n", " ds = ds.shuffle(10000)\n", " \n", " # Return the next batch of data\n", " feature_batch, label_batch = ds.make_one_shot_iterator().get_next()\n", " return feature_batch, label_batch\n", "\n", " return _input_fn" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "8zoGWAoohrwS", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "def create_predict_input_fn(features, labels, batch_size):\n", " \"\"\"A custom input_fn for sending mnist data to the estimator for predictions.\n", "\n", " Args:\n", " features: The features to base predictions on.\n", " labels: The labels of the prediction examples.\n", "\n", " Returns:\n", " A function that returns features and labels for predictions.\n", " \"\"\"\n", " def _input_fn():\n", " raw_features = {\"pixels\": features.values}\n", " raw_targets = np.array(labels)\n", " \n", " ds = Dataset.from_tensor_slices((raw_features, raw_targets)) # warning: 2GB limit\n", " ds = ds.batch(batch_size)\n", " \n", " \n", " # Return the next batch of data\n", " feature_batch, label_batch = ds.make_one_shot_iterator().get_next()\n", " return feature_batch, label_batch\n", "\n", " return _input_fn" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "G6DjSLZMu8Um", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "def train_linear_classification_model(\n", " learning_rate,\n", " steps,\n", " batch_size,\n", " training_examples,\n", " training_targets,\n", " validation_examples,\n", " validation_targets):\n", " \"\"\"Trains a linear classification model for the MNIST digits dataset.\n", " \n", " In addition to training, this function also prints training progress information,\n", " a plot of the training and validation loss over time, and a confusion\n", " matrix.\n", " \n", " Args:\n", " learning_rate: An `int`, the learning rate to use.\n", " steps: A non-zero `int`, the total number of training steps. A training step\n", " consists of a forward and backward pass using a single batch.\n", " batch_size: A non-zero `int`, the batch size.\n", " training_examples: A `DataFrame` containing the training features.\n", " training_targets: A `DataFrame` containing the training labels.\n", " validation_examples: A `DataFrame` containing the validation features.\n", " validation_targets: A `DataFrame` containing the validation labels.\n", " \n", " Returns:\n", " The trained `LinearClassifier` object.\n", " \"\"\"\n", "\n", " periods = 10\n", "\n", " steps_per_period = steps / periods \n", " # Create the input functions.\n", " predict_training_input_fn = create_predict_input_fn(\n", " training_examples, training_targets, batch_size)\n", " predict_validation_input_fn = create_predict_input_fn(\n", " validation_examples, validation_targets, batch_size)\n", " training_input_fn = create_training_input_fn(\n", " training_examples, training_targets, batch_size)\n", " \n", " # Create a LinearClassifier object.\n", " my_optimizer = tf.train.AdagradOptimizer(learning_rate=learning_rate)\n", " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", " classifier = tf.estimator.LinearClassifier(\n", " feature_columns=construct_feature_columns(),\n", " n_classes=10,\n", " optimizer=my_optimizer,\n", " config=tf.estimator.RunConfig(keep_checkpoint_max=1)\n", " )\n", "\n", " # Train the model, but do so inside a loop so that we can periodically assess\n", " # loss metrics.\n", " print(\"Training model...\")\n", " print(\"LogLoss error (on validation data):\")\n", " training_errors = []\n", " validation_errors = []\n", " for period in range (0, periods):\n", " # Train the model, starting from the prior state.\n", " classifier.train(\n", " input_fn=training_input_fn,\n", " steps=steps_per_period\n", " )\n", " \n", " # Take a break and compute probabilities.\n", " training_predictions = list(classifier.predict(input_fn=predict_training_input_fn))\n", " training_probabilities = np.array([item['probabilities'] for item in training_predictions])\n", " training_pred_class_id = np.array([item['class_ids'][0] for item in training_predictions])\n", " training_pred_one_hot = tf.keras.utils.to_categorical(training_pred_class_id,10)\n", " \n", " validation_predictions = list(classifier.predict(input_fn=predict_validation_input_fn))\n", " validation_probabilities = np.array([item['probabilities'] for item in validation_predictions]) \n", " validation_pred_class_id = np.array([item['class_ids'][0] for item in validation_predictions])\n", " validation_pred_one_hot = tf.keras.utils.to_categorical(validation_pred_class_id,10) \n", " \n", " # Compute training and validation errors.\n", " training_log_loss = metrics.log_loss(training_targets, training_pred_one_hot)\n", " validation_log_loss = metrics.log_loss(validation_targets, validation_pred_one_hot)\n", " # Occasionally print the current loss.\n", " print(\" period %02d : %0.2f\" % (period, validation_log_loss))\n", " # Add the loss metrics from this period to our list.\n", " training_errors.append(training_log_loss)\n", " validation_errors.append(validation_log_loss)\n", " print(\"Model training finished.\")\n", " # Remove event files to save disk space.\n", " _ = map(os.remove, glob.glob(os.path.join(classifier.model_dir, 'events.out.tfevents*')))\n", " \n", " # Calculate final predictions (not probabilities, as above).\n", " final_predictions = classifier.predict(input_fn=predict_validation_input_fn)\n", " final_predictions = np.array([item['class_ids'][0] for item in final_predictions])\n", " \n", " \n", " accuracy = metrics.accuracy_score(validation_targets, final_predictions)\n", " print(\"Final accuracy (on validation data): %0.2f\" % accuracy)\n", "\n", " # Output a graph of loss metrics over periods.\n", " plt.ylabel(\"LogLoss\")\n", " plt.xlabel(\"Periods\")\n", " plt.title(\"LogLoss vs. Periods\")\n", " plt.plot(training_errors, label=\"training\")\n", " plt.plot(validation_errors, label=\"validation\")\n", " plt.legend()\n", " plt.show()\n", " \n", " # Output a plot of the confusion matrix.\n", " cm = metrics.confusion_matrix(validation_targets, final_predictions)\n", " # Normalize the confusion matrix by row (i.e by the number of samples\n", " # in each class)\n", " cm_normalized = cm.astype(\"float\") / cm.sum(axis=1)[:, np.newaxis]\n", " ax = sns.heatmap(cm_normalized, cmap=\"bone_r\")\n", " ax.set_aspect(1)\n", " plt.title(\"Confusion matrix\")\n", " plt.ylabel(\"True label\")\n", " plt.xlabel(\"Predicted label\")\n", " plt.show()\n", "\n", " return classifier" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "ItHIUyv2u8Ur", "colab_type": "text" }, "cell_type": "markdown", "source": [ " **\u82b1\u8d39 5 \u5206\u949f\u7684\u65f6\u95f4\u4e86\u89e3\u4e00\u4e0b\u4f7f\u7528\u8fd9\u79cd\u5f62\u5f0f\u7684\u7ebf\u6027\u6a21\u578b\u65f6\uff0c\u51c6\u786e\u7387\u65b9\u9762\u8868\u73b0\u5982\u4f55\u3002\u5728\u672c\u6b21\u7ec3\u4e60\u4e2d\uff0c\u4e3a\u81ea\u5df1\u8bbe\u5b9a\u9650\u5236\uff0c\u4ec5\u4f7f\u7528\u6279\u91cf\u5927\u5c0f\u3001\u5b66\u4e60\u901f\u7387\u548c\u6b65\u6570\u8fd9\u4e09\u4e2a\u8d85\u53c2\u6570\u8fdb\u884c\u8bd5\u9a8c\u3002**\n", "\n", "\u5982\u679c\u60a8\u4ece\u4e0a\u8ff0\u4efb\u4f55\u8bd5\u9a8c\u4e2d\u5f97\u5230\u7684\u51c6\u786e\u7387\u7ea6\u4e3a 0.9\uff0c\u5373\u53ef\u505c\u6b62\u8bd5\u9a8c\u3002" ] }, { "metadata": { "id": "yaiIhIQqu8Uv", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "classifier = train_linear_classification_model(\n", " learning_rate=0.02,\n", " steps=100,\n", " batch_size=10,\n", " training_examples=training_examples,\n", " training_targets=training_targets,\n", " validation_examples=validation_examples,\n", " validation_targets=validation_targets)" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "266KQvZoMxMv", "colab_type": "text" }, "cell_type": "markdown", "source": [ " ### \u89e3\u51b3\u65b9\u6848\n", "\n", "\u70b9\u51fb\u4e0b\u65b9\u5373\u53ef\u67e5\u770b\u4e00\u79cd\u53ef\u80fd\u7684\u89e3\u51b3\u65b9\u6848\u3002" ] }, { "metadata": { "id": "lRWcn24DM3qa", "colab_type": "text" }, "cell_type": "markdown", "source": [ " \u4ee5\u4e0b\u662f\u4e00\u7ec4\u4f7f\u51c6\u786e\u7387\u5e94\u8be5\u7ea6\u4e3a 0.9 \u7684\u53c2\u6570\u3002" ] }, { "metadata": { "id": "TGlBMrUoM1K_", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "_ = train_linear_classification_model(\n", " learning_rate=0.03,\n", " steps=1000,\n", " batch_size=30,\n", " training_examples=training_examples,\n", " training_targets=training_targets,\n", " validation_examples=validation_examples,\n", " validation_targets=validation_targets)" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "mk095OfpPdOx", "colab_type": "text" }, "cell_type": "markdown", "source": [ " ## \u4efb\u52a1 2\uff1a\u4f7f\u7528\u795e\u7ecf\u7f51\u7edc\u66ff\u6362\u7ebf\u6027\u5206\u7c7b\u5668\n", "\n", "**\u4f7f\u7528 [`DNNClassifier`](https://www.tensorflow.org/api_docs/python/tf/contrib/learn/DNNClassifier) \u66ff\u6362\u4e0a\u9762\u7684 LinearClassifier\uff0c\u5e76\u67e5\u627e\u53ef\u5b9e\u73b0 0.95 \u6216\u66f4\u9ad8\u51c6\u786e\u7387\u7684\u53c2\u6570\u7ec4\u5408\u3002**\n", "\n", "\u60a8\u53ef\u80fd\u5e0c\u671b\u5c1d\u8bd5 Dropout \u7b49\u5176\u4ed6\u6b63\u5219\u5316\u65b9\u6cd5\u3002\u8fd9\u4e9b\u989d\u5916\u7684\u6b63\u5219\u5316\u65b9\u6cd5\u5df2\u8bb0\u5f55\u5728 `DNNClassifier` \u7c7b\u7684\u6ce8\u91ca\u4e2d\u3002" ] }, { "metadata": { "id": "rm8P_Ttwu8U4", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "#\n", "# YOUR CODE HERE: Replace the linear classifier with a neural network.\n", "#" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "TOfmiSvqu8U9", "colab_type": "text" }, "cell_type": "markdown", "source": [ " \u83b7\u5f97\u51fa\u8272\u7684\u6a21\u578b\u540e\uff0c\u901a\u8fc7\u8bc4\u4f30\u6211\u4eec\u5c06\u5728\u4e0b\u9762\u52a0\u8f7d\u7684\u6d4b\u8bd5\u6570\u636e\u8fdb\u884c\u4ed4\u7ec6\u68c0\u67e5\uff0c\u786e\u8ba4\u60a8\u6ca1\u6709\u8fc7\u62df\u5408\u9a8c\u8bc1\u96c6\u3002\n", "" ] }, { "metadata": { "id": "evlB5ubzu8VJ", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "mnist_test_dataframe = pd.read_csv(\n", " \"https://storage.googleapis.com/mledu-datasets/mnist_test.csv\",\n", " sep=\",\",\n", " header=None)\n", "\n", "test_targets, test_examples = parse_labels_and_features(mnist_test_dataframe)\n", "test_examples.describe()" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "PDuLd2Hcu8VL", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "#\n", "# YOUR CODE HERE: Calculate accuracy on the test set.\n", "#" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "6sfw3LH0Oycm", "colab_type": "text" }, "cell_type": "markdown", "source": [ " ### \u89e3\u51b3\u65b9\u6848\n", "\n", "\u70b9\u51fb\u4e0b\u65b9\u5373\u53ef\u67e5\u770b\u53ef\u80fd\u7684\u89e3\u51b3\u65b9\u6848\u3002" ] }, { "metadata": { "id": "XatDGFKEO374", "colab_type": "text" }, "cell_type": "markdown", "source": [ " \u9664\u4e86\u795e\u7ecf\u7f51\u7edc\u4e13\u7528\u914d\u7f6e\uff08\u4f8b\u5982\u9690\u85cf\u5355\u5143\u7684\u8d85\u53c2\u6570\uff09\u4e4b\u5916\uff0c\u4ee5\u4e0b\u4ee3\u7801\u4e0e\u539f\u59cb\u7684 `LinearClassifer` \u8bad\u7ec3\u4ee3\u7801\u51e0\u4e4e\u5b8c\u5168\u76f8\u540c\u3002" ] }, { "metadata": { "id": "kdNTx8jkPQUx", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "def train_nn_classification_model(\n", " learning_rate,\n", " steps,\n", " batch_size,\n", " hidden_units,\n", " training_examples,\n", " training_targets,\n", " validation_examples,\n", " validation_targets):\n", " \"\"\"Trains a neural network classification model for the MNIST digits dataset.\n", " \n", " In addition to training, this function also prints training progress information,\n", " a plot of the training and validation loss over time, as well as a confusion\n", " matrix.\n", " \n", " Args:\n", " learning_rate: An `int`, the learning rate to use.\n", " steps: A non-zero `int`, the total number of training steps. A training step\n", " consists of a forward and backward pass using a single batch.\n", " batch_size: A non-zero `int`, the batch size.\n", " hidden_units: A `list` of int values, specifying the number of neurons in each layer.\n", " training_examples: A `DataFrame` containing the training features.\n", " training_targets: A `DataFrame` containing the training labels.\n", " validation_examples: A `DataFrame` containing the validation features.\n", " validation_targets: A `DataFrame` containing the validation labels.\n", " \n", " Returns:\n", " The trained `DNNClassifier` object.\n", " \"\"\"\n", "\n", " periods = 10\n", " # Caution: input pipelines are reset with each call to train. \n", " # If the number of steps is small, your model may never see most of the data. \n", " # So with multiple `.train` calls like this you may want to control the length \n", " # of training with num_epochs passed to the input_fn. Or, you can do a really-big shuffle, \n", " # or since it's in-memory data, shuffle all the data in the `input_fn`.\n", " steps_per_period = steps / periods \n", " # Create the input functions.\n", " predict_training_input_fn = create_predict_input_fn(\n", " training_examples, training_targets, batch_size)\n", " predict_validation_input_fn = create_predict_input_fn(\n", " validation_examples, validation_targets, batch_size)\n", " training_input_fn = create_training_input_fn(\n", " training_examples, training_targets, batch_size)\n", " \n", " # Create the input functions.\n", " predict_training_input_fn = create_predict_input_fn(\n", " training_examples, training_targets, batch_size)\n", " predict_validation_input_fn = create_predict_input_fn(\n", " validation_examples, validation_targets, batch_size)\n", " training_input_fn = create_training_input_fn(\n", " training_examples, training_targets, batch_size)\n", " \n", " # Create feature columns.\n", " feature_columns = [tf.feature_column.numeric_column('pixels', shape=784)]\n", "\n", " # Create a DNNClassifier object.\n", " my_optimizer = tf.train.AdagradOptimizer(learning_rate=learning_rate)\n", " my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)\n", " classifier = tf.estimator.DNNClassifier(\n", " feature_columns=feature_columns,\n", " n_classes=10,\n", " hidden_units=hidden_units,\n", " optimizer=my_optimizer,\n", " config=tf.contrib.learn.RunConfig(keep_checkpoint_max=1)\n", " )\n", "\n", " # Train the model, but do so inside a loop so that we can periodically assess\n", " # loss metrics.\n", " print(\"Training model...\")\n", " print(\"LogLoss error (on validation data):\")\n", " training_errors = []\n", " validation_errors = []\n", " for period in range (0, periods):\n", " # Train the model, starting from the prior state.\n", " classifier.train(\n", " input_fn=training_input_fn,\n", " steps=steps_per_period\n", " )\n", " \n", " # Take a break and compute probabilities.\n", " training_predictions = list(classifier.predict(input_fn=predict_training_input_fn))\n", " training_probabilities = np.array([item['probabilities'] for item in training_predictions])\n", " training_pred_class_id = np.array([item['class_ids'][0] for item in training_predictions])\n", " training_pred_one_hot = tf.keras.utils.to_categorical(training_pred_class_id,10)\n", " \n", " validation_predictions = list(classifier.predict(input_fn=predict_validation_input_fn))\n", " validation_probabilities = np.array([item['probabilities'] for item in validation_predictions]) \n", " validation_pred_class_id = np.array([item['class_ids'][0] for item in validation_predictions])\n", " validation_pred_one_hot = tf.keras.utils.to_categorical(validation_pred_class_id,10) \n", " \n", " # Compute training and validation errors.\n", " training_log_loss = metrics.log_loss(training_targets, training_pred_one_hot)\n", " validation_log_loss = metrics.log_loss(validation_targets, validation_pred_one_hot)\n", " # Occasionally print the current loss.\n", " print(\" period %02d : %0.2f\" % (period, validation_log_loss))\n", " # Add the loss metrics from this period to our list.\n", " training_errors.append(training_log_loss)\n", " validation_errors.append(validation_log_loss)\n", " print(\"Model training finished.\")\n", " # Remove event files to save disk space.\n", " _ = map(os.remove, glob.glob(os.path.join(classifier.model_dir, 'events.out.tfevents*')))\n", " \n", " # Calculate final predictions (not probabilities, as above).\n", " final_predictions = classifier.predict(input_fn=predict_validation_input_fn)\n", " final_predictions = np.array([item['class_ids'][0] for item in final_predictions])\n", " \n", " \n", " accuracy = metrics.accuracy_score(validation_targets, final_predictions)\n", " print(\"Final accuracy (on validation data): %0.2f\" % accuracy)\n", "\n", " # Output a graph of loss metrics over periods.\n", " plt.ylabel(\"LogLoss\")\n", " plt.xlabel(\"Periods\")\n", " plt.title(\"LogLoss vs. Periods\")\n", " plt.plot(training_errors, label=\"training\")\n", " plt.plot(validation_errors, label=\"validation\")\n", " plt.legend()\n", " plt.show()\n", " \n", " # Output a plot of the confusion matrix.\n", " cm = metrics.confusion_matrix(validation_targets, final_predictions)\n", " # Normalize the confusion matrix by row (i.e by the number of samples\n", " # in each class)\n", " cm_normalized = cm.astype(\"float\") / cm.sum(axis=1)[:, np.newaxis]\n", " ax = sns.heatmap(cm_normalized, cmap=\"bone_r\")\n", " ax.set_aspect(1)\n", " plt.title(\"Confusion matrix\")\n", " plt.ylabel(\"True label\")\n", " plt.xlabel(\"Predicted label\")\n", " plt.show()\n", "\n", " return classifier" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "ZfzsTYGPPU8I", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "classifier = train_nn_classification_model(\n", " learning_rate=0.05,\n", " steps=1000,\n", " batch_size=30,\n", " hidden_units=[100, 100],\n", " training_examples=training_examples,\n", " training_targets=training_targets,\n", " validation_examples=validation_examples,\n", " validation_targets=validation_targets)" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "qXvrOgtUR-zD", "colab_type": "text" }, "cell_type": "markdown", "source": [ " \u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u6765\u9a8c\u8bc1\u6d4b\u8bd5\u96c6\u7684\u51c6\u786e\u7387\u3002" ] }, { "metadata": { "id": "scQNpDePSFjt", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "mnist_test_dataframe = pd.read_csv(\n", " \"https://storage.googleapis.com/mledu-datasets/mnist_test.csv\",\n", " sep=\",\",\n", " header=None)\n", "\n", "test_targets, test_examples = parse_labels_and_features(mnist_test_dataframe)\n", "test_examples.describe()" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "EVaWpWKvSHmu", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 } } }, "source": [ "predict_test_input_fn = create_predict_input_fn(\n", " test_examples, test_targets, batch_size=100)\n", "\n", "test_predictions = classifier.predict(input_fn=predict_test_input_fn)\n", "test_predictions = np.array([item['class_ids'][0] for item in test_predictions])\n", " \n", "accuracy = metrics.accuracy_score(test_targets, test_predictions)\n", "print(\"Accuracy on test data: %0.2f\" % accuracy)" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "WX2mQBAEcisO", "colab_type": "text" }, "cell_type": "markdown", "source": [ " ## \u4efb\u52a1 3\uff1a\u53ef\u89c6\u5316\u7b2c\u4e00\u4e2a\u9690\u85cf\u5c42\u7684\u6743\u91cd\u3002\n", "\n", "\u6211\u4eec\u6765\u82b1\u51e0\u5206\u949f\u65f6\u95f4\u770b\u770b\u6a21\u578b\u7684 `weights_` \u5c5e\u6027\uff0c\u4ee5\u6df1\u5165\u63a2\u7d22\u6211\u4eec\u7684\u795e\u7ecf\u7f51\u7edc\uff0c\u5e76\u4e86\u89e3\u5b83\u5b66\u5230\u4e86\u54ea\u4e9b\u89c4\u5f8b\u3002\n", "\n", "\u6a21\u578b\u7684\u8f93\u5165\u5c42\u6709 `784` \u4e2a\u6743\u91cd\uff0c\u5bf9\u5e94\u4e8e `28\u00d728` \u50cf\u7d20\u8f93\u5165\u56fe\u7247\u3002\u7b2c\u4e00\u4e2a\u9690\u85cf\u5c42\u5c06\u6709 `784\u00d7N` \u4e2a\u6743\u91cd\uff0c\u5176\u4e2d `N` \u6307\u7684\u662f\u8be5\u5c42\u4e2d\u7684\u8282\u70b9\u6570\u3002\u6211\u4eec\u53ef\u4ee5\u5c06\u8fd9\u4e9b\u6743\u91cd\u91cd\u65b0\u53d8\u56de `28\u00d728` \u50cf\u7d20\u7684\u56fe\u7247\uff0c\u5177\u4f53\u65b9\u6cd5\u662f\u5c06 `N` \u4e2a `1\u00d7784` \u6743\u91cd\u6570\u7ec4*\u53d8\u5f62*\u4e3a `N` \u4e2a `28\u00d728` \u5927\u5c0f\u6570\u7ec4\u3002\n", "\n", "\u8fd0\u884c\u4ee5\u4e0b\u5355\u5143\u683c\uff0c\u7ed8\u5236\u6743\u91cd\u66f2\u7ebf\u56fe\u3002\u8bf7\u6ce8\u610f\uff0c\u6b64\u5355\u5143\u683c\u8981\u6c42\u540d\u4e3a \"classifier\" \u7684 `DNNClassifier` \u5df2\u7ecf\u8fc7\u8bad\u7ec3\u3002" ] }, { "metadata": { "id": "eUC0Z8nbafgG", "colab_type": "code", "colab": { "autoexec": { "startup": false, "wait_interval": 0 }, "test": { "output": "ignore", "timeout": 600 } }, "cellView": "both" }, "source": [ "print(classifier.get_variable_names())\n", "\n", "weights0 = classifier.get_variable_value(\"dnn/hiddenlayer_0/kernel\")\n", "\n", "print(\"weights0 shape:\", weights0.shape)\n", "\n", "num_nodes = weights0.shape[1]\n", "num_rows = int(math.ceil(num_nodes / 10.0))\n", "fig, axes = plt.subplots(num_rows, 10, figsize=(20, 2 * num_rows))\n", "for coef, ax in zip(weights0.T, axes.ravel()):\n", " # Weights in coef is reshaped from 1x784 to 28x28.\n", " ax.matshow(coef.reshape(28, 28), cmap=plt.cm.pink)\n", " ax.set_xticks(())\n", " ax.set_yticks(())\n", "\n", "plt.show()" ], "cell_type": "code", "execution_count": 0, "outputs": [] }, { "metadata": { "id": "kL8MEhNgrx9N", "colab_type": "text" }, "cell_type": "markdown", "source": [ " \u795e\u7ecf\u7f51\u7edc\u7684\u7b2c\u4e00\u4e2a\u9690\u85cf\u5c42\u5e94\u8be5\u4f1a\u5bf9\u4e00\u4e9b\u7ea7\u522b\u7279\u522b\u4f4e\u7684\u7279\u5f81\u8fdb\u884c\u5efa\u6a21\uff0c\u56e0\u6b64\u53ef\u89c6\u5316\u6743\u91cd\u53ef\u80fd\u53ea\u663e\u793a\u4e00\u4e9b\u6a21\u7cca\u7684\u533a\u57df\uff0c\u4e5f\u53ef\u80fd\u53ea\u663e\u793a\u6570\u5b57\u7684\u67d0\u51e0\u4e2a\u90e8\u5206\u3002\u6b64\u5916\uff0c\u60a8\u53ef\u80fd\u8fd8\u4f1a\u770b\u5230\u4e00\u4e9b\u57fa\u672c\u4e0a\u662f\u566a\u70b9\uff08\u8fd9\u4e9b\u566a\u70b9\u8981\u4e48\u4e0d\u6536\u655b\uff0c\u8981\u4e48\u88ab\u66f4\u9ad8\u7684\u5c42\u5ffd\u7565\uff09\u7684\u795e\u7ecf\u5143\u3002\n", "\n", "\u5728\u8fed\u4ee3\u4e0d\u540c\u7684\u6b21\u6570\u540e\u505c\u6b62\u8bad\u7ec3\u5e76\u67e5\u770b\u6548\u679c\uff0c\u53ef\u80fd\u4f1a\u53d1\u73b0\u6709\u8da3\u7684\u7ed3\u679c\u3002\n", "\n", "**\u5206\u522b\u7528 10\u3001100 \u548c 1000 \u6b65\u8bad\u7ec3\u5206\u7c7b\u5668\u3002\u7136\u540e\u91cd\u65b0\u8fd0\u884c\u6b64\u53ef\u89c6\u5316\u3002**\n", "\n", "\u60a8\u770b\u5230\u4e0d\u540c\u7ea7\u522b\u7684\u6536\u655b\u4e4b\u95f4\u6709\u54ea\u4e9b\u76f4\u89c2\u4e0a\u7684\u5dee\u5f02\uff1f" ] } ] }