{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# MNIST-Neural Network-Two Hidden Layers" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 1.MNIST 데이터 로딩" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/Users/yhhan/git/aiclass/0.Professor/3.VanillaNN/MNIST_data/train-images-idx3-ubyte.gz\n", "/Users/yhhan/git/aiclass/0.Professor/3.VanillaNN/MNIST_data/train-labels-idx1-ubyte.gz\n", "/Users/yhhan/git/aiclass/0.Professor/3.VanillaNN/MNIST_data/t10k-images-idx3-ubyte.gz\n", "/Users/yhhan/git/aiclass/0.Professor/3.VanillaNN/MNIST_data/t10k-labels-idx1-ubyte.gz\n", "Converting train-images-idx3-ubyte.gz to NumPy Array ...\n", "Done\n", "Converting train-labels-idx1-ubyte.gz to NumPy Array ...\n", "Done\n", "Converting t10k-images-idx3-ubyte.gz to NumPy Array ...\n", "Done\n", "Converting t10k-labels-idx1-ubyte.gz to NumPy Array ...\n", "Done\n", "Creating pickle file ...\n", "Done!\n" ] } ], "source": [ "# coding: utf-8\n", "import urllib.request\n", "import os.path\n", "import gzip\n", "import pickle\n", "import os\n", "import numpy as np\n", "\n", "url_base = 'http://yann.lecun.com/exdb/mnist/'\n", "key_file = {\n", " 'train_img':'train-images-idx3-ubyte.gz',\n", " 'train_label':'train-labels-idx1-ubyte.gz',\n", " 'test_img':'t10k-images-idx3-ubyte.gz',\n", " 'test_label':'t10k-labels-idx1-ubyte.gz'\n", "}\n", "\n", "dataset_dir = os.path.dirname(\"/Users/yhhan/git/aiclass/0.Professor/3.VanillaNN/MNIST_data/.\")\n", "save_file = dataset_dir + \"/mnist.pkl\"\n", "\n", "train_num = 60000\n", "test_num = 10000\n", "img_dim = (1, 28, 28)\n", "img_size = 784\n", "\n", "def _download(file_name):\n", " file_path = dataset_dir + \"/\" + file_name\n", " print(file_path)\n", " if os.path.exists(file_path):\n", " return\n", "\n", " print(\"Downloading \" + file_name + \" ... \")\n", " urllib.request.urlretrieve(url_base + file_name, file_path)\n", " print(\"Done\")\n", " \n", "def download_mnist():\n", " for v in key_file.values():\n", " _download(v)\n", " \n", "def _load_label(file_name):\n", " file_path = dataset_dir + \"/\" + file_name\n", " \n", " print(\"Converting \" + file_name + \" to NumPy Array ...\")\n", " with gzip.open(file_path, 'rb') as f:\n", " labels = np.frombuffer(f.read(), np.uint8, offset=8)\n", " print(\"Done\")\n", " \n", " return labels\n", "\n", "def _load_img(file_name):\n", " file_path = dataset_dir + \"/\" + file_name\n", " \n", " print(\"Converting \" + file_name + \" to NumPy Array ...\") \n", " with gzip.open(file_path, 'rb') as f:\n", " data = np.frombuffer(f.read(), np.uint8, offset=16)\n", " data = data.reshape(-1, img_size)\n", " print(\"Done\")\n", " \n", " return data\n", " \n", "def _convert_numpy():\n", " dataset = {}\n", " dataset['train_img'] = _load_img(key_file['train_img'])\n", " dataset['train_label'] = _load_label(key_file['train_label'])\n", " dataset['test_img'] = _load_img(key_file['test_img'])\n", " dataset['test_label'] = _load_label(key_file['test_label'])\n", "\n", " dataset['validation_img'] = dataset['train_img'][55000:]\n", " dataset['validation_label'] = dataset['train_label'][55000:]\n", " dataset['train_img'] = dataset['train_img'][:55000]\n", " dataset['train_label'] = dataset['train_label'][:55000]\n", " return dataset\n", "\n", "def init_mnist():\n", " download_mnist()\n", " dataset = _convert_numpy()\n", " print(\"Creating pickle file ...\")\n", " with open(save_file, 'wb') as f:\n", " pickle.dump(dataset, f, -1)\n", " print(\"Done!\")\n", "\n", "def _change_one_hot_label(X):\n", " T = np.zeros((X.size, 10))\n", " for idx, row in enumerate(T):\n", " row[X[idx]] = 1\n", " \n", " return T\n", " \n", "\n", "def load_mnist(normalize=True, flatten=True, one_hot_label=False):\n", " if not os.path.exists(save_file):\n", " init_mnist()\n", " \n", " with open(save_file, 'rb') as f:\n", " dataset = pickle.load(f)\n", " \n", " if normalize:\n", " for key in ('train_img', 'validation_img', 'test_img'):\n", " dataset[key] = dataset[key].astype(np.float32)\n", " dataset[key] /= 255.0\n", " \n", " if one_hot_label:\n", " dataset['train_label'] = _change_one_hot_label(dataset['train_label'])\n", " dataset['validation_label'] = _change_one_hot_label(dataset['test_label'])\n", " dataset['test_label'] = _change_one_hot_label(dataset['test_label'])\n", " \n", " if not flatten:\n", " for key in ('train_img', 'validation_img', 'test_img'):\n", " dataset[key] = dataset[key].reshape(-1, 1, 28, 28)\n", "\n", " return (dataset['train_img'], dataset['train_label']), (dataset['validation_img'], dataset['validation_label']), (dataset['test_img'], dataset['test_label']) \n", "\n", "\n", "if __name__ == '__main__':\n", " init_mnist()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- Each image is 28 pixels by 28 pixels. We can interpret this as a big array of numbers:\n", "\n", "\n", "- flatten 1-D tensor of size 28x28 = 784.\n", " - Each entry in the tensor is a pixel intensity between 0 and 1, for a particular pixel in a particular image.\n", "$$[0, 0, 0, ..., 0.6, 0.7, 0.7, 0.5, ... 0.8, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.9, 0.3, ..., 0.4, 0.4, 0.4, ... 0, 0, 0]$$ " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- Number of train images is 55000.\n", "" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- A one-hot vector is a vector which is 0 in most entries, and 1 in a single entry.\n", "- In this case, the $n$th digit will be represented as a vector which is 1 in the nth entry. \n", " - For example, 3 would be $[0,0,0,1,0,0,0,0,0,0]$. \n", "" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(55000, 784)\n", "(55000,)\n", "(5000, 784)\n", "(5000,)\n", "(10000, 784)\n", "(10000,)\n" ] } ], "source": [ "(img_train, label_train), (img_validation, label_validation), (img_test, label_test) = load_mnist(flatten=True, normalize=False)\n", "print(img_train.shape)\n", "print(label_train.shape)\n", "print(img_validation.shape)\n", "print(label_validation.shape)\n", "print(img_test.shape)\n", "print(label_test.shape)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5\n", "0\n", "4\n", "1\n", "9\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABIEAAADjCAYAAAASRQSDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X2wnVV9L/DfAgqClHeMKSgBi3SUCVFepN4MwRIoV2kB\nacGMvFmHMEWQdpShxdSmg9gUwQ5Y3wAhgLkNTCMSab1oeZUqGUIKCgGMUKAJpyG8BAhSczHr/pFD\njcle+5yzzz57r5Pn85nJcPJ8z/M8v7PJNzlZ2XuvlHMOAAAAADZvW/R7AAAAAADGnkUgAAAAgAaw\nCAQAAADQABaBAAAAABrAIhAAAABAA1gEAgAAAGgAi0AAAAAADWARCAAAAKABLAIBAAAANMBWozk5\npXR0RFwWEVtGxFU55zlDfH4ezf1gvMs5p17cRzdhZHQT6qSbUCfdhDoNp5sp5856klLaMiJ+GhFH\nRsTyiLgvImbknJe2OUcpabRe/IGpmzByugl10k2ok25CnYbTzdG8HOyQiPhZzvmJnPPaiJgfEceO\n4npAd+gm1Ek3oU66CXXSTRgDo1kE2iMi/nODny8fPAb0l25CnXQT6qSbUCfdhDEwqvcEGo6U0syI\nmDnW9wFGRjehTroJddJNqJNuwsiMZhFoRUS8bYOf7zl47NfknK+IiCsivEYTekQ3oU66CXXSTaiT\nbsIYGM3Lwe6LiH1TSnunlLaOiI9ExMLujAWMgm5CnXQT6qSbUCfdhDHQ8TOBcs6vp5TOjohbY/2W\nfVfnnB/u2mRAR3QT6qSbUCfdhDrpJoyNjreI7+hmnp5Hw/ViO81O6CZNp5tQJ92EOukm1Gmst4gH\nAAAAYJywCAQAAADQABaBAAAAABrAIhAAAABAA1gEAgAAAGgAi0AAAAAADWARCAAAAKABLAIBAAAA\nNIBFIAAAAIAGsAgEAAAA0AAWgQAAAAAawCIQAAAAQANYBAIAAABoAItAAAAAAA1gEQgAAACgASwC\nAQAAADSARSAAAACABrAIBAAAANAAFoEAAAAAGmCrfg8AQNmBBx5YzM4+++xiduqppxaz6667rph9\n6UtfKmZLliwpZgAAQP08EwgAAACgASwCAQAAADSARSAAAACABrAIBAAAANAAFoEAAAAAGiDlnDs/\nOaUnI+KViPhlRLyecz5oiM/v/Gb8mi233LKY7bjjjl29V7sdiLbbbrtitt9++xWzT3ziE8Xskksu\nKWYzZswoZv/93/9dzObMmdPy+N/8zd8UzxkLOefUi/vo5vgyZcqUYnb77bcXsx122KHrs7z00kvF\nbNddd+36/Wqhm4xnRxxxRDGbN29eMZs2bVoxe+yxx0Y1U7foJjWYNWtWMWv3veQWW5T/vf3www8v\nZnfdddew5uon3YQ6Daeb3dgi/gM55+e6cB2gu3QT6qSbUCfdhDrpJnSRl4MBAAAANMBoF4FyRPxr\nSun+lNLMbgwEdIVuQp10E+qkm1An3YQuG+3LwabmnFeklN4SEd9PKT2ac757w08YLKvCQm/pJtRJ\nN6FOugl10k3oslE9EyjnvGLwv89GxE0RcUiLz7ki53zQUG/iBXSPbkKddBPqpJtQJ92E7ut4ESil\n9OaU0m++8XFEHBURD3VrMKAzugl10k2ok25CnXQTxsZoXg42ISJuSim9cZ3/k3P+v12Zahx6+9vf\nXsy23nrrYvb+97+/mE2dOrWY7bTTTsXshBNOKGa9tHz58mJ2+eWXF7Pjjz++mL3yyivF7MEHHyxm\n42GrzS7SzUodcsgm/3gVERELFiwonrPjjjsWs5zLu6C268ratWuLWbtt4A899NBitmTJko7u1zDV\nd/Owww5rebzdr4ubbrpprMZhmA4++OBidt999/VwknGr+m4y9k4//fRidv755xezdevWdXS/dn+G\n8z90E8ZAx4tAOecnIuKALs4CdIFuQp10E+qkm1An3YSxYYt4AAAAgAawCAQAAADQABaBAAAAABrA\nIhAAAABAA1gEAgAAAGiA0WwR3zhTpkwpZrfffnsxa7fF83jXblvMWbNmFbM1a9YUs3nz5hWzgYGB\nYvbiiy8Ws8cee6yYwUhtt912xey9731vMfvmN7/Z8vjEiRNHPdPGli1bVswuvvjiYjZ//vxi9m//\n9m/FrF3f//Zv/7aYUZfDDz+85fF99923eI4t4ntjiy3K/2639957F7O99tqrmA1uuwxE+6686U1v\n6uEkUIf3ve99xezkk09ueXzatGnFc9797nd3NMenP/3pYvbMM88Us6lTpxaz0vfkERGLFi0a3mDj\nmGcCAQAAADSARSAAAACABrAIBAAAANAAFoEAAAAAGsAiEAAAAEADWAQCAAAAaABbxI/A008/Xcye\nf/75YlbLFvHttrtbvXp1MfvABz5QzNauXVvMrr/++uENBuPM17/+9WI2Y8aMHk5S1m6r+u23376Y\n3XXXXcWstH14RMTkyZOHNRd1O/XUU1se/9GPftTjSdjYxIkTi9kZZ5xRzNptg/voo4+OaiYYb6ZP\nn17MzjnnnI6u2a5HxxxzTDFbuXJlR/eDbjrppJOK2WWXXVbMdtttt5bHU0rFc+68885itvvuuxez\nL3zhC8WsnXaztLvfRz7ykY7uN554JhAAAABAA1gEAgAAAGgAi0AAAAAADWARCAAAAKABLAIBAAAA\nNIBFIAAAAIAGsEX8CLzwwgvF7Lzzzitm7baH/Pd///didvnllw9vsI088MADLY8feeSRxXNeffXV\nYvbud7+7mJ177rnDHwzGkQMPPLCYfehDHypm7bajLGm3Lft3vvOdYnbJJZcUs2eeeaaYtft958UX\nXyxmv/d7v1fMOvm6qc8WW/i3oVpdddVVHZ23bNmyLk8CdZs6dWoxu+aaa4rZjjvu2NH92m1f/dRT\nT3V0TRiprbYq/7X+oIMOKmZXXnllMdtuu+2K2d13393y+IUXXlg855577ilm22yzTTG78cYbi9lR\nRx1VzNpZvHhxR+dtLny3BwAAANAAFoEAAAAAGsAiEAAAAEADWAQCAAAAaACLQAAAAAANYBEIAAAA\noAGG3CI+pXR1RBwTEc/mnPcfPLZLRNwQEZMi4smIODHnXN5XuAG+/e1vF7Pbb7+9mL3yyivF7IAD\nDihmH//4x4tZadvodtvAt/Pwww8Xs5kzZ3Z0TUZPN0dvypQpxez73/9+Mdthhx2KWc65mH33u99t\neXzGjBnFc6ZNm1bMZs2aVczabSe9atWqYvbggw8Ws3Xr1hWzD33oQ8Xsve99bzFbsmRJMRuvau/m\n5MmTi9mECRN6OAkj0en21e1+L2ua2rtJd5x22mnF7Ld+67c6uuadd95ZzK677rqOrsmv6ObonXzy\nycWs3feE7bT78+Okk05qefzll1/u6F6l60V0vg388uXLi9m1117b0TU3F8N5JtDciDh6o2N/ERG3\n5Zz3jYjbBn8O9Nbc0E2o0dzQTajR3NBNqNHc0E3omSEXgXLOd0fECxsdPjYi3lg+uzYijuvyXMAQ\ndBPqpJtQJ92EOukm9Fan7wk0Iec8MPjxf0WE549DHXQT6qSbUCfdhDrpJoyRId8TaCg555xSKr4J\nRkppZkR44xjoMd2EOukm1Ek3oU66Cd3V6TOBVqaUJkZEDP732dIn5pyvyDkflHM+qMN7AcOnm1An\n3YQ66SbUSTdhjHS6CLQwIt546/3TIuLm7owDjJJuQp10E+qkm1An3YQxMpwt4v8xIg6PiN1SSssj\n4q8jYk5E3JhS+nhEPBURJ47lkONdp1vlvfTSSx2dd8YZZ7Q8fsMNNxTPabf1M3XSzeF55zvfWczO\nO++8YtZuO+bnnnuumA0MDBSz0naUa9asKZ7zz//8zx1lvbbtttsWs0996lPF7KMf/ehYjNNXtXfz\ngx/8YDFr9/+RsTdhQvktL/bee++OrrlixYpOx9ns1N5Nhm+33XYrZn/yJ39SzNp9v7t69epi9rnP\nfW54g9ER3RyeCy+8sJhdcMEFxSzn4ivp4itf+UoxmzVrVjHr9O+3JZ/5zGe6er2IiE9+8pPFbNWq\nVV2/33gy5CJQznlGITqiy7MAI6CbUCfdhDrpJtRJN6G3On05GAAAAADjiEUgAAAAgAawCAQAAADQ\nABaBAAAAABrAIhAAAABAAwy5Oxj9M3v27GJ24IEHFrNp06a1PD59+vTiOd/73veGPRfUZptttilm\nl1xySTFrt1X2K6+8UsxOPfXUYrZ48eJi1tTtt9/+9rf3ewQ2sN9++434nIcffngMJmFj7X6/ard9\n/E9/+tNi1u73MqjdpEmTWh5fsGBB1+/1pS99qZjdcccdXb8ftPLZz362mLXbBn7t2rXF7NZbby1m\n559/fjF77bXXilnJm970pmJ21FFHFbN23yumlIrZ5z73uWJ28803F7Om80wgAAAAgAawCAQAAADQ\nABaBAAAAABrAIhAAAABAA1gEAgAAAGgAi0AAAAAADWCL+Iq9+uqrxeyMM84oZkuWLGl5/Morryye\n027ry3ZbXn/5y18uZjnnYgbd9J73vKeYtdsGvp1jjz22mN11110dXRPGq/vuu6/fI1Rnhx12KGZH\nH310MTv55JOLWbvtc9u58MILi9nq1as7uibUoNSlyZMnd3S92267rZhddtllHV0TRmqnnXYqZmed\ndVYxa/d3q3bbwB933HHDG2wEfvu3f7vl8Xnz5hXPOfDAAzu61z/90z8Vs4svvrijazadZwIBAAAA\nNIBFIAAAAIAGsAgEAAAA0AAWgQAAAAAawCIQAAAAQAPYHWycevzxx4vZ6aef3vL4NddcUzznlFNO\n6Sh785vfXMyuu+66YjYwMFDMYKS++MUvFrOUUjFrt8uXHcA2tcUW5X83WLduXQ8nodd22WWXnt7v\ngAMOKGbtOj19+vRitueeexazrbfeuuXxj370o8Vz2vXhtddeK2aLFi0qZr/4xS+K2VZblb9lu//+\n+4sZ1K7dzkVz5swZ8fXuueeeYnbaaacVs5deemnE94JOlP7MiYjYbbfdOrrmJz/5yWL2lre8pZh9\n7GMfK2Z/+Id/WMz233//lse333774jntdjdrl33zm98sZu1206bMM4EAAAAAGsAiEAAAAEADWAQC\nAAAAaACLQAAAAAANYBEIAAAAoAEsAgEAAAA0wJBbxKeUro6IYyLi2Zzz/oPHZkfEGRGxavDTLsg5\n/8tYDcnI3HTTTS2PL1u2rHhOuy22jzjiiGL2+c9/vpjttddexeyiiy4qZitWrChm/ErTunnMMccU\nsylTphSzdltOLly4cFQzNU27beDbPc4PPPDAWIxTrdq72W778tL/x6997WvFcy644IJRz7SxyZMn\nF7N2W8S//vrrxeznP/95MVu6dGnL41dffXXxnMWLFxezu+66q5itXLmymC1fvryYbbvttsXs0Ucf\nLWb8Su3d3JxNmjSpmC1YsKCr93riiSeKWbv+0T9N6+batWuL2apVq4rZ7rvvXsz+4z/+o5i1+x6t\nU88880zL4y+//HLxnIkTJxaz5557rph95zvfGf5gDMtwngk0NyKObnH873POUwZ/bBaFhHFmbugm\n1Ghu6CbUaG7oJtRobugm9MyQi0A557sj4oUezAKMgG5CnXQT6qSbUCfdhN4azXsCnZNS+nFK6eqU\n0s5dmwgYLd2EOukm1Ek3oU66CWOg00Wgr0bEPhExJSIGIuLS0iemlGamlBanlMovnge6RTehTroJ\nddJNqJNuwhjpaBEo57wy5/zLnPO6iLgyIg5p87lX5JwPyjkf1OmQwPDoJtRJN6FOugl10k0YOx0t\nAqWUNnxr7+Mj4qHujAOMhm5CnXQT6qSbUCfdhLEznC3i/zEiDo+I3VJKyyPiryPi8JTSlIjIEfFk\nRJw5hjPSJQ89VP6988QTTyxmf/AHf1DMrrnmmmJ25pnlXxb77rtvMTvyyCOLGb/StG622x556623\nLmbPPvtsMbvhhhtGNdN4tc022xSz2bNnd3TN22+/vZj95V/+ZUfXHK9q7+ZZZ51VzJ566qmWx9//\n/veP1TgtPf3008Xs29/+djF75JFHitm99947qpm6ZebMmcWs3fa/7ba9Znhq7+bm7Pzzzy9m69at\n6+q95syZ09XrMfaa1s3Vq1cXs+OOO66Y3XLLLcVsl112KWaPP/54Mbv55puL2dy5c4vZCy+0fh/v\n+fPnF89pt0V8u/PoviEXgXLOM1oc/sYYzAKMgG5CnXQT6qSbUCfdhN4aze5gAAAAAIwTFoEAAAAA\nGsAiEAAAAEADWAQCAAAAaACLQAAAAAANMOTuYDRDu60Kr7/++mJ21VVXFbOttir/8jrssMOK2eGH\nH17M7rzzzmIGrfziF78oZgMDAz2cpLfabQM/a9asYnbeeecVs+XLlxezSy+9tJitWbOmmFGXv/u7\nv+v3CJu9I444oqPzFixY0OVJoLumTJlSzI466qiu3qvdttaPPfZYV+8FvbRo0aJitvvuu/dwkvZK\nf5ebNm1a8Zx169YVsyeeeGLUMzF8ngkEAAAA0AAWgQAAAAAawCIQAAAAQANYBAIAAABoAItAAAAA\nAA1gEQgAAACgAWwR3yCTJ08uZn/0R39UzA4++OBi1m4b+HaWLl1azO6+++6OrgmtLFy4sN8jjJl2\n2/G22+r9pJNOKmbttt094YQThjcY0HU33XRTv0eAtr73ve8Vs5133rmja957770tj59++ukdXQ/o\njm233bbl8XbbwOeci9n8+fNHPRPD55lAAAAAAA1gEQgAAACgASwCAQAAADSARSAAAACABrAIBAAA\nANAAFoEAAAAAGsAW8ePUfvvtV8zOPvvslsc//OEPF89561vfOuqZNvbLX/6ymA0MDBSzdlsL0lwp\npY6y4447rpide+65o5qpF/78z/+8mP3VX/1VMdtxxx2L2bx584rZqaeeOrzBAGADu+66azHr9Hu7\nr3zlKy2Pr1mzpqPrAd1x66239nsERsEzgQAAAAAawCIQAAAAQANYBAIAAABoAItAAAAAAA1gEQgA\nAACgASwCAQAAADTAkFvEp5TeFhHXRcSEiMgRcUXO+bKU0i4RcUNETIqIJyPixJzzi2M36uap3dbs\nM2bMKGalbeAjIiZNmjSakUZk8eLFxeyiiy4qZgsXLhyLcRqlad3MOXeUtevY5ZdfXsyuvvrqYvb8\n888Xs0MPPbSYnXLKKS2PH3DAAcVz9txzz2L29NNPF7N2W3eWttylO5rWTbonpVTM3vnOdxaze++9\ndyzG2ezo5uhdc801xWyLLbr/b8s//OEPu35N6qOb48/v//7v93sERmE4v1u/HhGfyjm/KyIOjYhP\npJTeFRF/ERG35Zz3jYjbBn8O9I5uQp10E+qkm1An3YQeGnIRKOc8kHNeMvjxKxHxSETsERHHRsS1\ng592bUQcN1ZDApvSTaiTbkKddBPqpJvQW0O+HGxDKaVJEfGeiFgUERNyzgOD0X/F+qfvtTpnZkTM\n7HxEYCi6CXXSTaiTbkKddBPG3rBfvJtS2j4iFkTEn+WcX94wy+vfkKPlm3LknK/IOR+Ucz5oVJMC\nLekm1Ek3oU66CXXSTeiNYS0CpZR+I9YXcl7O+VuDh1emlCYO5hMj4tmxGREo0U2ok25CnXQT6qSb\n0DtDLgKl9VtVfCMiHsk5f3GDaGFEnDb48WkRcXP3xwNKdBPqpJtQJ92EOukm9NZw3hPof0XEKRHx\nk5TSA4PHLoiIORFxY0rp4xHxVEScODYjjg8TJrR8iWpERLzrXe8qZv/wD/9QzH7nd35nVDONxKJF\ni4rZF77whWJ2883l34vXrVs3qpkYkm4Ow5ZbblnMzjrrrGJ2wgknFLOXX365mO27777DG2yY2m2P\ne8cddxSzz372s12dgxHRTTqy/tUOrY3F9tsNpJvDMGXKlGI2ffr0Ytbu+761a9cWsy9/+cvFbOXK\nlcWMzYpujjP77LNPv0dgFIZcBMo53xMRqRAf0d1xgOHSTaiTbkKddBPqpJvQW/5ZCQAAAKABLAIB\nAAAANIBFIAAAAIAGsAgEAAAA0AAWgQAAAAAaYDhbxDfKLrvsUsy+/vWvF7N222n2egu90pbSl156\nafGcW2+9tZi99tpro54JRutHP/pRMbvvvvuK2cEHH9zR/d761rcWswkTJnR0zeeff77l8fnz5xfP\nOffcczu6F7B5+d3f/d1iNnfu3N4NwmZvp512Kmbt/mxsZ8WKFcXs05/+dEfXBPrnBz/4QcvjW2xR\nfo7JunXrxmocRsgzgQAAAAAawCIQAAAAQANYBAIAAABoAItAAAAAAA1gEQgAAACgASwCAQAAADTA\nZr1F/Pve976Wx88777ziOYccckgx22OPPUY900j8/Oc/L2aXX355Mfv85z/f8virr7466pmgX5Yv\nX17MPvzhDxezM888s5jNmjVrVDO1ctlllxWzr371qy2P/+xnP+v6HMD4k1Lq9wgAMKSHHnqo5fFl\ny5YVz9lnn32K2Tve8Y5itmrVquEPxrB4JhAAAABAA1gEAgAAAGgAi0AAAAAADWARCAAAAKABLAIB\nAAAANMBmvTvY8ccfP6Ljo7F06dJidssttxSz119/vZhdeumlxWz16tXDGwwaYGBgoJjNnj27owxg\nLHz3u98tZn/8x3/cw0mgtUcffbSY/fCHPyxmU6dOHYtxgHGktEt1RMRVV11VzC666KJids455xSz\ndn8Hp8wzgQAAAAAawCIQAAAAQANYBAIAAABoAItAAAAAAA1gEQgAAACgASwCAQAAADRAyjm3/4SU\n3hYR10XEhIjIEXFFzvmylNLsiDgjIlYNfuoFOed/GeJa7W8Gm7mcc+rWtXQTukc3oU66CXXSTVrZ\nYYcditmNN95YzKZPn17MvvWtbxWzj33sY8Xs1VdfLWabs+F0c6thXOf1iPhUznlJSuk3I+L+lNL3\nB7O/zzlfMpohgY7pJtRJN6FOugl10k3ooSEXgXLOAxExMPjxKymlRyJij7EeDGhPN6FOugl10k2o\nk25Cb43oPYFSSpMi4j0RsWjw0DkppR+nlK5OKe3c5dmAYdJNqJNuQp10E+qkmzD2hr0IlFLaPiIW\nRMSf5ZxfjoivRsQ+ETEl1q/cXlo4b2ZKaXFKaXEX5gU2optQJ92EOukm1Ek3oTeGfGPoiIiU0m9E\nxC0RcWvO+Yst8kkRcUvOef8hruONumi0br6JXoRuQrfoJtRJN6FOukkr3hi6/4bTzSGfCZRSShHx\njYh4ZMNCppQmbvBpx0fEQ50MCXRGN6FOugl10k2ok25Cbw1ni/ipEfGDiPhJRKwbPHxBRMyI9U/N\nyxHxZEScOfimXu2uZWWWRuvydpq6CV2im1An3YQ66SYj1e5ZQhdddFEx+9M//dNiNnny5GK2dOnS\n4Q22menKFvE553siotWF/qWToYDu0E2ok25CnXQT6qSb0Fsj2h0MAAAAgPHJIhAAAABAA1gEAgAA\nAGgAi0AAAAAADWARCAAAAKABhtwivqs3s2UfDdfN7TS7STdpOt2EOukm1Ek3oU7D6aZnAgEAAAA0\ngEUgAAAAgAawCAQAAADQABaBAAAAABrAIhAAAABAA1gEAgAAAGiArXp8v+ci4qnBj3cb/HkNapnF\nHJuqZZZuzLFXNwYZI7rZnjk2Vcssutkftcxijk3VMotu9l4tc0TUM0stc0TUM4tu9l4tc0TUM4s5\nNtWzbqac8yjv05mU0uKc80F9uflGapnFHJuqZZZa5uiFmr7WWmYxx6ZqmaWWOXqhpq+1llnMsala\nZqlljl6o5WutZY6IemapZY6IemapZY5eqOVrrWWOiHpmMcemejmLl4MBAAAANIBFIAAAAIAG6Oci\n0BV9vPfGapnFHJuqZZZa5uiFmr7WWmYxx6ZqmaWWOXqhpq+1llnMsalaZqlljl6o5WutZY6Iemap\nZY6IemapZY5eqOVrrWWOiHpmMcemejZL394TCAAAAIDe8XIwAAAAgAboyyJQSunolNJjKaWfpZT+\noh8zDM7xZErpJymlB1JKi3t876tTSs+mlB7a4NguKaXvp5SWDf535z7NMTultGLwcXkgpfTBHszx\ntpTSHSmlpSmlh1NK5w4e78djUpql549Lr+mmbraYo4puNrmXEbo5eG/d/PU5dLMCuqmbLebQzT6r\npZeDs/Slm7X0ss0sutnHbvb85WAppS0j4qcRcWRELI+I+yJiRs55aU8HWT/LkxFxUM75uT7c+7CI\nWBMR1+Wc9x88dnFEvJBznjP4G9bOOefz+zDH7IhYk3O+ZCzvvdEcEyNiYs55SUrpNyPi/og4LiJO\nj94/JqVZTowePy69pJv/c2/d/PU5quhmU3sZoZsb3Fs3f30O3ewz3fyfe+vmr8+hm31UUy8H53ky\n+tDNWnrZZpbZoZt962Y/ngl0SET8LOf8RM55bUTMj4hj+zBHX+Wc746IFzY6fGxEXDv48bWx/hdD\nP+bouZzzQM55yeDHr0TEIxGxR/TnMSnNsrnTzdDNFnNU0c0G9zJCNyNCN1vMoZv9p5uhmy3m0M3+\n0suop5dtZuk53fyVfiwC7RER/7nBz5dH/35DyhHxryml+1NKM/s0w4Ym5JwHBj/+r4iY0MdZzkkp\n/Xjw6Xs9eargG1JKkyLiPRGxKPr8mGw0S0QfH5ce0M0y3Yx6utmwXkboZju6GbrZR7pZppuhm31S\nUy8j6upmTb2M0M2+dbPpbww9Nec8JSL+d0R8YvCpalXI61+n16+t274aEftExJSIGIiIS3t145TS\n9hGxICL+LOf88oZZrx+TFrP07XFpIN1srfHd1Mu+083WdFM3+003W9NN3ey3KrvZ515G6GZfu9mP\nRaAVEfG2DX6+5+Cxnss5rxj877MRcVOsf/pgP60cfI3gG68VfLYfQ+ScV+acf5lzXhcRV0aPHpeU\n0m/E+iLMyzl/a/BwXx6TVrP063HpId0s080KutnQXkboZju6qZv9pJtluqmb/VJNLyOq62YVvYzQ\nzX53sx+LQPdFxL4ppb1TSltHxEciYmGvh0gpvXnwjZgipfTmiDgqIh5qf9aYWxgRpw1+fFpE3NyP\nId4owaDT0xArAAABJUlEQVTjowePS0opRcQ3IuKRnPMXN4h6/piUZunH49Jjulmmm33uZoN7GaGb\n7eimbvaTbpbppm72SxW9jKiym1X0MkI3W83R08ck59zzHxHxwVj/ru2PR8Rn+jTDPhHx4OCPh3s9\nR0T8Y6x/mtf/i/WvVf14ROwaEbdFxLKI+NeI2KVPc1wfET+JiB/H+lJM7MEcU2P9U+9+HBEPDP74\nYJ8ek9IsPX9cev1DN3WzxRxVdLPJvRz8+nVTNzeeQzcr+KGbutliDt3s848aejk4R9+6WUsv28yi\nm33sZs+3iAcAAACg95r+xtAAAAAAjWARCAAAAKABLAIBAAAANIBFIAAAAIAGsAgEAAAA0AAWgQAA\nAAAawCIQAAAAQANYBAIAAABogP8P1fUH3NxJUesAAAAASUVORK5CYII=\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "\n", "fig = plt.figure(figsize=(20, 5))\n", "for i in range(5):\n", " print(label_train[i])\n", " img = img_train[i]\n", " img = img.reshape(28, 28)\n", " img.shape = (28, 28)\n", " plt.subplot(150 + (i+1))\n", " plt.imshow(img, cmap='gray')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 2. Neural Network 모델 구성 " ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": false }, "outputs": [], "source": [ "def sigmoid(x):\n", " return 1 / (1 + np.exp(-x)) \n", "\n", "def softmax(x):\n", " c = np.max(x)\n", " exp_x = np.exp(x-c)\n", " sum_exp_x = np.sum(exp_x)\n", " y = exp_x / sum_exp_x\n", " return y\n", "\n", "def init_network():\n", " network = {}\n", " network['W1'] = np.zeros([784, 1024])\n", " network['b1'] = np.zeros([1024])\n", " network['W2'] = np.zeros([1024, 1024])\n", " network['b2'] = np.zeros([1024])\n", " network['W3'] = np.zeros([1024, 10])\n", " network['b3'] = np.zeros([10])\n", " return network\n", " \n", "def predict(network, x):\n", " W1, W2, W3 = network['W1'], network['W2'], network['W3']\n", " b1, b2, b3 = network['b1'], network['b2'], network['b3']\n", "\n", " a1 = np.dot(x, W1) + b1\n", " z1 = sigmoid(a1)\n", " a2 = np.dot(z1, W2) + b2\n", " z2 = sigmoid(a2)\n", " a3 = np.dot(z2, W3) + b3\n", " y = softmax(a3)\n", "\n", " return y" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 3. MNIST Test 테이터에 대한 단순 Feed Forward" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Accuracy:0.098\n" ] } ], "source": [ "_, _, (img_test, label_test) = load_mnist(flatten=True, normalize=False)\n", "network = init_network()\n", "accuracy_cnt = 0\n", "\n", "for i in range(len(img_test)):\n", " y = predict(network, img_test[i])\n", " p = np.argmax(y)\n", " if p == label_test[i]:\n", " accuracy_cnt += 1\n", "\n", "print(\"Accuracy:\" + str(float(accuracy_cnt) / len(img_test)))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 4. MNIST Test 테이터에 대하여 Batch를 활용한 Feed Forward" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Accuracy:0.098\n" ] } ], "source": [ "_, _, (img_test, label_test) = load_mnist(flatten=True, normalize=False)\n", "network = init_network()\n", "accuracy_cnt = 0\n", "batch_size = 100\n", "\n", "for i in range(0, len(img_test), batch_size):\n", " x_batch = img_test[i: i + batch_size]\n", " y_batch = predict(network, x_batch)\n", " p = np.argmax(y_batch, axis=1)\n", " accuracy_cnt += np.sum(p == label_test[i: i + batch_size])\n", "\n", "print(\"Accuracy:\" + str(float(accuracy_cnt) / len(img_test)))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 5. 2층 신경망을 이용한 학습 및 테스트 " ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def relu(x):\n", " return np.maximum(0, x)\n", "\n", "class TwoLayerNet:\n", " def __init__(self, input_size, hidden_layer1_size, hidden_layer2_size, output_size, weight_init_std=0.01):\n", " self.params = {}\n", " self.params['W1'] = weight_init_std * np.random.randn(input_size, hidden_layer1_size)\n", " self.params['b1'] = np.zeros(hidden_layer1_size)\n", " self.params['W2'] = weight_init_std * np.random.randn(hidden_layer1_size, hidden_layer2_size)\n", " self.params['b2'] = np.zeros(hidden_layer2_size)\n", " self.params['W3'] = weight_init_std * np.random.randn(hidden_layer2_size, output_size)\n", " self.params['b3'] = np.zeros(output_size)\n", " print(\"W1-shape: {0}, b1-shape: {1}, W2-shape: {2}, b2-shape: {3}, W3-shape: {4}, b3-shape: {5}\".format(\n", " self.params['W1'].shape,\n", " self.params['b1'].shape,\n", " self.params['W2'].shape,\n", " self.params['b2'].shape,\n", " self.params['W3'].shape,\n", " self.params['b3'].shape\n", " ))\n", "\n", " def predict(self, x):\n", " W1, W2, W3 = self.params['W1'], self.params['W2'], self.params['W3']\n", " b1, b2, b3 = self.params['b1'], self.params['b2'], self.params['b3']\n", " \n", " a1 = np.dot(x, W1) + b1\n", " z1 = relu(a1)\n", " a2 = np.dot(z1, W2) + b2\n", " z2 = relu(a2)\n", " a3 = np.dot(z2, W3) + b3\n", " y = softmax(a3)\n", " \n", " return y\n", "\n", " def cross_entropy_error(self, x, t):\n", " y = self.predict(x)\n", " if y.ndim == 1:\n", " t = t.reshape(1, t.size)\n", " y = y.reshape(1, y.size)\n", "\n", " if t.size == y.size:\n", " t = t.argmax(axis=1)\n", "\n", " batch_size = y.shape[0]\n", " return -np.sum(np.log(y[np.arange(batch_size), t])) / batch_size\n", " \n", " def accuracy(self, x, t):\n", " y = self.predict(x)\n", " y = np.argmax(y, axis=1)\n", " \n", " accuracy = np.sum(y == t) / float(x.shape[0])\n", " return accuracy\n", "\n", " def numerical_derivative(self, params, x, z_target):\n", " delta = 1e-4 # 0.0001\n", " grad = np.zeros_like(params)\n", " \n", " it = np.nditer(params, flags=['multi_index'], op_flags=['readwrite'])\n", " while not it.finished:\n", " idx = it.multi_index\n", " temp_val = params[idx]\n", "\n", " #f(x + delta) 계산\n", " params[idx] = params[idx] + delta\n", " fxh1 = self.cross_entropy_error(x, z_target)\n", " \n", " #f(x - delta) 계산\n", " params[idx] = params[idx] - delta\n", " fxh2 = self.cross_entropy_error(x, z_target)\n", " \n", " #f(x + delta) - f(x - delta) / 2 * delta 계산\n", " grad[idx] = (fxh1 - fxh2) / (2 * delta)\n", " params[idx] = temp_val\n", " it.iternext()\n", " return grad\n", " \n", " def learning(self, learning_rate, x_batch, t_batch):\n", " for key in ('W1', 'b1', 'W2', 'b2', 'W3', 'b3'):\n", " grad = self.numerical_derivative(self.params[key], x_batch, t_batch)\n", " self.params[key] = self.params[key] - learning_rate * grad" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Learning and Validation\n", "- 아래 코드의 수행시간은 매우 길기 때문에 Hidden Layer에 포함되는 Neuron을 5개만 두었음 " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "W1-shape: (784, 5), b1-shape: (5,), W2-shape: (5, 5), b2-shape: (5,), W3-shape: (5, 10), b3-shape: (10,)\n" ] } ], "source": [ "import math\n", "(img_train, label_train), (img_validation, label_validation), (img_test, label_test) = load_mnist(flatten=True, normalize=False)\n", "\n", "network = TwoLayerNet(input_size=784, hidden_layer1_size=5, hidden_layer2_size=5, output_size=10)\n", "\n", "num_epochs = 50\n", "train_size = img_train.shape[0]\n", "batch_size = 1000\n", "learning_rate = 0.1\n", "\n", "train_error_list = []\n", "validation_error_list = []\n", "\n", "test_accuracy_list = []\n", "epoch_list = []\n", "\n", "num_batch = math.ceil(train_size / batch_size)\n", "\n", "for i in range(num_epochs):\n", "# batch_mask = np.random.choice(train_size, batch_size)\n", "# x_batch = img_train[batch_mask]\n", "# t_batch = label_train[batch_mask]\n", "# network.learning(learning_rate, x_batch, t_batch)\n", "\n", " j = 0\n", " for j in range(num_batch):\n", " x_batch = img_train[j * batch_size : j * batch_size + batch_size]\n", " t_batch = label_train[j * batch_size : j * batch_size + batch_size]\n", " network.learning(learning_rate, x_batch, t_batch) \n", " network.learning(learning_rate, x_batch, t_batch)\n", " \n", " epoch_list.append(i)\n", " \n", " train_loss = network.cross_entropy_error(x_batch, t_batch)\n", " train_error_list.append(train_loss)\n", " \n", " validation_loss = network.cross_entropy_error(img_validation, label_validation)\n", " validation_error_list.append(validation_loss) \n", " \n", " test_accuracy = network.accuracy(img_test, label_test)\n", " test_accuracy_list.append(test_accuracy) \n", " \n", " print(\"Epoch: {0:5d}, Train Error: {1:7.5f}, Validation Error: {2:7.5f} - Test Accuracy: {3:7.5f}\".format(\n", " i,\n", " train_loss,\n", " validation_loss,\n", " test_accuracy\n", " ))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Analysis with Graph\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "# Draw Graph about Error Values & Accuracy Values\n", "def draw_error_values_and_accuracy(epoch_list, train_error_list, validation_error_list, test_accuracy_list):\n", " # Draw Error Values and Accuracy\n", " fig = plt.figure(figsize=(20, 5))\n", " plt.subplot(121)\n", " plt.plot(epoch_list[1:], train_error_list[1:], 'r', label='Train')\n", " plt.plot(epoch_list[1:], validation_error_list[1:], 'g', label='Validation')\n", " plt.ylabel('Total Error')\n", " plt.xlabel('Epochs')\n", " plt.grid(True)\n", " plt.legend(loc='upper right')\n", "\n", " plt.subplot(122)\n", " plt.plot(epoch_list[1:], test_accuracy_list[1:], 'b', label='Test')\n", " plt.ylabel('Accuracy')\n", " plt.xlabel('Epochs')\n", " plt.yticks(np.arange(0.0, 1.0, 0.05))\n", " plt.grid(True)\n", " plt.legend(loc='lower right')\n", " plt.show()\n", "\n", "draw_error_values_and_accuracy(epoch_list, train_error_list, validation_error_list, test_accuracy_list)\n", " \n", "def draw_false_prediction(diff_index_list):\n", " fig = plt.figure(figsize=(20, 5))\n", " for i in range(5):\n", " j = diff_index_list[i]\n", " print(\"False Prediction Index: %s, Prediction: %s, Ground Truth: %s\" % (j, prediction[j], ground_truth[j]))\n", " img = np.array(img_test[j])\n", " img.shape = (28, 28)\n", " plt.subplot(150 + (i+1))\n", " plt.imshow(img, cmap='gray')\n", " \n", "prediction = np.argmax(network.predict(img_test), axis=1)\n", "ground_truth = np.argmax(label_test, axis=1)\n", " \n", "print(prediction)\n", "print(ground_truth)\n", "\n", "diff_index_list = []\n", "for i in range(len(img_test)):\n", " if (prediction[i] != ground_truth[i]):\n", " diff_index_list.append(i)\n", "\n", "print(\"Total Test Image: {0}, Number of False Prediction: {1}\".format(len(img_test), len(diff_index_list)))\n", "print(\"Test Accuracy:\", float(len(img_test) - len(diff_index_list)) / float(len(img_test)))\n", "draw_false_prediction(diff_index_list)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3.0 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }