{ "cells": [ { "cell_type": "code", "execution_count": 42, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "import numpy\n", "import pandas as pd\n", "from keras.models import Sequential\n", "from keras.layers import Dense\n", "from mnist import MNIST\n", "from sklearn.model_selection import GridSearchCV\n", "from keras.layers import Dropout\n", "from keras.wrappers.scikit_learn import KerasClassifier\n", "from keras.constraints import maxnorm\n", "from keras.models import Model \n", "from keras.layers import * \n", "from keras import optimizers\n", "%matplotlib inline" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Цель вычислительного эксперимента:\n", "Решить задачу: выбора алгоритма оптимизации (SGD, Nesterov Momentum, Adam) с использованием нейронных сетей простой структуры, со структурными параметрами: структура сети\n" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "model = Sequential()" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "model.add(Dense(units=785, activation='relu', input_dim=100))\n", "model.add(Dense(units=10, activation='softmax'))" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "model.compile(loss='categorical_crossentropy',\n", " optimizer='sgd',\n", " metrics=['accuracy'])" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "train = pd.read_csv(\"/home/jeny/mnist_train.csv\")\n", "test = pd.read_csv(\"/home/jeny/mnist_test.csv\")\n", "train = train.iloc[:40000, :]\n", "test = train.iloc[:40000, :]" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(40000, 785)" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train.shape" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
500.10.20.30.40.50.60.70.8...0.6080.6090.6100.6110.6120.6130.6140.6150.6160.617
00000000000...0000000000
14000000000...0000000000
21000000000...0000000000
39000000000...0000000000
42000000000...0000000000
\n", "

5 rows × 785 columns

\n", "
" ], "text/plain": [ " 5 0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 ... 0.608 0.609 0.610 \\\n", "0 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 \n", "1 4 0 0 0 0 0 0 0 0 0 ... 0 0 0 \n", "2 1 0 0 0 0 0 0 0 0 0 ... 0 0 0 \n", "3 9 0 0 0 0 0 0 0 0 0 ... 0 0 0 \n", "4 2 0 0 0 0 0 0 0 0 0 ... 0 0 0 \n", "\n", " 0.611 0.612 0.613 0.614 0.615 0.616 0.617 \n", "0 0 0 0 0 0 0 0 \n", "1 0 0 0 0 0 0 0 \n", "2 0 0 0 0 0 0 0 \n", "3 0 0 0 0 0 0 0 \n", "4 0 0 0 0 0 0 0 \n", "\n", "[5 rows x 785 columns]" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train.head()" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(40000, 784)" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "x_train = train.iloc[:,1:]\n", "y_train = train.iloc[:, 0:1]\n", "\n", "x_test = test.iloc[:, 1:]\n", "y_test = test.iloc[:, 0:1]\n", "x_train.shape" ] }, { "cell_type": "code", "execution_count": 43, "metadata": {}, "outputs": [], "source": [ "def create_model(neurons=1):\n", " model = Sequential()\n", " model.add(Dense(30, activation='sigmoid', input_shape=(784, )))\n", " model.add(Dense(1, activation='linear'))\n", " sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)\n", " model.compile(loss='mean_squared_error', optimizer=sgd, metrics=['accuracy'])\n", " return model\n" ] }, { "cell_type": "code", "execution_count": 44, "metadata": {}, "outputs": [], "source": [ "model = KerasClassifier(build_fn=create_model, epochs=15, batch_size=80 , verbose=1)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Adam" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 1/15\n", "40000/40000 [==============================] - 8s 208us/step - loss: 6.3400 - acc: 0.1247\n", "Epoch 2/15\n", "40000/40000 [==============================] - 8s 199us/step - loss: 3.0765 - acc: 0.2440\n", "Epoch 3/15\n", "40000/40000 [==============================] - 9s 220us/step - loss: 2.4657 - acc: 0.3314\n", "Epoch 4/15\n", "40000/40000 [==============================] - 9s 213us/step - loss: 2.1668 - acc: 0.3524\n", "Epoch 5/15\n", "40000/40000 [==============================] - 9s 214us/step - loss: 1.9750 - acc: 0.3693\n", "Epoch 6/15\n", "40000/40000 [==============================] - 9s 236us/step - loss: 1.8766 - acc: 0.3680\n", "Epoch 7/15\n", "40000/40000 [==============================] - 9s 234us/step - loss: 1.7650 - acc: 0.3695\n", "Epoch 8/15\n", "40000/40000 [==============================] - 9s 224us/step - loss: 1.7523 - acc: 0.3675\n", "Epoch 9/15\n", "40000/40000 [==============================] - 9s 234us/step - loss: 1.7314 - acc: 0.3870\n", "Epoch 10/15\n", "40000/40000 [==============================] - 10s 239us/step - loss: 1.7128 - acc: 0.3867\n", "Epoch 11/15\n", "40000/40000 [==============================] - 12s 303us/step - loss: 1.6656 - acc: 0.3875\n", "Epoch 12/15\n", "40000/40000 [==============================] - 11s 285us/step - loss: 1.6655 - acc: 0.3934\n", "Epoch 13/15\n", "40000/40000 [==============================] - 11s 285us/step - loss: 1.5482 - acc: 0.4147\n", "Epoch 14/15\n", "40000/40000 [==============================] - 12s 291us/step - loss: 1.5239 - acc: 0.4387\n", "Epoch 15/15\n", "40000/40000 [==============================] - 11s 285us/step - loss: 1.5659 - acc: 0.4480\n" ] } ], "source": [ "neurons = [5,10,15,20,50,100]\n", "param_grid = dict(neurons=neurons)\n", "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)\n", "grid_result = grid.fit(x_train, y_train)" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Best: 0.467875 using {'neurons': 100}\n", "0.397100 (0.040261) with: {'neurons': 5}\n", "0.438450 (0.016187) with: {'neurons': 10}\n", "0.413425 (0.057741) with: {'neurons': 15}\n", "0.408500 (0.023974) with: {'neurons': 20}\n", "0.404650 (0.024113) with: {'neurons': 50}\n", "0.467875 (0.008149) with: {'neurons': 100}\n" ] } ], "source": [ "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n", "means = grid_result.cv_results_['mean_test_score']\n", "stds = grid_result.cv_results_['std_test_score']\n", "params = grid_result.cv_results_['params']\n", "for mean, stdev, param in zip(means, stds, params):\n", " print(\"%f (%f) with: %r\" % (mean, stdev, param))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# SGD" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 1/15\n", "40000/40000 [==============================] - 7s 167us/step - loss: 4.6629 - acc: 0.1422\n", "Epoch 2/15\n", "40000/40000 [==============================] - 7s 184us/step - loss: 4.1693 - acc: 0.1953\n", "Epoch 3/15\n", "40000/40000 [==============================] - 6s 155us/step - loss: 3.8125 - acc: 0.2124\n", "Epoch 4/15\n", "40000/40000 [==============================] - 6s 161us/step - loss: 3.8286 - acc: 0.2261\n", "Epoch 5/15\n", "40000/40000 [==============================] - 6s 159us/step - loss: 3.6430 - acc: 0.2216\n", "Epoch 6/15\n", "40000/40000 [==============================] - 7s 179us/step - loss: 3.7137 - acc: 0.2265\n", "Epoch 7/15\n", "40000/40000 [==============================] - 6s 161us/step - loss: 3.3175 - acc: 0.2452\n", "Epoch 8/15\n", "40000/40000 [==============================] - 7s 177us/step - loss: 3.6765 - acc: 0.2280\n", "Epoch 9/15\n", "40000/40000 [==============================] - 8s 207us/step - loss: 3.9299 - acc: 0.2344\n", "Epoch 10/15\n", "40000/40000 [==============================] - 7s 183us/step - loss: 3.7395 - acc: 0.2350\n", "Epoch 11/15\n", "40000/40000 [==============================] - 8s 203us/step - loss: 3.9657 - acc: 0.1853\n", "Epoch 12/15\n", "40000/40000 [==============================] - 8s 190us/step - loss: 3.9659 - acc: 0.2689\n", "Epoch 13/15\n", "40000/40000 [==============================] - 8s 202us/step - loss: 3.1007 - acc: 0.2712\n", "Epoch 14/15\n", "40000/40000 [==============================] - 8s 210us/step - loss: 3.3459 - acc: 0.3252\n", "Epoch 15/15\n", "40000/40000 [==============================] - 9s 222us/step - loss: 3.2098 - acc: 0.2271\n" ] } ], "source": [ "neurons = [5,10,15,20,50,100]\n", "param_grid = dict(neurons=neurons)\n", "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)\n", "grid_result = grid.fit(x_train, y_train)" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Best: 0.304075 using {'neurons': 10}\n", "0.260050 (0.005875) with: {'neurons': 5}\n", "0.304075 (0.048464) with: {'neurons': 10}\n", "0.264075 (0.048997) with: {'neurons': 15}\n", "0.269700 (0.080197) with: {'neurons': 20}\n", "0.239000 (0.035026) with: {'neurons': 50}\n", "0.257000 (0.019166) with: {'neurons': 100}\n" ] } ], "source": [ "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n", "means = grid_result.cv_results_['mean_test_score']\n", "stds = grid_result.cv_results_['std_test_score']\n", "params = grid_result.cv_results_['params']\n", "for mean, stdev, param in zip(means, stds, params):\n", " print(\"%f (%f) with: %r\" % (mean, stdev, param))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Nesterov" ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_split.py:2053: FutureWarning: You should specify a value for 'cv' instead of relying on the default value. The default value will change from 3 to 5 in version 0.22.\n", " warnings.warn(CV_WARNING, FutureWarning)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Epoch 1/15\n", "40000/40000 [==============================] - 6s 149us/step - loss: 8.3136 - acc: 0.0929\n", "Epoch 2/15\n", "40000/40000 [==============================] - 8s 204us/step - loss: 8.4450 - acc: 0.0941\n", "Epoch 3/15\n", "40000/40000 [==============================] - 10s 242us/step - loss: 8.4571 - acc: 0.0913\n", "Epoch 4/15\n", "40000/40000 [==============================] - 9s 235us/step - loss: 8.4487 - acc: 0.0939\n", "Epoch 5/15\n", "40000/40000 [==============================] - 9s 225us/step - loss: 8.4414 - acc: 0.0959\n", "Epoch 6/15\n", "40000/40000 [==============================] - 9s 237us/step - loss: 8.4476 - acc: 0.0953\n", "Epoch 7/15\n", "40000/40000 [==============================] - 9s 218us/step - loss: 8.4386 - acc: 0.0942\n", "Epoch 8/15\n", "40000/40000 [==============================] - 10s 259us/step - loss: 8.4513 - acc: 0.0950\n", "Epoch 9/15\n", "40000/40000 [==============================] - 9s 225us/step - loss: 8.4470 - acc: 0.0935\n", "Epoch 10/15\n", "40000/40000 [==============================] - 9s 224us/step - loss: 8.4345 - acc: 0.0946\n", "Epoch 11/15\n", "40000/40000 [==============================] - 10s 257us/step - loss: 8.4481 - acc: 0.0949\n", "Epoch 12/15\n", "40000/40000 [==============================] - 11s 263us/step - loss: 8.4718 - acc: 0.0955\n", "Epoch 13/15\n", "40000/40000 [==============================] - 11s 267us/step - loss: 8.4606 - acc: 0.0926\n", "Epoch 14/15\n", "40000/40000 [==============================] - 10s 249us/step - loss: 8.4306 - acc: 0.0960\n", "Epoch 15/15\n", "40000/40000 [==============================] - 10s 251us/step - loss: 8.4419 - acc: 0.0947\n" ] } ], "source": [ "neurons = [5,10,15,20,50,100]\n", "param_grid = dict(neurons=neurons)\n", "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)\n", "grid_result = grid.fit(x_train, y_train)" ] }, { "cell_type": "code", "execution_count": 46, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Best: 0.122800 using {'neurons': 100}\n", "0.101550 (0.005835) with: {'neurons': 5}\n", "0.114650 (0.023514) with: {'neurons': 10}\n", "0.111050 (0.028868) with: {'neurons': 15}\n", "0.107925 (0.010288) with: {'neurons': 20}\n", "0.100300 (0.011810) with: {'neurons': 50}\n", "0.122800 (0.028337) with: {'neurons': 100}\n" ] } ], "source": [ "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n", "means = grid_result.cv_results_['mean_test_score']\n", "stds = grid_result.cv_results_['std_test_score']\n", "params = grid_result.cv_results_['params']\n", "for mean, stdev, param in zip(means, stds, params):\n", " print(\"%f (%f) with: %r\" % (mean, stdev, param))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.7" } }, "nbformat": 4, "nbformat_minor": 2 }