{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Christian Schuhegger \n", "last updated: 2019-02-27 \n", "\n", "CPython 3.6.8\n", "IPython 7.3.0\n", "\n", "numpy 1.14.6\n", "scipy 1.2.0\n", "pandas 0.24.1\n", "matplotlib 3.0.2\n", "seaborn 0.9.0\n", "mxnet 1.3.1\n" ] } ], "source": [ "%load_ext watermark\n", "%watermark -a 'Christian Schuhegger' -u -d -v -p numpy,scipy,pandas,matplotlib,seaborn,mxnet" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", "import numpy as np, scipy, scipy.stats as stats, pandas as pd, matplotlib.pyplot as plt, seaborn as sns\n", "import sklearn, sklearn.pipeline, sklearn.model_selection, sklearn.preprocessing\n", "import logging, time, datetime, tqdm\n", "import mxnet as mx\n", "from mxnet import gluon, nd, autograd, metric\n", "import sys,os,subprocess,glob,multiprocessing\n", "\n", "\n", "pd.set_option('display.max_columns', 500)\n", "pd.set_option('display.width', 1000)\n", "# pd.set_option('display.float_format', lambda x: '%.2f' % x)\n", "np.set_printoptions(edgeitems=10)\n", "np.set_printoptions(linewidth=1000)\n", "np.set_printoptions(suppress=True)\n", "np.core.arrayprint._line_width = 180\n", "\n", "sns.set()" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "def get_gpu_name():\n", " try:\n", " out_str = subprocess.run([\"nvidia-smi\", \"--query-gpu=gpu_name\", \"--format=csv\"], stdout=subprocess.PIPE).stdout\n", " out_list = out_str.decode(\"utf-8\").split('\\n')\n", " out_list = out_list[1:-1]\n", " return out_list\n", " except Exception as e:\n", " print(e)\n", "\n", "\n", "def get_cuda_version():\n", " \"\"\"Get CUDA version\"\"\"\n", " if sys.platform == 'win32':\n", " raise NotImplementedError(\"Implement this!\")\n", " # This breaks on linux:\n", " #cuda=!ls \"C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\"\n", " #path = \"C:\\\\Program Files\\\\NVIDIA GPU Computing Toolkit\\\\CUDA\\\\\" + str(cuda[0]) +\"\\\\version.txt\"\n", " elif sys.platform == 'linux' or sys.platform == 'darwin':\n", " path = '/usr/local/cuda/version.txt'\n", " else:\n", " raise ValueError(\"Not in Windows, Linux or Mac\")\n", " if os.path.isfile(path):\n", " with open(path, 'r') as f:\n", " data = f.read().replace('\\n','')\n", " return data\n", " else:\n", " return \"No CUDA in this machine\"\n", "\n", "def get_cudnn_version():\n", " \"\"\"Get CUDNN version\"\"\"\n", " if sys.platform == 'win32':\n", " raise NotImplementedError(\"Implement this!\")\n", " # This breaks on linux:\n", " #cuda=!ls \"C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\"\n", " #candidates = [\"C:\\\\Program Files\\\\NVIDIA GPU Computing Toolkit\\\\CUDA\\\\\" + str(cuda[0]) +\"\\\\include\\\\cudnn.h\"]\n", " elif sys.platform == 'linux':\n", " candidates = ['/usr/include/x86_64-linux-gnu/cudnn_v[0-99].h',\n", " '/usr/local/cuda/include/cudnn.h',\n", " '/usr/include/cudnn.h']\n", " elif sys.platform == 'darwin':\n", " candidates = ['/usr/local/cuda/include/cudnn.h',\n", " '/usr/include/cudnn.h']\n", " else:\n", " raise ValueError(\"Not in Windows, Linux or Mac\")\n", " for c in candidates:\n", " file = glob.glob(c)\n", " if file: break\n", " if file:\n", " with open(file[0], 'r') as f:\n", " version = ''\n", " for line in f:\n", " if \"#define CUDNN_MAJOR\" in line:\n", " version = line.split()[-1]\n", " if \"#define CUDNN_MINOR\" in line:\n", " version += '.' + line.split()[-1]\n", " if \"#define CUDNN_PATCHLEVEL\" in line:\n", " version += '.' + line.split()[-1]\n", " if version:\n", " return version\n", " else:\n", " return \"Cannot find CUDNN version\"\n", " else:\n", " return \"No CUDNN in this machine\"" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "OS: linux\n", "Python: 3.6.8 |Anaconda, Inc.| (default, Dec 30 2018, 01:22:34) \n", "[GCC 7.3.0]\n", "MXNet: 1.3.1\n", "Numpy: 1.14.6\n", "GPU: ['GeForce GTX 1080']\n", "CPU cores: 12\n", "CUDA Version 9.0.176\n", "CuDNN Version 7.4.2\n" ] } ], "source": [ "print(\"OS: \", sys.platform)\n", "print(\"Python: \", sys.version)\n", "print(\"MXNet: \", mx.__version__)\n", "print(\"Numpy: \", np.__version__)\n", "print(\"GPU: \", get_gpu_name())\n", "print(\"CPU cores:\", multiprocessing.cpu_count())\n", "print(get_cuda_version())\n", "print(\"CuDNN Version \", get_cudnn_version())" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from IPython.display import display, HTML\n", "\n", "from IPython.display import display_html\n", "def display_side_by_side(*args):\n", " html_str=''\n", " for df in args:\n", " if type(df) == np.ndarray:\n", " df = pd.DataFrame(df)\n", " html_str+=df.to_html()\n", " html_str = html_str.replace('table','table style=\"display:inline\"')\n", " # print(html_str)\n", " display_html(html_str,raw=True)\n", "\n", "CSS = \"\"\"\n", ".output {\n", " flex-direction: row;\n", "}\n", "\"\"\"\n", "\n", "def display_graphs_side_by_side(*args):\n", " html_str=''\n", " for g in args:\n", " html_str += ''\n", " html_str += g._repr_svg_()\n", " html_str += ''\n", " html_str += ''\n", " display_html(html_str,raw=True)\n", " \n", "\n", "display(HTML(\"\"))" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "logging.basicConfig(level=logging.DEBUG, format='%(asctime)s:%(name)s:%(levelname)s: %(message)s')\n", "log = logging.getLogger('std')" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "N_in = 1000\n", "N_subjects = 260 * 10 # 100000\n", "N_subjects = 100000\n", "W = stats.norm(loc=0, scale=1).rvs(size=(2,N_in), random_state=np.random.RandomState(42))\n", "X = stats.norm(loc=0, scale=1).rvs(size=(N_subjects,N_in), random_state=np.random.RandomState(43))\n", "y = np.sum(W[1:,:] * X + W[0,:], axis=1)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "((100000,), (100000, 1000))" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "y.shape, X.shape" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "count 100000.000000\n", "mean 19.234805\n", "std 31.516783\n", "min -147.380827\n", "25% -2.102370\n", "50% 19.273695\n", "75% 40.475011\n", "max 147.253509\n", "dtype: float64" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pd.Series(y).describe()" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, test_size = 0.1, random_state = 42)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "# Maximize training performance with Gluon data loader workers\n", "# https://aws.amazon.com/blogs/machine-learning/maximize-training-performance-with-gluon-data-loader-workers/\n", "def to_gluon_iter(x_in, y_in, batch_size=256, workers=(multiprocessing.cpu_count()//2)):\n", " x_nd = nd.array(x_in)\n", " y_nd = nd.array(y_in)\n", " dataset = mx.gluon.data.ArrayDataset(x_nd, y_nd)\n", "\n", " itr = mx.gluon.data.DataLoader(dataset, batch_size = batch_size, shuffle = None, num_workers=workers)# , last_batch = 'rollover'\n", " return itr\n", "\n", "def to_sym_iter(x_in, y_in, batch_size=256):\n", " itr = mx.io.NDArrayIter(x_in, y_in, batch_size, shuffle=None , label_name='lin_reg_label')\n", " return itr" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "class DataIterLoader():\n", " def __init__(self, data_iter):\n", " self.data_iter = data_iter\n", "\n", " def __iter__(self):\n", " self.data_iter.reset()\n", " return self\n", "\n", " def __next__(self):\n", " batch = self.data_iter.__next__()\n", " assert len(batch.data) == len(batch.label) == 1\n", " # print('len(batch_data): {}; batch.data[0].shape: {}'.format(len(batch.data), batch.data[0].shape))\n", " data = batch.data[0]\n", " label = batch.label[0]\n", " return data, label\n", "\n", " def next(self):\n", " return self.__next__() # for Python 2\n", "\n", "# See:\n", "# Appendix: Upgrading from Module DataIter to Gluon DataLoader\n", "# https://mxnet.incubator.apache.org/versions/master/tutorials/gluon/datasets.html\n", "batch_size=256\n", "gluon_train_iter = to_gluon_iter(X_train, y_train, batch_size=batch_size)\n", "gluon_valid_iter = to_gluon_iter(X_test , y_test, batch_size=batch_size)\n", "\n", "sym_train_iter = to_sym_iter(X_train, y_train, batch_size=batch_size)\n", "sym_valid_iter = to_sym_iter(X_test, y_test, batch_size=batch_size)\n", "\n", "gluon_train_iter = DataIterLoader(sym_train_iter)\n", "gluon_valid_iter = DataIterLoader(sym_valid_iter)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "def create_aux():\n", " epochs=20\n", " model_ctx=mx.cpu()\n", " loss_function = mx.gluon.loss.L2Loss()\n", " init_function = mx.init.Xavier()\n", " optimizer = mx.optimizer.Adam()\n", " return epochs, model_ctx, loss_function, init_function, optimizer" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "def create_gluon_model():\n", " ACTIVATION = 'relu'\n", " net = mx.gluon.nn.HybridSequential(prefix='MLP_')\n", " with net.name_scope():\n", " net.add(\n", " mx.gluon.nn.Dense(300, activation=ACTIVATION, prefix='fc-1_'),\n", " mx.gluon.nn.Dense(100, activation=ACTIVATION, prefix='fc-2_'),\n", " mx.gluon.nn.Dense(1 , activation=None , prefix='predictions')\n", " )\n", " return net" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "def create_sym_model():\n", " ACTIVATION = 'relu'\n", "\n", " data = mx.sym.Variable('data')\n", " Y = mx.sym.Variable('lin_reg_label')\n", " fc1 = mx.sym.FullyConnected(data, name='fc1', num_hidden=300)\n", " act1 = mx.sym.Activation(fc1, name='relu1', act_type=ACTIVATION)\n", " fc2 = mx.sym.FullyConnected(act1, name='fc2', num_hidden=100)\n", " act2 = mx.sym.Activation(fc2, name='relu2', act_type=ACTIVATION)\n", " fc3 = mx.sym.FullyConnected(act2, name='fc3', num_hidden=1)\n", " lro = mx.sym.LinearRegressionOutput(data=fc3, label=Y, name=\"lro\")\n", " \n", " return lro " ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "epochs, model_ctx, loss_function, init_function, optimizer = create_aux()" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[1.0114946365356445, 0, 2.688013]\n", "[2.0326144695281982, 1, 0.9665312]\n", "[3.0164339542388916, 2, 0.48773876]\n", "[3.9574451446533203, 3, 0.28398213]\n", "[4.913841009140015, 4, 0.19583277]\n", "[5.915135383605957, 5, 0.15492727]\n", "[6.879735946655273, 6, 0.14247555]\n", "[7.859589576721191, 7, 0.18341143]\n", "[8.832339763641357, 8, 0.8992266]\n", "[9.830835580825806, 9, 3.629622]\n", "[10.792657375335693, 10, 1.9046185]\n", "[11.763952732086182, 11, 1.7743697]\n", "[12.71415376663208, 12, 0.5882967]\n", "[13.694055318832397, 13, 1.5400267]\n", "[14.645780086517334, 14, 1.0786914]\n", "[15.640945672988892, 15, 1.2148588]\n", "[16.607487201690674, 16, 1.0880853]\n", "[17.586325645446777, 17, 0.6401208]\n", "[18.567651748657227, 18, 0.7938393]\n", "[19.549959897994995, 19, 0.47012243]\n", "workers: 0; time: 19.550041675567627\n", "[2.080505847930908, 0, 0.46424496]\n", "[3.9421205520629883, 1, 0.48203924]\n", "[6.162583112716675, 2, 0.49990627]\n", "[8.28538727760315, 3, 0.3972701]\n", "[10.105726480484009, 4, 0.4251598]\n", "[11.934930562973022, 5, 0.70309675]\n", "[13.798903703689575, 6, 1.5159985]\n", "[15.691807985305786, 7, 2.4252243]\n", "[17.546621561050415, 8, 1.616506]\n", "[19.324228763580322, 9, 3.8505468]\n", "[21.208154916763306, 10, 1.9401183]\n", "[23.062612056732178, 11, 1.0572857]\n", "[24.97093892097473, 12, 0.78012973]\n", "[26.971498727798462, 13, 0.39198607]\n", "[28.860776901245117, 14, 0.2522775]\n", "[30.770732879638672, 15, 0.4382887]\n", "[32.62504196166992, 16, 0.74195063]\n", "[34.49631190299988, 17, 0.6759308]\n", "[36.76907277107239, 18, 1.1210217]\n", "[38.66449570655823, 19, 0.9971001]\n", "workers: 2; time: 38.66468834877014\n", "[1.8759772777557373, 0, 0.5457086]\n", "[3.9425806999206543, 1, 0.27232322]\n", "[5.816941976547241, 2, 0.14699128]\n", "[8.007057428359985, 3, 0.10915594]\n", "[10.081457376480103, 4, 0.12385155]\n", "[11.983157634735107, 5, 0.17104238]\n", "[13.958155155181885, 6, 0.23911487]\n", "[15.885420083999634, 7, 0.3623786]\n", "[17.843082189559937, 8, 0.5243365]\n", "[19.805946111679077, 9, 0.45486805]\n", "[21.74500060081482, 10, 1.5060887]\n", "[23.67216157913208, 11, 1.9099464]\n", "[25.843996047973633, 12, 0.714357]\n", "[27.72542119026184, 13, 0.28021395]\n", "[29.729376316070557, 14, 0.17045738]\n", "[31.75821566581726, 15, 0.110555805]\n", "[33.6689875125885, 16, 0.17122094]\n", "[35.82129526138306, 17, 0.42039445]\n", "[37.803547620773315, 18, 0.34069037]\n", "[39.806894302368164, 19, 0.5471134]\n", "workers: 4; time: 39.807068824768066\n" ] } ], "source": [ "sym_train_iter = to_sym_iter(X_train, y_train, batch_size=batch_size)\n", "sym_valid_iter = to_sym_iter(X_test, y_test, batch_size=batch_size)\n", "gluon_model = create_gluon_model()\n", "# gluon_model.hybridize()\n", "gluon_model.hybridize(static_shape=True, static_alloc=True)\n", "gluon_model.collect_params().initialize(init_function, ctx=model_ctx)\n", "\n", "trainer = gluon.Trainer(gluon_model.collect_params(), optimizer)\n", "\n", "nr_batches = len(X_train) // batch_size\n", "total = epochs * (nr_batches + 1)\n", "\n", "def train(num_workers):\n", " mx.random.seed(1)\n", " \n", " if num_workers > 1:\n", " gluon_train_iter = to_gluon_iter(X_train, y_train, batch_size=batch_size, workers=num_workers)\n", " gluon_valid_iter = to_gluon_iter(X_test , y_test, batch_size=batch_size, workers=num_workers)\n", " else:\n", " gluon_train_iter = DataIterLoader(sym_train_iter)\n", " gluon_valid_iter = DataIterLoader(sym_valid_iter)\n", "\n", " time1 = time.time()\n", " for e in range(epochs):\n", " for i, (x_, y_) in enumerate(gluon_train_iter):\n", " x_ = x_.as_in_context(model_ctx)\n", " y_ = y_.as_in_context(model_ctx)\n", " if num_workers > 1:\n", " nd.waitall()\n", " with autograd.record():\n", " output = gluon_model(x_)\n", " loss = loss_function(output, y_)\n", "\n", " loss.backward()\n", " last_batch_loss = nd.mean(loss).asscalar()\n", " trainer.step(x_.shape[0])\n", " if num_workers > 1:\n", " nd.waitall()\n", " t = time.time()\n", " print([t-time1, e, last_batch_loss])\n", " \n", " time2 = time.time()\n", " print('workers: {}; time: {}'.format(num_workers, time2-time1))\n", "\n", "for workers in range(0, 6, 2):\n", " train(num_workers=workers)" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2.737390181114425" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "gluon_predict_iter = mx.gluon.data.DataLoader(mx.gluon.data.ArrayDataset(nd.array(X_test)), batch_size=batch_size)\n", "y_gluon_pred = nd.zeros(X_test.shape[0])\n", "for i, (data) in enumerate(gluon_predict_iter):\n", " data = data.as_in_context(model_ctx)\n", " output = gluon_model(data)\n", " y_gluon_pred[i * batch_size : i * batch_size + output.shape[0]] = output[:,0]\n", "\n", "s = sklearn.metrics.mean_squared_error(y_test, y_gluon_pred.asnumpy())\n", "s" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.9976410425625697" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sklearn.metrics.explained_variance_score(y_test, y_gluon_pred.asnumpy())" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "epochs, model_ctx, loss_function, init_function, optimizer = create_aux()" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/local/cs/local/install/anaconda3-5.3.1-Linux-x86_64/envs/mxnet/lib/python3.6/site-packages/mxnet/module/base_module.py:504: UserWarning: Optimizer created manually outside Module but rescale_grad is not normalized to 1.0/batch_size/num_workers (1.0 vs. 0.00390625). Is this intended?\n", " optimizer_params=optimizer_params)\n", "2019-02-27 14:51:46,594:root:INFO: Epoch[0] Train-mse=151.881052\n", "2019-02-27 14:51:46,595:root:INFO: Epoch[0] Time cost=1.018\n", "2019-02-27 14:51:46,664:root:INFO: Epoch[0] Validation-mse=4.889890\n", "2019-02-27 14:51:47,685:root:INFO: Epoch[1] Train-mse=2.949202\n", "2019-02-27 14:51:47,686:root:INFO: Epoch[1] Time cost=1.021\n", "2019-02-27 14:51:47,746:root:INFO: Epoch[1] Validation-mse=3.650367\n", "2019-02-27 14:51:48,753:root:INFO: Epoch[2] Train-mse=1.537151\n", "2019-02-27 14:51:48,754:root:INFO: Epoch[2] Time cost=1.007\n", "2019-02-27 14:51:48,835:root:INFO: Epoch[2] Validation-mse=3.233742\n", "2019-02-27 14:51:49,828:root:INFO: Epoch[3] Train-mse=0.924725\n", "2019-02-27 14:51:49,829:root:INFO: Epoch[3] Time cost=0.993\n", "2019-02-27 14:51:49,911:root:INFO: Epoch[3] Validation-mse=3.063358\n", "2019-02-27 14:51:50,909:root:INFO: Epoch[4] Train-mse=0.610477\n", "2019-02-27 14:51:50,910:root:INFO: Epoch[4] Time cost=0.998\n", "2019-02-27 14:51:50,992:root:INFO: Epoch[4] Validation-mse=3.018220\n", "2019-02-27 14:51:51,994:root:INFO: Epoch[5] Train-mse=0.441097\n", "2019-02-27 14:51:51,995:root:INFO: Epoch[5] Time cost=1.001\n", "2019-02-27 14:51:52,069:root:INFO: Epoch[5] Validation-mse=3.035564\n", "2019-02-27 14:51:53,068:root:INFO: Epoch[6] Train-mse=0.355614\n", "2019-02-27 14:51:53,069:root:INFO: Epoch[6] Time cost=0.998\n", "2019-02-27 14:51:53,134:root:INFO: Epoch[6] Validation-mse=3.111674\n", "2019-02-27 14:51:54,152:root:INFO: Epoch[7] Train-mse=0.355230\n", "2019-02-27 14:51:54,153:root:INFO: Epoch[7] Time cost=1.018\n", "2019-02-27 14:51:54,230:root:INFO: Epoch[7] Validation-mse=3.327117\n", "2019-02-27 14:51:55,202:root:INFO: Epoch[8] Train-mse=0.669648\n", "2019-02-27 14:51:55,203:root:INFO: Epoch[8] Time cost=0.972\n", "2019-02-27 14:51:55,273:root:INFO: Epoch[8] Validation-mse=4.119022\n", "2019-02-27 14:51:56,318:root:INFO: Epoch[9] Train-mse=2.828806\n", "2019-02-27 14:51:56,319:root:INFO: Epoch[9] Time cost=1.046\n", "2019-02-27 14:51:56,380:root:INFO: Epoch[9] Validation-mse=5.946182\n", "2019-02-27 14:51:57,537:root:INFO: Epoch[10] Train-mse=4.074934\n", "2019-02-27 14:51:57,538:root:INFO: Epoch[10] Time cost=1.157\n", "2019-02-27 14:51:57,601:root:INFO: Epoch[10] Validation-mse=3.578255\n", "2019-02-27 14:51:58,611:root:INFO: Epoch[11] Train-mse=2.713380\n", "2019-02-27 14:51:58,612:root:INFO: Epoch[11] Time cost=1.011\n", "2019-02-27 14:51:58,681:root:INFO: Epoch[11] Validation-mse=2.832748\n", "2019-02-27 14:51:59,680:root:INFO: Epoch[12] Train-mse=2.544748\n", "2019-02-27 14:51:59,681:root:INFO: Epoch[12] Time cost=0.999\n", "2019-02-27 14:51:59,740:root:INFO: Epoch[12] Validation-mse=3.092040\n", "2019-02-27 14:52:00,819:root:INFO: Epoch[13] Train-mse=2.752240\n", "2019-02-27 14:52:00,820:root:INFO: Epoch[13] Time cost=1.080\n", "2019-02-27 14:52:00,894:root:INFO: Epoch[13] Validation-mse=2.507709\n", "2019-02-27 14:52:01,868:root:INFO: Epoch[14] Train-mse=2.272493\n", "2019-02-27 14:52:01,869:root:INFO: Epoch[14] Time cost=0.974\n", "2019-02-27 14:52:01,939:root:INFO: Epoch[14] Validation-mse=2.545414\n", "2019-02-27 14:52:02,992:root:INFO: Epoch[15] Train-mse=1.962754\n", "2019-02-27 14:52:02,993:root:INFO: Epoch[15] Time cost=1.053\n", "2019-02-27 14:52:03,052:root:INFO: Epoch[15] Validation-mse=3.936031\n", "2019-02-27 14:52:04,065:root:INFO: Epoch[16] Train-mse=2.390086\n", "2019-02-27 14:52:04,065:root:INFO: Epoch[16] Time cost=1.013\n", "2019-02-27 14:52:04,142:root:INFO: Epoch[16] Validation-mse=4.206038\n", "2019-02-27 14:52:05,148:root:INFO: Epoch[17] Train-mse=3.163517\n", "2019-02-27 14:52:05,149:root:INFO: Epoch[17] Time cost=1.006\n", "2019-02-27 14:52:05,209:root:INFO: Epoch[17] Validation-mse=4.414568\n", "2019-02-27 14:52:06,207:root:INFO: Epoch[18] Train-mse=3.450250\n", "2019-02-27 14:52:06,207:root:INFO: Epoch[18] Time cost=0.997\n", "2019-02-27 14:52:06,273:root:INFO: Epoch[18] Validation-mse=5.286100\n", "2019-02-27 14:52:07,255:root:INFO: Epoch[19] Train-mse=2.802456\n", "2019-02-27 14:52:07,256:root:INFO: Epoch[19] Time cost=0.982\n", "2019-02-27 14:52:07,329:root:INFO: Epoch[19] Validation-mse=3.315309\n" ] } ], "source": [ "sym_train_iter = to_sym_iter(X_train, y_train, batch_size=batch_size)\n", "sym_valid_iter = to_sym_iter(X_test, y_test, batch_size=batch_size)\n", "\n", "sym_model = create_sym_model()\n", "\n", "sym_model_module = mx.mod.Module(symbol = sym_model, data_names = ['data'], label_names = ['lin_reg_label'], context = model_ctx)\n", "\n", "freq = int((len(X_train) * epochs / batch_size) // 10)\n", "batch_end_callback = mx.callback.Speedometer(batch_size, frequent=freq, auto_reset=False)\n", "\n", "time1 = time.time()\n", "\n", "sym_model_module.fit(sym_train_iter, \n", " sym_valid_iter,\n", " optimizer=optimizer,\n", " initializer=init_function,\n", " num_epoch=epochs,\n", " eval_metric='mse',\n", " batch_end_callback=batch_end_callback\n", " )\n", "time2 = time.time()" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "21.75580096244812\n" ] } ], "source": [ "print(time2-time1)" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "3.3060569400329425" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "y_sym_pred = sym_model_module.predict(sym_valid_iter)\n", "s = sklearn.metrics.mean_squared_error(y_test, y_sym_pred.asnumpy())\n", "s" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.9966796086836532" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sklearn.metrics.explained_variance_score(y_test, y_sym_pred.asnumpy())" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.8" } }, "nbformat": 4, "nbformat_minor": 2 }