{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"import mxnet as mx\n",
"from sklearn.metrics import mean_squared_error\n",
"from sklearn.metrics import log_loss\n",
"from sklearn.metrics import roc_auc_score\n",
"def mx_log_loss(actual, predicted):\n",
" return log_loss(actual, predicted)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import logging\n",
"logging.getLogger().setLevel(logging.DEBUG)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"from sklearn.datasets import load_boston\n",
"boston = load_boston()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Regression (Boston House Prices)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"((506, 13), (506,))\n"
]
}
],
"source": [
"x = pd.DataFrame(boston.data, columns=boston.feature_names)\n",
"y = boston.target\n",
"print((x.shape, y.shape))"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"flg_train = np.random.choice([False, True], len(y), replace=True, p=[0.3, 0.7])\n",
"flg_valid = np.logical_not(flg_train)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"batch_size = 100\n",
"train_iter = mx.io.NDArrayIter(x[flg_train].values, y[flg_train],\n",
" batch_size, shuffle=True,\n",
" label_name='label')\n",
"valid_iter = mx.io.NDArrayIter(x[flg_valid].values, y[flg_valid],\n",
" batch_size,\n",
" label_name='label')\n",
"test_iter = mx.io.NDArrayIter(x[flg_valid].values)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"data = mx.sym.Variable('data')\n",
"label = mx.sym.Variable('label')\n",
"fc = mx.sym.FullyConnected(data, num_hidden=3)\n",
"act = mx.sym.Activation(fc, act_type='relu')\n",
"fco = mx.sym.FullyConnected(act, num_hidden=1)\n",
"net1 = mx.sym.LinearRegressionOutput(fco, label)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### 1. Build-in function"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:root:Start training with [cpu(0)]\n",
"INFO:root:Epoch[0] Resetting Data Iterator\n",
"INFO:root:Epoch[0] Time cost=0.017\n",
"INFO:root:Epoch[0] Validation-mse=1395687.562500\n",
"INFO:root:Epoch[1] Resetting Data Iterator\n",
"INFO:root:Epoch[1] Time cost=0.016\n",
"INFO:root:Epoch[1] Validation-mse=600322.187500\n",
"INFO:root:Epoch[2] Resetting Data Iterator\n",
"INFO:root:Epoch[2] Time cost=0.014\n",
"INFO:root:Epoch[2] Validation-mse=258119.859375\n",
"INFO:root:Epoch[3] Resetting Data Iterator\n",
"INFO:root:Epoch[3] Time cost=0.037\n",
"INFO:root:Epoch[3] Validation-mse=110928.750000\n",
"INFO:root:Epoch[4] Resetting Data Iterator\n",
"INFO:root:Epoch[4] Time cost=0.008\n",
"INFO:root:Epoch[4] Validation-mse=47643.732422\n",
"INFO:root:Epoch[5] Resetting Data Iterator\n",
"INFO:root:Epoch[5] Time cost=0.007\n",
"INFO:root:Epoch[5] Validation-mse=20451.433594\n",
"INFO:root:Epoch[6] Resetting Data Iterator\n",
"INFO:root:Epoch[6] Time cost=0.006\n",
"INFO:root:Epoch[6] Validation-mse=8778.733398\n",
"INFO:root:Epoch[7] Resetting Data Iterator\n",
"INFO:root:Epoch[7] Time cost=0.006\n",
"INFO:root:Epoch[7] Validation-mse=3775.464355\n",
"INFO:root:Epoch[8] Resetting Data Iterator\n",
"INFO:root:Epoch[8] Time cost=0.005\n",
"INFO:root:Epoch[8] Validation-mse=1635.793518\n",
"INFO:root:Epoch[9] Resetting Data Iterator\n",
"INFO:root:Epoch[9] Time cost=0.005\n",
"INFO:root:Epoch[9] Validation-mse=723.968018\n",
"INFO:root:Epoch[10] Resetting Data Iterator\n",
"INFO:root:Epoch[10] Time cost=0.005\n",
"INFO:root:Epoch[10] Validation-mse=337.514572\n",
"INFO:root:Epoch[11] Resetting Data Iterator\n",
"INFO:root:Epoch[11] Time cost=0.005\n",
"INFO:root:Epoch[11] Validation-mse=175.133366\n",
"INFO:root:Epoch[12] Resetting Data Iterator\n",
"INFO:root:Epoch[12] Time cost=0.019\n",
"INFO:root:Epoch[12] Validation-mse=107.841146\n",
"INFO:root:Epoch[13] Resetting Data Iterator\n",
"INFO:root:Epoch[13] Time cost=0.007\n",
"INFO:root:Epoch[13] Validation-mse=80.584860\n",
"INFO:root:Epoch[14] Resetting Data Iterator\n",
"INFO:root:Epoch[14] Time cost=0.007\n",
"INFO:root:Epoch[14] Validation-mse=69.974375\n"
]
}
],
"source": [
"model1 = mx.model.FeedForward(\n",
" symbol=net1,\n",
" num_epoch=15,\n",
" learning_rate=0.1\n",
")\n",
"model1.fit(\n",
" X=train_iter,\n",
" eval_data=valid_iter,\n",
" eval_metric='MSE'\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"82.7316956117\n"
]
},
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pred = model1.predict(test_iter)\n",
"print(mean_squared_error(y[flg_valid], pred))\n",
"mx.visualization.plot_network(net1, node_attrs={'fixedsize': 'false'})"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### 2. Custom function"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:root:Start training with [cpu(0)]\n",
"INFO:root:Epoch[0] Resetting Data Iterator\n",
"INFO:root:Epoch[0] Time cost=0.011\n",
"INFO:root:Epoch[0] Validation-mse=123389273633193984.000000\n",
"INFO:root:Epoch[1] Resetting Data Iterator\n",
"INFO:root:Epoch[1] Time cost=0.007\n",
"INFO:root:Epoch[1] Validation-mse=3472040824668160.000000\n",
"INFO:root:Epoch[2] Resetting Data Iterator\n",
"INFO:root:Epoch[2] Time cost=0.007\n",
"INFO:root:Epoch[2] Validation-mse=97658467254272.000000\n",
"INFO:root:Epoch[3] Resetting Data Iterator\n",
"INFO:root:Epoch[3] Time cost=0.007\n",
"INFO:root:Epoch[3] Validation-mse=2744262066176.000000\n",
"INFO:root:Epoch[4] Resetting Data Iterator\n",
"INFO:root:Epoch[4] Time cost=0.006\n",
"INFO:root:Epoch[4] Validation-mse=76978479104.000000\n",
"INFO:root:Epoch[5] Resetting Data Iterator\n",
"INFO:root:Epoch[5] Time cost=0.007\n",
"INFO:root:Epoch[5] Validation-mse=2157003904.000000\n",
"INFO:root:Epoch[6] Resetting Data Iterator\n",
"INFO:root:Epoch[6] Time cost=0.007\n",
"INFO:root:Epoch[6] Validation-mse=61591550.000000\n",
"INFO:root:Epoch[7] Resetting Data Iterator\n",
"INFO:root:Epoch[7] Time cost=0.006\n",
"INFO:root:Epoch[7] Validation-mse=2112167.125000\n",
"INFO:root:Epoch[8] Resetting Data Iterator\n",
"INFO:root:Epoch[8] Time cost=0.006\n",
"INFO:root:Epoch[8] Validation-mse=172325.460938\n",
"INFO:root:Epoch[9] Resetting Data Iterator\n",
"INFO:root:Epoch[9] Time cost=0.006\n",
"INFO:root:Epoch[9] Validation-mse=50338.280273\n",
"INFO:root:Epoch[10] Resetting Data Iterator\n",
"INFO:root:Epoch[10] Time cost=0.007\n",
"INFO:root:Epoch[10] Validation-mse=29807.058594\n",
"INFO:root:Epoch[11] Resetting Data Iterator\n",
"INFO:root:Epoch[11] Time cost=0.009\n",
"INFO:root:Epoch[11] Validation-mse=24232.414062\n",
"INFO:root:Epoch[12] Resetting Data Iterator\n",
"INFO:root:Epoch[12] Time cost=0.009\n",
"INFO:root:Epoch[12] Validation-mse=22382.235352\n",
"INFO:root:Epoch[13] Resetting Data Iterator\n",
"INFO:root:Epoch[13] Time cost=0.010\n",
"INFO:root:Epoch[13] Validation-mse=21697.045898\n",
"INFO:root:Epoch[14] Resetting Data Iterator\n",
"INFO:root:Epoch[14] Time cost=0.007\n",
"INFO:root:Epoch[14] Validation-mse=21428.656250\n"
]
}
],
"source": [
"loss = mx.sym.square(\n",
" mx.sym.Reshape(fco, shape=(-1,)) - label\n",
")\n",
"net2 = mx.sym.MakeLoss(loss)\n",
"model2 = mx.model.FeedForward(\n",
" symbol=net2,\n",
" num_epoch=15,\n",
" learning_rate=0.1\n",
")\n",
"model2.fit(\n",
" X=train_iter,\n",
" eval_data=valid_iter,\n",
" eval_metric='MSE'\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"76.778488739\n"
]
},
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pred = np.sqrt(model2.predict(test_iter))\n",
"print(mean_squared_error(y[flg_valid], pred))\n",
"mx.visualization.plot_network(net2, node_attrs={'fixedsize': 'false'})"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Classification (Boston House Prices)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"((506, 13), (506,))\n"
]
}
],
"source": [
"x = pd.DataFrame(boston.data, columns=boston.feature_names)\n",
"y = (boston.target >= boston.target.mean()).astype(np.int8)\n",
"print((x.shape, y.shape))"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"flg_train = np.random.choice([False, True], len(y), replace=True, p=[0.3, 0.7])\n",
"flg_valid = np.logical_not(flg_train)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"batch_size = 100\n",
"train_iter = mx.io.NDArrayIter(x[flg_train].values, y[flg_train],\n",
" batch_size, shuffle=True,\n",
" label_name='sm_label')\n",
"valid_iter = mx.io.NDArrayIter(x[flg_valid].values, y[flg_valid],\n",
" batch_size,\n",
" label_name='sm_label')\n",
"test_iter = mx.io.NDArrayIter(x[flg_valid].values)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"data = mx.sym.Variable('data')\n",
"label = mx.sym.Variable('sm_label')\n",
"fc = mx.sym.FullyConnected(data, num_hidden=3)\n",
"act = mx.sym.Activation(fc, act_type='relu')\n",
"fco = mx.sym.FullyConnected(act, num_hidden=2)\n",
"net1 = mx.sym.SoftmaxOutput(fco, name='sm')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### 1. Build-in function"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:root:Start training with [cpu(0)]\n",
"INFO:root:Epoch[0] Resetting Data Iterator\n",
"INFO:root:Epoch[0] Time cost=0.007\n",
"INFO:root:Epoch[0] Validation-cross-entropy=0.694896\n",
"INFO:root:Epoch[1] Resetting Data Iterator\n",
"INFO:root:Epoch[1] Time cost=0.005\n",
"INFO:root:Epoch[1] Validation-cross-entropy=0.691014\n",
"INFO:root:Epoch[2] Resetting Data Iterator\n",
"INFO:root:Epoch[2] Time cost=0.006\n",
"INFO:root:Epoch[2] Validation-cross-entropy=0.688830\n",
"INFO:root:Epoch[3] Resetting Data Iterator\n",
"INFO:root:Epoch[3] Time cost=0.008\n",
"INFO:root:Epoch[3] Validation-cross-entropy=0.687697\n",
"INFO:root:Epoch[4] Resetting Data Iterator\n",
"INFO:root:Epoch[4] Time cost=0.007\n",
"INFO:root:Epoch[4] Validation-cross-entropy=0.687203\n",
"INFO:root:Epoch[5] Resetting Data Iterator\n",
"INFO:root:Epoch[5] Time cost=0.006\n",
"INFO:root:Epoch[5] Validation-cross-entropy=0.687085\n",
"INFO:root:Epoch[6] Resetting Data Iterator\n",
"INFO:root:Epoch[6] Time cost=0.005\n",
"INFO:root:Epoch[6] Validation-cross-entropy=0.687178\n",
"INFO:root:Epoch[7] Resetting Data Iterator\n",
"INFO:root:Epoch[7] Time cost=0.006\n",
"INFO:root:Epoch[7] Validation-cross-entropy=0.687380\n",
"INFO:root:Epoch[8] Resetting Data Iterator\n",
"INFO:root:Epoch[8] Time cost=0.006\n",
"INFO:root:Epoch[8] Validation-cross-entropy=0.687629\n",
"INFO:root:Epoch[9] Resetting Data Iterator\n",
"INFO:root:Epoch[9] Time cost=0.007\n",
"INFO:root:Epoch[9] Validation-cross-entropy=0.687889\n",
"INFO:root:Epoch[10] Resetting Data Iterator\n",
"INFO:root:Epoch[10] Time cost=0.005\n",
"INFO:root:Epoch[10] Validation-cross-entropy=0.688140\n",
"INFO:root:Epoch[11] Resetting Data Iterator\n",
"INFO:root:Epoch[11] Time cost=0.011\n",
"INFO:root:Epoch[11] Validation-cross-entropy=0.688370\n",
"INFO:root:Epoch[12] Resetting Data Iterator\n",
"INFO:root:Epoch[12] Time cost=0.008\n",
"INFO:root:Epoch[12] Validation-cross-entropy=0.688575\n",
"INFO:root:Epoch[13] Resetting Data Iterator\n",
"INFO:root:Epoch[13] Time cost=0.008\n",
"INFO:root:Epoch[13] Validation-cross-entropy=0.688754\n",
"INFO:root:Epoch[14] Resetting Data Iterator\n",
"INFO:root:Epoch[14] Time cost=0.008\n",
"INFO:root:Epoch[14] Validation-cross-entropy=0.688909\n"
]
}
],
"source": [
"model1 = mx.model.FeedForward(\n",
" symbol=net1,\n",
" num_epoch=15,\n",
" learning_rate=0.1\n",
")\n",
"model1.fit(\n",
" X=train_iter,\n",
" eval_data=valid_iter,\n",
" eval_metric='CE'\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.687112347189\n",
"0.5\n"
]
},
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pred = model1.predict(test_iter)[:, 1]\n",
"print(log_loss(y[flg_valid], pred))\n",
"print(roc_auc_score(y[flg_valid], pred))\n",
"mx.visualization.plot_network(net1, node_attrs={'fixedsize': 'false'})"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### 2. Custom function"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"train_iter = mx.io.NDArrayIter(x[flg_train].values, y[flg_train],\n",
" batch_size, shuffle=True,\n",
" label_name='label')\n",
"valid_iter = mx.io.NDArrayIter(x[flg_valid].values, y[flg_valid],\n",
" batch_size,\n",
" label_name='label')\n",
"test_iter = mx.io.NDArrayIter(x[flg_valid].values)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"data = mx.sym.Variable('data')\n",
"label = mx.sym.Variable('label')\n",
"fc = mx.sym.FullyConnected(data, num_hidden=3)\n",
"act = mx.sym.Activation(fc, act_type='relu')\n",
"fco = mx.sym.FullyConnected(act, num_hidden=1)\n",
"p = mx.sym.Activation(fco, act_type='sigmoid')\n",
"eps = 1e-6\n",
"p = mx.sym.minimum(mx.sym.maximum(p, eps), 1.0-eps)\n",
"q = 1.0 - p\n",
"lp = mx.sym.Reshape(mx.sym.log(p), shape=(-1,))\n",
"lq = mx.sym.Reshape(mx.sym.log(q), shape=(-1,))\n",
"loss = label * lp + (1.0 - label) * lq\n",
"net2 = mx.sym.MakeLoss(loss)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:root:Start training with [cpu(0)]\n",
"INFO:root:Epoch[0] Resetting Data Iterator\n",
"INFO:root:Epoch[0] Time cost=0.005\n",
"INFO:root:Epoch[0] Validation-mae=8.105288\n",
"INFO:root:Epoch[1] Resetting Data Iterator\n",
"INFO:root:Epoch[1] Time cost=0.005\n",
"INFO:root:Epoch[1] Validation-mae=8.105288\n",
"INFO:root:Epoch[2] Resetting Data Iterator\n",
"INFO:root:Epoch[2] Time cost=0.010\n",
"INFO:root:Epoch[2] Validation-mae=8.105288\n",
"INFO:root:Epoch[3] Resetting Data Iterator\n",
"INFO:root:Epoch[3] Time cost=0.008\n",
"INFO:root:Epoch[3] Validation-mae=8.105288\n",
"INFO:root:Epoch[4] Resetting Data Iterator\n",
"INFO:root:Epoch[4] Time cost=0.007\n",
"INFO:root:Epoch[4] Validation-mae=8.105288\n",
"INFO:root:Epoch[5] Resetting Data Iterator\n",
"INFO:root:Epoch[5] Time cost=0.007\n",
"INFO:root:Epoch[5] Validation-mae=8.105288\n",
"INFO:root:Epoch[6] Resetting Data Iterator\n",
"INFO:root:Epoch[6] Time cost=0.008\n",
"INFO:root:Epoch[6] Validation-mae=8.105288\n",
"INFO:root:Epoch[7] Resetting Data Iterator\n",
"INFO:root:Epoch[7] Time cost=0.007\n",
"INFO:root:Epoch[7] Validation-mae=8.105288\n",
"INFO:root:Epoch[8] Resetting Data Iterator\n",
"INFO:root:Epoch[8] Time cost=0.007\n",
"INFO:root:Epoch[8] Validation-mae=8.105288\n",
"INFO:root:Epoch[9] Resetting Data Iterator\n",
"INFO:root:Epoch[9] Time cost=0.007\n",
"INFO:root:Epoch[9] Validation-mae=8.105288\n",
"INFO:root:Epoch[10] Resetting Data Iterator\n",
"INFO:root:Epoch[10] Time cost=0.006\n",
"INFO:root:Epoch[10] Validation-mae=8.105288\n",
"INFO:root:Epoch[11] Resetting Data Iterator\n",
"INFO:root:Epoch[11] Time cost=0.007\n",
"INFO:root:Epoch[11] Validation-mae=8.105288\n",
"INFO:root:Epoch[12] Resetting Data Iterator\n",
"INFO:root:Epoch[12] Time cost=0.007\n",
"INFO:root:Epoch[12] Validation-mae=8.105288\n",
"INFO:root:Epoch[13] Resetting Data Iterator\n",
"INFO:root:Epoch[13] Time cost=0.006\n",
"INFO:root:Epoch[13] Validation-mae=8.105288\n",
"INFO:root:Epoch[14] Resetting Data Iterator\n",
"INFO:root:Epoch[14] Time cost=0.006\n",
"INFO:root:Epoch[14] Validation-mae=8.105288\n"
]
}
],
"source": [
"model2 = mx.model.FeedForward(\n",
" symbol=net2,\n",
" num_epoch=15,\n",
" learning_rate=0.1\n",
")\n",
"model2.fit(\n",
" X=train_iter,\n",
" eval_data=valid_iter,\n",
" eval_metric='MAE'\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {
"collapsed": false,
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"6.0696948799\n",
"0.5\n"
]
},
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"\n",
"\n",
"\n"
],
"text/plain": [
""
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pred = np.exp(model2.predict(test_iter))\n",
"print(log_loss(y[flg_valid], pred))\n",
"print(roc_auc_score(y[flg_valid], pred))\n",
"mx.visualization.plot_network(net2, node_attrs={'fixedsize': 'false'})"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}