{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# High-level CNTK Example"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import os\n",
"import sys\n",
"import cntk\n",
"from cntk.layers import Convolution2D, MaxPooling, Dense, Dropout\n",
"from common.params import *\n",
"from common.utils import *"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"OS: linux\n",
"Python: 3.5.2 |Anaconda custom (64-bit)| (default, Jul 2 2016, 17:53:06) \n",
"[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
"Numpy: 1.13.3\n",
"CNTK: 2.2\n",
"GPU: ['Tesla K80']\n"
]
}
],
"source": [
"print(\"OS: \", sys.platform)\n",
"print(\"Python: \", sys.version)\n",
"print(\"Numpy: \", np.__version__)\n",
"print(\"CNTK: \", cntk.__version__)\n",
"print(\"GPU: \", get_gpu_name())"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"def create_symbol():\n",
" # Weight initialiser from uniform distribution\n",
" # Activation (unless states) is None\n",
" with cntk.layers.default_options(init = cntk.glorot_uniform(), activation = cntk.relu):\n",
" x = Convolution2D(filter_shape=(3, 3), num_filters=50, pad=True)(features)\n",
" x = Convolution2D(filter_shape=(3, 3), num_filters=50, pad=True)(x)\n",
" x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x)\n",
" x = Dropout(0.25)(x)\n",
"\n",
" x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x)\n",
" x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x)\n",
" x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x)\n",
" x = Dropout(0.25)(x) \n",
" \n",
" x = Dense(512)(x)\n",
" x = Dropout(0.5)(x)\n",
" x = Dense(N_CLASSES, activation=None)(x)\n",
" return x"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"def init_model(m):\n",
" # Loss (dense labels); check if support for sparse labels\n",
" loss = cntk.cross_entropy_with_softmax(m, labels)\n",
" # Momentum SGD\n",
" # https://github.com/Microsoft/CNTK/blob/master/Manual/Manual_How_to_use_learners.ipynb\n",
" # unit_gain=False: momentum_direction = momentum*old_momentum_direction + gradient\n",
" # if unit_gain=True then ...(1-momentum)*gradient\n",
" learner = cntk.momentum_sgd(m.parameters, \n",
" lr=cntk.learning_rate_schedule(LR, cntk.UnitType.minibatch) , \n",
" momentum=cntk.momentum_schedule(MOMENTUM),\n",
" unit_gain=False)\n",
" return loss, learner"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Preparing train set...\n",
"Preparing test set...\n",
"(50000, 3, 32, 32) (10000, 3, 32, 32) (50000, 10) (10000, 10)\n",
"float32 float32 float32 float32\n",
"CPU times: user 846 ms, sys: 543 ms, total: 1.39 s\n",
"Wall time: 1.39 s\n"
]
}
],
"source": [
"%%time\n",
"# Data into format for library\n",
"x_train, x_test, y_train, y_test = cifar_for_library(channel_first=True, one_hot=True)\n",
"# CNTK format\n",
"y_train = y_train.astype(np.float32)\n",
"y_test = y_test.astype(np.float32)\n",
"print(x_train.shape, x_test.shape, y_train.shape, y_test.shape)\n",
"print(x_train.dtype, x_test.dtype, y_train.dtype, y_test.dtype)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 17.1 ms, sys: 32.7 ms, total: 49.8 ms\n",
"Wall time: 78.5 ms\n"
]
}
],
"source": [
"%%time\n",
"# Placeholders\n",
"features = cntk.input_variable((3, 32, 32), np.float32)\n",
"labels = cntk.input_variable(N_CLASSES, np.float32)\n",
"# Load symbol\n",
"sym = create_symbol()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 78.9 ms, sys: 221 ms, total: 300 ms\n",
"Wall time: 306 ms\n"
]
}
],
"source": [
"%%time\n",
"loss, learner = init_model(sym)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 2min 8s, sys: 24.9 s, total: 2min 32s\n",
"Wall time: 2min 51s\n"
]
},
{
"data": {
"text/plain": [
"{'epoch_summaries': [{'loss': 1.8166315625, 'metric': 0.0, 'samples': 50000},\n",
" {'loss': 1.3537084375, 'metric': 0.0, 'samples': 50000},\n",
" {'loss': 1.12093609375, 'metric': 0.0, 'samples': 50000},\n",
" {'loss': 0.97167546875, 'metric': 0.0, 'samples': 50000},\n",
" {'loss': 0.86488921875, 'metric': 0.0, 'samples': 50000},\n",
" {'loss': 0.769997734375, 'metric': 0.0, 'samples': 50000},\n",
" {'loss': 0.707360078125, 'metric': 0.0, 'samples': 50000},\n",
" {'loss': 0.64719390625, 'metric': 0.0, 'samples': 50000},\n",
" {'loss': 0.592496171875, 'metric': 0.0, 'samples': 50000},\n",
" {'loss': 0.5582487109375, 'metric': 0.0, 'samples': 50000}],\n",
" 'updates': [{'loss': 1.8166693029269365, 'metric': 0.0, 'samples': 49984},\n",
" {'loss': 1.353612999909971, 'metric': 0.0, 'samples': 49984},\n",
" {'loss': 1.120971445237476, 'metric': 0.0, 'samples': 49984},\n",
" {'loss': 0.9715477702864916, 'metric': 0.0, 'samples': 49984},\n",
" {'loss': 0.8647872006542093, 'metric': 0.0, 'samples': 49984},\n",
" {'loss': 0.7700418821522887, 'metric': 0.0, 'samples': 49984},\n",
" {'loss': 0.7073556506832186, 'metric': 0.0, 'samples': 49984},\n",
" {'loss': 0.6472827945567582, 'metric': 0.0, 'samples': 49984},\n",
" {'loss': 0.5925403941761364, 'metric': 0.0, 'samples': 49984},\n",
" {'loss': 0.5583446859244958, 'metric': 0.0, 'samples': 49984}]}"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"# 171s\n",
"loss.train((x_train, y_train), \n",
" minibatch_size=BATCHSIZE, \n",
" max_epochs=EPOCHS,\n",
" parameter_learners=[learner])"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 964 ms, sys: 220 ms, total: 1.18 s\n",
"Wall time: 1.43 s\n"
]
}
],
"source": [
"%%time\n",
"# Predict and then score accuracy\n",
"# (We don't need softmax -> monotonic function)\n",
"n_samples = (y_test.shape[0]//BATCHSIZE)*BATCHSIZE\n",
"y_guess = np.zeros(n_samples, dtype=np.int)\n",
"y_truth = np.argmax(y_test[:n_samples], axis=-1)\n",
"c = 0\n",
"for data, label in yield_mb(x_test, y_test, BATCHSIZE):\n",
" predicted_label_probs = sym.eval({features : data})\n",
" y_guess[c*BATCHSIZE:(c+1)*BATCHSIZE] = np.argmax(predicted_label_probs, axis=-1)\n",
" c += 1"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Accuracy: 0.773237179487\n"
]
}
],
"source": [
"print(\"Accuracy: \", sum(y_guess == y_truth)/len(y_guess))"
]
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python [default]",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}