{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Due to limited rendering on Github, the plots are not visible here. Click on [this](http://nbviewer.ipython.org/github/manneshiva/benchmark_ml_frameworks/blob/c2468939ca4847a48cbe7ca74d0adf57df3b8a61/visualize_report.ipynb) link to view them with nbviewer"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Run the benchmark"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"scrolled": false
},
"outputs": [],
"source": [
"# inside docker\n",
"!python benchmark.py --frameworks dl4j tensorflow gensim originalc --file /benmark_nn_frameworks/data/text8-split --epochs 4 --batch_size 32 --workers 7 --size 100 --platform local "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Load Report file"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2017-09-05T12:50:56.350010Z",
"start_time": "2017-09-05T12:50:55.952449Z"
}
},
"outputs": [
{
"data": {
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from plotly.offline import download_plotlyjs, init_notebook_mode, iplot\n",
"from plotly.graph_objs import *\n",
"import json\n",
"init_notebook_mode()\n",
"with open('./local-report.json','r') as f:\n",
" report = json.loads(f.read())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## System Information"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"ExecuteTime": {
"end_time": "2017-09-05T12:50:56.354291Z",
"start_time": "2017-09-05T12:50:56.351640Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU INFO\n",
"Model name:Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz, Architecture:x86_64, CPU(s):8, MemTotal:16382532 kB, \n",
"GPU INFO\n",
"Model Name : GeForce GTX 1060, Total FB Memory : 3011 MiB, CUDA Version : CUDA Version 8.0.61\n"
]
}
],
"source": [
"print report['systeminfo']"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Training Parameters"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"ExecuteTime": {
"end_time": "2017-09-05T12:50:56.720964Z",
"start_time": "2017-09-05T12:50:56.717532Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"epochs\": 4, \n",
" \"frameworks\": [\n",
" \"gensim\", \n",
" \"originalc\", \n",
" \"tensorflow\", \n",
" \"dl4j\"\n",
" ], \n",
" \"workers\": 3, \n",
" \"negative\": 5, \n",
" \"batch_size\": 32, \n",
" \"sample\": 0.001, \n",
" \"platform\": \"local\", \n",
" \"window\": 5, \n",
" \"fname\": \"/benmark_nn_frameworks/data/text8-split\", \n",
" \"min_count\": 5, \n",
" \"alpha\": 0.025, \n",
" \"sg\": 1, \n",
" \"size\": 100\n",
"}\n"
]
}
],
"source": [
"print json.dumps(report['trainingparams'], indent=2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Generate graphics"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Time to train(in seconds) and peak memory(in MiB) results"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2017-09-05T12:50:57.249712Z",
"start_time": "2017-09-05T12:50:57.222826Z"
}
},
"outputs": [
{
"data": {
"application/vnd.plotly.v1+json": {
"data": [
{
"orientation": "v",
"type": "bar",
"x": [
"tensorflow-gpu",
"tensorflow",
"dl4j",
"gensim",
"originalc"
],
"y": [
6323,
1584,
622,
152,
112
]
}
],
"layout": {
"annotations": [
{
"showarrow": false,
"text": "6323",
"x": "tensorflow-gpu",
"xanchor": "center",
"y": 6323,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "1584",
"x": "tensorflow",
"xanchor": "center",
"y": 1584,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "622",
"x": "dl4j",
"xanchor": "center",
"y": 622,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "152",
"x": "gensim",
"xanchor": "center",
"y": 152,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "112",
"x": "originalc",
"xanchor": "center",
"y": 112,
"yanchor": "bottom"
}
],
"title": "Time Report",
"xaxis": {
"title": "Framework"
},
"yaxis": {
"title": "Training time (in seconds)"
}
}
},
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"X = list(report['time'].keys())\n",
"Y = list(report['time'].values())\n",
"data = [Bar(\n",
" x = X,\n",
" y = Y,\n",
" orientation = 'v',\n",
")]\n",
"layout = Layout(\n",
" title = 'Time Report',\n",
" xaxis = dict(title = 'Framework'),\n",
" yaxis = dict(title = 'Training time (in seconds)'),\n",
" annotations=[\n",
" dict(x=xi,y=yi,\n",
" text=str(yi),\n",
" xanchor='center',\n",
" yanchor='bottom',\n",
" showarrow=False,\n",
" ) for xi, yi in zip(X, Y)]\n",
")\n",
"fig = dict(data=data, layout=layout)\n",
"iplot(fig) "
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"ExecuteTime": {
"end_time": "2017-09-05T12:50:57.417718Z",
"start_time": "2017-09-05T12:50:57.406605Z"
}
},
"outputs": [
{
"data": {
"application/vnd.plotly.v1+json": {
"data": [
{
"orientation": "v",
"type": "bar",
"x": [
"tensorflow-gpu",
"tensorflow",
"dl4j",
"gensim",
"originalc"
],
"y": [
912,
851,
2227,
173,
576
]
}
],
"layout": {
"annotations": [
{
"showarrow": false,
"text": "912",
"x": "tensorflow-gpu",
"xanchor": "center",
"y": 912,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "851",
"x": "tensorflow",
"xanchor": "center",
"y": 851,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "2227",
"x": "dl4j",
"xanchor": "center",
"y": 2227,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "173",
"x": "gensim",
"xanchor": "center",
"y": 173,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "576",
"x": "originalc",
"xanchor": "center",
"y": 576,
"yanchor": "bottom"
}
],
"title": "Memory Report",
"xaxis": {
"title": "Framework"
},
"yaxis": {
"title": "Peak memory (in MB)"
}
}
},
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"X = list(report['memory'].keys())\n",
"Y = list(report['memory'].values())\n",
"data = [Bar(\n",
" x = X,\n",
" y = Y,\n",
" orientation = 'v',\n",
")]\n",
"layout = Layout(\n",
" title = 'Memory Report',\n",
" xaxis = dict(title = 'Framework'),\n",
" yaxis = dict(title = 'Peak memory (in MB)'),\n",
" annotations=[\n",
" dict(x=xi,y=yi,\n",
" text=str(yi),\n",
" xanchor='center',\n",
" yanchor='bottom',\n",
" showarrow=False,\n",
" ) for xi, yi in zip(X, Y)]\n",
")\n",
"fig = dict(data=data, layout=layout)\n",
"iplot(fig) "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Results of evaluation on the popular **Word Similarities** task. This task measures how well the notion of word similarity according to humans is captured by the word vector representations. Two lists are obtained by sorting the word pairs according to human similarity and vector-space similarity. Spearman’s correlation/rho between these\n",
"ranked lists is the used to signify how well the vector space agrees with human judgement. "
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"ExecuteTime": {
"end_time": "2017-09-05T12:50:58.269375Z",
"start_time": "2017-09-05T12:50:58.259813Z"
}
},
"outputs": [
{
"data": {
"application/vnd.plotly.v1+json": {
"data": [
{
"name": "gensim",
"type": "bar",
"x": [
"EN-MC-30.txt",
"EN-MEN-TR-3k.txt",
"EN-MTurk-287.txt",
"EN-MTurk-771.txt",
"EN-RG-65.txt",
"EN-RW-STANFORD.txt",
"EN-SIMLEX-999.txt",
"EN-VERB-143.txt",
"EN-WS-353-ALL.txt",
"EN-WS-353-REL.txt",
"EN-WS-353-SIM.txt",
"EN-YP-130.txt"
],
"y": [
0.6273506525124154,
0.5749379521720063,
0.6112290512869081,
0.5374870212181332,
0.6067904434280808,
0.3727225662727729,
0.30381388653258024,
0.36026296472945274,
0.6763248574076292,
0.6478678014639323,
0.7171054671724001,
0.2717682602251255
]
},
{
"name": "originalc",
"type": "bar",
"x": [
"EN-MC-30.txt",
"EN-MEN-TR-3k.txt",
"EN-MTurk-287.txt",
"EN-MTurk-771.txt",
"EN-RG-65.txt",
"EN-RW-STANFORD.txt",
"EN-SIMLEX-999.txt",
"EN-VERB-143.txt",
"EN-WS-353-ALL.txt",
"EN-WS-353-REL.txt",
"EN-WS-353-SIM.txt",
"EN-YP-130.txt"
],
"y": [
0.5924112937169385,
0.5695438863743623,
0.6342773404945852,
0.5420032770412138,
0.5427544255904393,
0.37516647133304476,
0.3037663314703478,
0.32267968097690436,
0.6696017443780884,
0.6329157538755797,
0.7109321400908909,
0.27737576447058787
]
},
{
"name": "tensorflow",
"type": "bar",
"x": [
"EN-MC-30.txt",
"EN-MEN-TR-3k.txt",
"EN-MTurk-287.txt",
"EN-MTurk-771.txt",
"EN-RG-65.txt",
"EN-RW-STANFORD.txt",
"EN-SIMLEX-999.txt",
"EN-VERB-143.txt",
"EN-WS-353-ALL.txt",
"EN-WS-353-REL.txt",
"EN-WS-353-SIM.txt",
"EN-YP-130.txt"
],
"y": [
0.4444197421310016,
0.16052817248358836,
0.3756825298371262,
0.17302795364211315,
0.22159084807366772,
-0.004982551617814706,
0.07736498403227814,
0.22515982799933462,
0.2862875912104748,
0.2727734616934233,
0.3324351193126426,
0.1350227033351287
]
},
{
"name": "dl4j",
"type": "bar",
"x": [
"EN-MC-30.txt",
"EN-MEN-TR-3k.txt",
"EN-MTurk-287.txt",
"EN-MTurk-771.txt",
"EN-RG-65.txt",
"EN-RW-STANFORD.txt",
"EN-SIMLEX-999.txt",
"EN-VERB-143.txt",
"EN-WS-353-ALL.txt",
"EN-WS-353-REL.txt",
"EN-WS-353-SIM.txt",
"EN-YP-130.txt"
],
"y": [
0.5748303424759025,
0.6307647140809282,
0.6271122918339227,
0.5525747949370353,
0.6386773205697595,
0.3489887119355651,
0.2500710053321096,
0.36628618470156765,
0.6542508629230332,
0.6300186879363063,
0.7129865185135347,
0.3909341267017549
]
},
{
"name": "tensorflow-gpu",
"type": "bar",
"x": [
"EN-MC-30.txt",
"EN-MEN-TR-3k.txt",
"EN-MTurk-287.txt",
"EN-MTurk-771.txt",
"EN-RG-65.txt",
"EN-RW-STANFORD.txt",
"EN-SIMLEX-999.txt",
"EN-VERB-143.txt",
"EN-WS-353-ALL.txt",
"EN-WS-353-REL.txt",
"EN-WS-353-SIM.txt",
"EN-YP-130.txt"
],
"y": [
0.4551018390875806,
0.15720353365948944,
0.3713067996226006,
0.1658818606591078,
0.22810372633838347,
-0.002939107285440279,
0.07696690937731122,
0.2268109343756639,
0.28691045112688235,
0.27731504417242325,
0.3328180403085004,
0.1410947562036659
]
}
],
"layout": {
"barmode": "group",
"title": "Word Pairs Evaluation Report",
"xaxis": {
"tickangle": -45,
"title": "Dataset"
},
"yaxis": {
"title": "Spearman's Rho"
}
}
},
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"data = []\n",
"for framework in report['frameworks']:\n",
" X = [x[0] for x in report['wordpairs'][framework]]\n",
" Y = [x[1] for x in report['wordpairs'][framework]]\n",
" trace = Bar(\n",
" x = X,\n",
" y = Y,\n",
" name =framework\n",
" )\n",
" data.append(trace)\n",
"layout = Layout(\n",
" title = 'Word Pairs Evaluation Report',\n",
" xaxis = dict(title = 'Dataset', tickangle = -45),\n",
" yaxis = dict(title = 'Spearman\\'s Rho'),\n",
" barmode = 'group'\n",
")\n",
"fig = dict(data=data, layout=layout)\n",
"iplot(fig) "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Results of evaluation on the popular **Word Analogy** task. The aim of this task is to find the missing word b' in the relation: a is to a' as b is to b'. In other words we look at the most similar word vector to b' (= a' + b - a) and compare it with the human analogy and report the accuracy. "
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"ExecuteTime": {
"end_time": "2017-09-05T12:51:00.324281Z",
"start_time": "2017-09-05T12:51:00.311329Z"
},
"scrolled": false
},
"outputs": [
{
"data": {
"application/vnd.plotly.v1+json": {
"data": [
{
"name": "gensim",
"type": "bar",
"x": [
"capital-common-countries",
"capital-world",
"currency",
"city-in-state",
"family",
"gram1-adjective-to-adverb",
"gram2-opposite",
"gram3-comparative",
"gram4-superlative",
"gram5-present-participle",
"gram6-nationality-adjective",
"gram7-past-tense",
"gram8-plural",
"gram9-plural-verbs",
"total"
],
"y": [
"47.628458498",
"25.9641873278",
"17.5373134328",
"13.112667091",
"65.0326797386",
"13.8888888889",
"16.339869281",
"56.0317460317",
"34.5849802372",
"30.9475806452",
"67.3231218089",
"25.9009009009",
"40.9274193548",
"32.4615384615",
"35.0342354092"
]
},
{
"name": "originalc",
"type": "bar",
"x": [
"capital-common-countries",
"capital-world",
"currency",
"city-in-state",
"family",
"gram1-adjective-to-adverb",
"gram2-opposite",
"gram3-comparative",
"gram4-superlative",
"gram5-present-participle",
"gram6-nationality-adjective",
"gram7-past-tense",
"gram8-plural",
"gram9-plural-verbs",
"total"
],
"y": [
"45.0592885375",
"25.8953168044",
"19.4029850746",
"13.3036282623",
"65.0326797386",
"13.4920634921",
"19.2810457516",
"52.3015873016",
"33.2015810277",
"30.1411290323",
"66.8854850474",
"25.0",
"38.810483871",
"33.0769230769",
"34.2435604826"
]
},
{
"name": "tensorflow",
"type": "bar",
"x": [
"capital-common-countries",
"capital-world",
"currency",
"city-in-state",
"family",
"gram1-adjective-to-adverb",
"gram2-opposite",
"gram3-comparative",
"gram4-superlative",
"gram5-present-participle",
"gram6-nationality-adjective",
"gram7-past-tense",
"gram8-plural",
"gram9-plural-verbs",
"total"
],
"y": [
"2.17391304348",
"0.964187327824",
"0.746268656716",
"0.8911521324",
"22.5490196078",
"1.5873015873",
"1.30718954248",
"7.85714285714",
"0.98814229249",
"1.71370967742",
"6.78336980306",
"1.8018018018",
"2.52016129032",
"1.69230769231",
"3.2605151614"
]
},
{
"name": "dl4j",
"type": "bar",
"x": [
"capital-common-countries",
"capital-world",
"currency",
"city-in-state",
"family",
"gram1-adjective-to-adverb",
"gram2-opposite",
"gram3-comparative",
"gram4-superlative",
"gram5-present-participle",
"gram6-nationality-adjective",
"gram7-past-tense",
"gram8-plural",
"gram9-plural-verbs",
"total"
],
"y": [
"56.6666666667",
"42.1875",
"0.0",
"27.1844660194",
"0.0",
"10.7142857143",
"10.0",
"70.0",
"16.6666666667",
"23.2142857143",
"40.1869158879",
"36.6666666667",
"66.6666666667",
"25.0",
"33.4890965732"
]
},
{
"name": "tensorflow-gpu",
"type": "bar",
"x": [
"capital-common-countries",
"capital-world",
"currency",
"city-in-state",
"family",
"gram1-adjective-to-adverb",
"gram2-opposite",
"gram3-comparative",
"gram4-superlative",
"gram5-present-participle",
"gram6-nationality-adjective",
"gram7-past-tense",
"gram8-plural",
"gram9-plural-verbs",
"total"
],
"y": [
"1.97628458498",
"0.757575757576",
"0.746268656716",
"0.763844684914",
"24.8366013072",
"1.85185185185",
"0.326797385621",
"8.09523809524",
"1.38339920949",
"1.31048387097",
"6.56455142232",
"1.95195195195",
"2.52016129032",
"1.69230769231",
"3.2605151614"
]
}
],
"layout": {
"barmode": "group",
"title": "Analogies Task(Questions&Answers) Report",
"xaxis": {
"tickangle": -45
},
"yaxis": {
"title": "Accuracy (in %)"
}
}
},
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"data = []\n",
"for framework in report['frameworks']:\n",
" X = [x[0] for x in report['qa'][framework]]\n",
" Y = [x[1] for x in report['qa'][framework]]\n",
" trace = Bar(\n",
" x = X,\n",
" y = Y,\n",
" name =framework\n",
" )\n",
" data.append(trace)\n",
"layout = Layout(\n",
" title = 'Analogies Task(Questions&Answers) Report',\n",
" xaxis = dict(tickangle = -45),\n",
" yaxis = dict(title = 'Accuracy (in %)'),\n",
" barmode = 'group'\n",
")\n",
"fig = dict(data=data, layout=layout)\n",
"iplot(fig) "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.12"
}
},
"nbformat": 4,
"nbformat_minor": 2
}