{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Run the benchmark"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"scrolled": false
},
"outputs": [],
"source": [
"!python benchmark.py --frameworks tensorflow gensim originalc --file data/text8 --epochs 4 --batch_size 16 --workers 7 --size 100 "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Load Report file"
]
},
{
"cell_type": "code",
"execution_count": 54,
"metadata": {
"ExecuteTime": {
"end_time": "2017-08-02T22:10:25.416057Z",
"start_time": "2017-08-02T22:10:25.199767Z"
}
},
"outputs": [
{
"data": {
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from plotly.offline import download_plotlyjs, init_notebook_mode, iplot\n",
"from plotly.graph_objs import *\n",
"import json\n",
"init_notebook_mode()\n",
"with open('./report.json','r') as f:\n",
" report = json.loads(f.read())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## System Information"
]
},
{
"cell_type": "code",
"execution_count": 55,
"metadata": {
"ExecuteTime": {
"end_time": "2017-08-02T22:10:28.493627Z",
"start_time": "2017-08-02T22:10:28.490236Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU INFO\n",
"Model name:Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz , Architecture:x86_64 , CPU(s):8 , MemTotal:16382544 kB , \n",
"GPU INFO\n",
"Model Name : GeForce GTX 1060, Total FB Memory : 3011 MiB, CUDA Version : CUDA Version 8.0.61\n"
]
}
],
"source": [
"print report['systeminfo']"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Training Parameters"
]
},
{
"cell_type": "code",
"execution_count": 56,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"sample=0.001, log_level='INFO', workers=7, negative=5, batch_size=16, epochs=4, frameworks=['tensorflow', 'gensim', 'originalc', 'dl4j'], window=5, file='data/text8', alpha=0.025, min_count=5, sg=1, size=100\n"
]
}
],
"source": [
"print report['trainingparams']"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Generate graphics"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Time to train(in seconds) and peak memory(in MiB) results"
]
},
{
"cell_type": "code",
"execution_count": 57,
"metadata": {
"ExecuteTime": {
"end_time": "2017-08-02T22:10:30.968396Z",
"start_time": "2017-08-02T22:10:30.732119Z"
}
},
"outputs": [
{
"data": {
"application/vnd.plotly.v1+json": {
"data": [
{
"orientation": "v",
"type": "bar",
"x": [
"tensorflow-gpu",
"tensorflow",
"dl4j",
"gensim",
"originalc"
],
"y": [
12058.6517849,
2864.14668012,
537.181186914,
89.6162199974,
117.926698208
]
}
],
"layout": {
"annotations": [
{
"showarrow": false,
"text": "12058.6517849",
"x": "tensorflow-gpu",
"xanchor": "center",
"y": 12058.6517849,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "2864.14668012",
"x": "tensorflow",
"xanchor": "center",
"y": 2864.14668012,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "537.181186914",
"x": "dl4j",
"xanchor": "center",
"y": 537.181186914,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "89.6162199974",
"x": "gensim",
"xanchor": "center",
"y": 89.6162199974,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "117.926698208",
"x": "originalc",
"xanchor": "center",
"y": 117.926698208,
"yanchor": "bottom"
}
],
"title": "Time Report",
"xaxis": {
"title": "Framework"
},
"yaxis": {
"title": "Training time (in seconds)"
}
}
},
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"X = list(report['time'].keys())\n",
"Y = list(report['time'].values())\n",
"data = [Bar(\n",
" x = X,\n",
" y = Y,\n",
" orientation = 'v',\n",
")]\n",
"layout = Layout(\n",
" title = 'Time Report',\n",
" xaxis = dict(title = 'Framework'),\n",
" yaxis = dict(title = 'Training time (in seconds)'),\n",
" annotations=[\n",
" dict(x=xi,y=yi,\n",
" text=str(yi),\n",
" xanchor='center',\n",
" yanchor='bottom',\n",
" showarrow=False,\n",
" ) for xi, yi in zip(X, Y)]\n",
")\n",
"fig = dict(data=data, layout=layout)\n",
"iplot(fig) "
]
},
{
"cell_type": "code",
"execution_count": 58,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.plotly.v1+json": {
"data": [
{
"orientation": "v",
"type": "bar",
"x": [
"tensorflow-gpu",
"tensorflow",
"dl4j",
"gensim",
"originalc"
],
"y": [
849.40234375,
1046.4765625,
2213.4765625,
177.65234375,
576.73046875
]
}
],
"layout": {
"annotations": [
{
"showarrow": false,
"text": "849.40234375",
"x": "tensorflow-gpu",
"xanchor": "center",
"y": 849.40234375,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "1046.4765625",
"x": "tensorflow",
"xanchor": "center",
"y": 1046.4765625,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "2213.4765625",
"x": "dl4j",
"xanchor": "center",
"y": 2213.4765625,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "177.65234375",
"x": "gensim",
"xanchor": "center",
"y": 177.65234375,
"yanchor": "bottom"
},
{
"showarrow": false,
"text": "576.73046875",
"x": "originalc",
"xanchor": "center",
"y": 576.73046875,
"yanchor": "bottom"
}
],
"title": "Memory Report",
"xaxis": {
"title": "Framework"
},
"yaxis": {
"title": "Peak memory (in MB)"
}
}
},
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"X = list(report['memory'].keys())\n",
"Y = list(report['memory'].values())\n",
"data = [Bar(\n",
" x = X,\n",
" y = Y,\n",
" orientation = 'v',\n",
")]\n",
"layout = Layout(\n",
" title = 'Memory Report',\n",
" xaxis = dict(title = 'Framework'),\n",
" yaxis = dict(title = 'Peak memory (in MB)'),\n",
" annotations=[\n",
" dict(x=xi,y=yi,\n",
" text=str(yi),\n",
" xanchor='center',\n",
" yanchor='bottom',\n",
" showarrow=False,\n",
" ) for xi, yi in zip(X, Y)]\n",
")\n",
"fig = dict(data=data, layout=layout)\n",
"iplot(fig) "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Results of evaluation on the popular **Word Similarities** task. This task measures how well the notion of word similarity according to humans is captured by the word vector representations. Two lists are obtained by sorting the word pairs according to human similarity and vector-space similarity. Spearman’s correlation/rho between these\n",
"ranked lists is the used to signify how well the vector space agrees with human judgement. "
]
},
{
"cell_type": "code",
"execution_count": 59,
"metadata": {
"ExecuteTime": {
"end_time": "2017-08-02T22:10:32.742187Z",
"start_time": "2017-08-02T22:10:32.101884Z"
}
},
"outputs": [
{
"data": {
"application/vnd.plotly.v1+json": {
"data": [
{
"name": "tensorflow",
"type": "bar",
"x": [
"EN-MC-30.txt",
"EN-MEN-TR-3k.txt",
"EN-MTurk-287.txt",
"EN-MTurk-771.txt",
"EN-RG-65.txt",
"EN-RW-STANFORD.txt",
"EN-SIMLEX-999.txt",
"EN-VERB-143.txt",
"EN-WS-353-ALL.txt",
"EN-WS-353-REL.txt",
"EN-WS-353-SIM.txt",
"EN-YP-130.txt"
],
"y": [
0.313341510726,
0.154052493852,
0.101561002149,
0.184105766648,
0.147392117508,
0.275997532272,
0.203732761405,
0.201555643421,
0.306226184912,
0.156922697539,
0.396452812343,
0.0517916846781
]
},
{
"name": "tensorflow-gpu",
"type": "bar",
"x": [
"EN-MC-30.txt",
"EN-MEN-TR-3k.txt",
"EN-MTurk-287.txt",
"EN-MTurk-771.txt",
"EN-RG-65.txt",
"EN-RW-STANFORD.txt",
"EN-SIMLEX-999.txt",
"EN-VERB-143.txt",
"EN-WS-353-ALL.txt",
"EN-WS-353-REL.txt",
"EN-WS-353-SIM.txt",
"EN-YP-130.txt"
],
"y": [
0.312673879667,
0.15477720377,
0.102687941289,
0.184906724377,
0.148987554129,
0.276176689978,
0.202910305301,
0.201372634066,
0.306792850071,
0.157747232482,
0.396964586982,
0.0566164062043
]
},
{
"name": "gensim",
"type": "bar",
"x": [
"EN-MC-30.txt",
"EN-MEN-TR-3k.txt",
"EN-MTurk-287.txt",
"EN-MTurk-771.txt",
"EN-RG-65.txt",
"EN-RW-STANFORD.txt",
"EN-SIMLEX-999.txt",
"EN-VERB-143.txt",
"EN-WS-353-ALL.txt",
"EN-WS-353-REL.txt",
"EN-WS-353-SIM.txt",
"EN-YP-130.txt"
],
"y": [
0.638922924215,
0.572162702124,
0.615465682319,
0.531067933339,
0.596715151884,
0.377758565211,
0.30751215466,
0.345557861412,
0.67760512661,
0.64432112493,
0.718558528594,
0.307469004802
]
},
{
"name": "dl4j",
"type": "bar",
"x": [
"EN-MC-30.txt",
"EN-MEN-TR-3k.txt",
"EN-MTurk-287.txt",
"EN-MTurk-771.txt",
"EN-RG-65.txt",
"EN-RW-STANFORD.txt",
"EN-SIMLEX-999.txt",
"EN-VERB-143.txt",
"EN-WS-353-ALL.txt",
"EN-WS-353-REL.txt",
"EN-WS-353-SIM.txt",
"EN-YP-130.txt"
],
"y": [
0.549237818517,
0.627651963449,
0.629652227188,
0.542225451574,
0.62611052526,
0.322276521194,
0.25533755791,
0.343092262852,
0.656941828666,
0.622602806244,
0.715721564486,
0.423814658771
]
},
{
"name": "originalc",
"type": "bar",
"x": [
"EN-MC-30.txt",
"EN-MEN-TR-3k.txt",
"EN-MTurk-287.txt",
"EN-MTurk-771.txt",
"EN-RG-65.txt",
"EN-RW-STANFORD.txt",
"EN-SIMLEX-999.txt",
"EN-VERB-143.txt",
"EN-WS-353-ALL.txt",
"EN-WS-353-REL.txt",
"EN-WS-353-SIM.txt",
"EN-YP-130.txt"
],
"y": [
0.573717624043,
0.596396450381,
0.64559414611,
0.548739299088,
0.552633019468,
0.377401624178,
0.311729324303,
0.371090683037,
0.681141650011,
0.654496713312,
0.713501933086,
0.303990376923
]
}
],
"layout": {
"barmode": "group",
"title": "Word Pairs Evaluation Report",
"xaxis": {
"tickangle": -45,
"title": "Dataset"
},
"yaxis": {
"title": "Spearman's Rho"
}
}
},
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"data = []\n",
"for framework in report['frameworks']:\n",
" X = [x[0] for x in report['wordpairs'][framework]]\n",
" Y = [x[1] for x in report['wordpairs'][framework]]\n",
" trace = Bar(\n",
" x = X,\n",
" y = Y,\n",
" name =framework\n",
" )\n",
" data.append(trace)\n",
"layout = Layout(\n",
" title = 'Word Pairs Evaluation Report',\n",
" xaxis = dict(title = 'Dataset', tickangle = -45),\n",
" yaxis = dict(title = 'Spearman\\'s Rho'),\n",
" barmode = 'group'\n",
")\n",
"fig = dict(data=data, layout=layout)\n",
"iplot(fig) "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Results of evaluation on the popular **Word Analogy** task. The aim of this task is to find the missing word b' in the relation: a is to a' as b is to b'. In other words we look at the most similar word vector to b' (= a' + b - a) and compare it with the human analogy and report the accuracy. "
]
},
{
"cell_type": "code",
"execution_count": 60,
"metadata": {
"ExecuteTime": {
"end_time": "2017-08-02T22:10:35.321449Z",
"start_time": "2017-08-02T22:10:34.601671Z"
},
"scrolled": false
},
"outputs": [
{
"data": {
"application/vnd.plotly.v1+json": {
"data": [
{
"name": "tensorflow",
"type": "bar",
"x": [
"capital-common-countries",
"capital-world",
"currency",
"city-in-state",
"family",
"gram1-adjective-to-adverb",
"gram2-opposite",
"gram3-comparative",
"gram4-superlative",
"gram5-present-participle",
"gram6-nationality-adjective",
"gram7-past-tense",
"gram8-plural",
"gram9-plural-verbs",
"total"
],
"y": [
5.92885375494,
2.82369146006,
0,
4.83768300446,
37.5816993464,
4.10052910053,
10.4575163399,
12.0634920635,
4.15019762846,
13.2056451613,
8.75273522976,
7.20720720721,
12.6008064516,
5.53846153846,
8.20019563091
]
},
{
"name": "tensorflow-gpu",
"type": "bar",
"x": [
"capital-common-countries",
"capital-world",
"currency",
"city-in-state",
"family",
"gram1-adjective-to-adverb",
"gram2-opposite",
"gram3-comparative",
"gram4-superlative",
"gram5-present-participle",
"gram6-nationality-adjective",
"gram7-past-tense",
"gram8-plural",
"gram9-plural-verbs",
"total"
],
"y": [
5.33596837945,
2.82369146006,
0,
4.96499045194,
36.9281045752,
4.36507936508,
9.15032679739,
11.4285714286,
4.54545454545,
12.7016129032,
8.53391684902,
6.90690690691,
12.8024193548,
4.76923076923,
7.98826214542
]
},
{
"name": "gensim",
"type": "bar",
"x": [
"capital-common-countries",
"capital-world",
"currency",
"city-in-state",
"family",
"gram1-adjective-to-adverb",
"gram2-opposite",
"gram3-comparative",
"gram4-superlative",
"gram5-present-participle",
"gram6-nationality-adjective",
"gram7-past-tense",
"gram8-plural",
"gram9-plural-verbs",
"total"
],
"y": [
51.581027668,
28.650137741,
17.5373134328,
14.1947803947,
65.0326797386,
15.0793650794,
15.6862745098,
55.1587301587,
34.5849802372,
30.8467741935,
68.7089715536,
27.5525525526,
37.8024193548,
32.9230769231,
35.7189435931
]
},
{
"name": "dl4j",
"type": "bar",
"x": [
"capital-common-countries",
"capital-world",
"currency",
"city-in-state",
"family",
"gram1-adjective-to-adverb",
"gram2-opposite",
"gram3-comparative",
"gram4-superlative",
"gram5-present-participle",
"gram6-nationality-adjective",
"gram7-past-tense",
"gram8-plural",
"gram9-plural-verbs",
"total"
],
"y": [
46.6666666667,
42.96875,
0,
20.3883495146,
0,
7.14285714286,
10,
70,
0,
23.2142857143,
46.7289719626,
36.6666666667,
66.6666666667,
13.8888888889,
31.4641744548
]
},
{
"name": "originalc",
"type": "bar",
"x": [
"capital-common-countries",
"capital-world",
"currency",
"city-in-state",
"family",
"gram1-adjective-to-adverb",
"gram2-opposite",
"gram3-comparative",
"gram4-superlative",
"gram5-present-participle",
"gram6-nationality-adjective",
"gram7-past-tense",
"gram8-plural",
"gram9-plural-verbs",
"total"
],
"y": [
57.7075098814,
34.7107438017,
20.8955223881,
15.849777212,
68.6274509804,
12.962962963,
17.3202614379,
57.7777777778,
33.2015810277,
32.3588709677,
70.8242159008,
26.2012012012,
44.6572580645,
34.1538461538,
38.0176067819
]
}
],
"layout": {
"barmode": "group",
"title": "Analogies Task(Questions&Answers) Report",
"xaxis": {
"tickangle": -45
},
"yaxis": {
"title": "Accuracy (in %)"
}
}
},
"text/html": [
""
],
"text/vnd.plotly.v1+html": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"data = []\n",
"for framework in report['frameworks']:\n",
" X = [x[0] for x in report['qa'][framework]]\n",
" Y = [x[1] for x in report['qa'][framework]]\n",
" trace = Bar(\n",
" x = X,\n",
" y = Y,\n",
" name =framework\n",
" )\n",
" data.append(trace)\n",
"layout = Layout(\n",
" title = 'Analogies Task(Questions&Answers) Report',\n",
" xaxis = dict(tickangle = -45),\n",
" yaxis = dict(title = 'Accuracy (in %)'),\n",
" barmode = 'group'\n",
")\n",
"fig = dict(data=data, layout=layout)\n",
"iplot(fig) "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.12"
}
},
"nbformat": 4,
"nbformat_minor": 2
}