{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Source: https://github.com/vgrabovets/benchmarkit"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from benchmarkit import benchmark, benchmark_analyze, benchmark_run"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Benchmark time"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"N = 10000\n",
"seq_list = list(range(N))\n",
"seq_set = set(range(N))\n",
"\n",
"SAVE_PATH = '/tmp/benchmark_time.jsonl'"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"@benchmark(num_iters=100, save_params=True)\n",
"def search_in_list(num_items=N):\n",
" return num_items - 1 in seq_list\n",
"\n",
"\n",
"@benchmark(num_iters=100, save_params=True)\n",
"def search_in_set(num_items=N):\n",
" return num_items - 1 in seq_set"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"File: \u001b[36mbenchmark_time.jsonl\u001b[0m\n",
"Function: \u001b[36msearch_in_list\u001b[0m\n",
" date time branch commit commit_date comment best_time mean_time num_items mean_time_diff\n",
"0 2019-05-21 18:50:44 master 3a1e755 2019-05-20 initial benchmark search 0.1022 0.1125 10000 NaN\n",
"-------------------------------------------------------------------------------------------------------------------------------\n",
"\n",
"File: \u001b[36mbenchmark_time.jsonl\u001b[0m\n",
"Function: \u001b[36msearch_in_set\u001b[0m\n",
" date time branch commit commit_date comment best_time mean_time num_items mean_time_diff\n",
"0 2019-05-21 18:50:47 master 3a1e755 2019-05-20 initial benchmark search 0.003 0.0041 10000 NaN\n",
"-------------------------------------------------------------------------------------------------------------------------------\n",
"\n",
"File: \u001b[36mbenchmark_time.jsonl\u001b[0m\n",
"\u001b[36mTotal\u001b[0m\n",
" date time branch commit commit_date comment best_time mean_time mean_time_diff\n",
"0 2019-05-21 18:50:44 master 3a1e755 2019-05-20 initial benchmark search 0.1052 0.1166 NaN\n",
"--------------------------------------------------------------------------------------------------------------------\n"
]
}
],
"source": [
"benchmark_results = benchmark_run([search_in_list, search_in_set], SAVE_PATH, comment='initial benchmark search', extra_fields=['num_items'])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Change `N` and repeat benchmark (enough to change `N` in cell 2 and restart cells 3 and 4):"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"File: \u001b[36mbenchmark_time.jsonl\u001b[0m\n",
"Function: \u001b[36msearch_in_list\u001b[0m\n",
" date time branch commit commit_date comment best_time mean_time num_items mean_time_diff\n",
"\u001b[32m0 2019-05-21 18:50:44 master 3a1e755 2019-05-20 initial benchmark search 0.1022 0.1125 10000 NaN\u001b[0m\n",
"\u001b[31m1 2019-05-21 18:51:02 master 3a1e755 2019-05-20 million items 9.6100 10.1782 1000000 10.0657\u001b[0m\n",
"-------------------------------------------------------------------------------------------------------------------------------\n",
"\n",
"File: \u001b[36mbenchmark_time.jsonl\u001b[0m\n",
"Function: \u001b[36msearch_in_set\u001b[0m\n",
" date time branch commit commit_date comment best_time mean_time num_items mean_time_diff\n",
"\u001b[32m0 2019-05-21 18:50:47 master 3a1e755 2019-05-20 initial benchmark search 0.0030 0.0041 10000 NaN\u001b[0m\n",
"\u001b[31m1 2019-05-21 18:51:06 master 3a1e755 2019-05-20 million items 0.0032 0.0043 1000000 0.0002\u001b[0m\n",
"-------------------------------------------------------------------------------------------------------------------------------\n",
"\n",
"File: \u001b[36mbenchmark_time.jsonl\u001b[0m\n",
"\u001b[36mTotal\u001b[0m\n",
" date time branch commit commit_date comment best_time mean_time mean_time_diff\n",
"\u001b[32m0 2019-05-21 18:50:44 master 3a1e755 2019-05-20 initial benchmark search 0.1052 0.1166 NaN\u001b[0m\n",
"\u001b[31m1 2019-05-21 18:51:02 master 3a1e755 2019-05-20 million items 9.6132 10.1825 10.0659\u001b[0m\n",
"--------------------------------------------------------------------------------------------------------------------\n"
]
}
],
"source": [
"N = 1000000\n",
"seq_list = list(range(N))\n",
"seq_set = set(range(N))\n",
"\n",
"@benchmark(num_iters=100, save_params=True)\n",
"def search_in_list(num_items=N):\n",
" return num_items - 1 in seq_list\n",
"\n",
"\n",
"@benchmark(num_iters=100, save_params=True)\n",
"def search_in_set(num_items=N):\n",
" return num_items - 1 in seq_set\n",
"\n",
"benchmark_results = benchmark_run([search_in_list, search_in_set], SAVE_PATH, comment='million items', extra_fields=['num_items'])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`benchmark_results` contains benchmark data for the last run"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[{'name': 'search_in_list',\n",
" 'best_time': 9.61,\n",
" 'mean_time': 10.1782,\n",
" 'date': '2019-05-21',\n",
" 'time': '18:51:02',\n",
" 'branch': 'master',\n",
" 'commit': '3a1e755',\n",
" 'commit_date': '2019-05-20',\n",
" 'num_items': 1000000,\n",
" 'comment': 'million items',\n",
" '_id': 'b443bfb9-78ef-4894-9fa5-aaa41025731c'},\n",
" {'name': 'search_in_set',\n",
" 'best_time': 0.0032,\n",
" 'mean_time': 0.0043,\n",
" 'date': '2019-05-21',\n",
" 'time': '18:51:06',\n",
" 'branch': 'master',\n",
" 'commit': '3a1e755',\n",
" 'commit_date': '2019-05-20',\n",
" 'num_items': 1000000,\n",
" 'comment': 'million items',\n",
" '_id': 'b443bfb9-78ef-4894-9fa5-aaa41025731c'}]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"benchmark_results"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"run `benchmark_run` from command line (without `!` in the real terminal):"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r\n",
"File: \u001b[36mbenchmark_functions.jsonl\u001b[0m\r\n",
"Function: \u001b[36msearch_in_list\u001b[0m\r\n",
" date time branch commit commit_date comment best_time mean_time num_items mean_time_diff\r\n",
"0 2019-05-21 18:51:16 master 3a1e755 2019-05-20 million items 9.591 10.0661 1000000 NaN\r\n",
"--------------------------------------------------------------------------------------------------------------------\r\n",
"\r\n",
"File: \u001b[36mbenchmark_functions.jsonl\u001b[0m\r\n",
"Function: \u001b[36msearch_in_set\u001b[0m\r\n",
" date time branch commit commit_date comment best_time mean_time num_items mean_time_diff\r\n",
"0 2019-05-21 18:51:19 master 3a1e755 2019-05-20 million items 0.0022 0.0032 1000000 NaN\r\n",
"--------------------------------------------------------------------------------------------------------------------\r\n",
"\r\n",
"File: \u001b[36mbenchmark_functions.jsonl\u001b[0m\r\n",
"\u001b[36mTotal\u001b[0m\r\n",
" date time branch commit commit_date comment best_time mean_time mean_time_diff\r\n",
"0 2019-05-21 18:51:16 master 3a1e755 2019-05-20 million items 9.5932 10.0693 NaN\r\n",
"---------------------------------------------------------------------------------------------------------\r\n"
]
}
],
"source": [
"!benchmark_run ../test_data/time/benchmark_functions.py --save_dir /tmp/ --comment \"million items\" --extra_fields num_items"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`benchmark_analyze` outputs results of benchmark stored in the file"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"File: \u001b[36mbenchmark_time.jsonl\u001b[0m\n",
"Function: \u001b[36msearch_in_list\u001b[0m\n",
" date time branch commit commit_date comment best_time mean_time num_items mean_time_diff\n",
"\u001b[32m0 2019-05-21 18:50:44 master 3a1e755 2019-05-20 initial benchmark search 0.1022 0.1125 10000 NaN\u001b[0m\n",
"\u001b[31m1 2019-05-21 18:51:02 master 3a1e755 2019-05-20 million items 9.6100 10.1782 1000000 10.0657\u001b[0m\n",
"-------------------------------------------------------------------------------------------------------------------------------\n",
"\n",
"File: \u001b[36mbenchmark_time.jsonl\u001b[0m\n",
"Function: \u001b[36msearch_in_set\u001b[0m\n",
" date time branch commit commit_date comment best_time mean_time num_items mean_time_diff\n",
"\u001b[32m0 2019-05-21 18:50:47 master 3a1e755 2019-05-20 initial benchmark search 0.0030 0.0041 10000 NaN\u001b[0m\n",
"\u001b[31m1 2019-05-21 18:51:06 master 3a1e755 2019-05-20 million items 0.0032 0.0043 1000000 0.0002\u001b[0m\n",
"-------------------------------------------------------------------------------------------------------------------------------\n",
"\n",
"File: \u001b[36mbenchmark_time.jsonl\u001b[0m\n",
"\u001b[36mTotal\u001b[0m\n",
" date time branch commit commit_date comment best_time mean_time mean_time_diff\n",
"\u001b[32m0 2019-05-21 18:50:44 master 3a1e755 2019-05-20 initial benchmark search 0.1052 0.1166 NaN\u001b[0m\n",
"\u001b[31m1 2019-05-21 18:51:02 master 3a1e755 2019-05-20 million items 9.6132 10.1825 10.0659\u001b[0m\n",
"--------------------------------------------------------------------------------------------------------------------\n"
]
}
],
"source": [
"benchmark_df = benchmark_analyze(SAVE_PATH, extra_fields=['num_items'])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`benchmark_df` contains pandas DataFrame with the results"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>date</th>\n",
" <th>time</th>\n",
" <th>name</th>\n",
" <th>branch</th>\n",
" <th>commit</th>\n",
" <th>commit_date</th>\n",
" <th>comment</th>\n",
" <th>best_time</th>\n",
" <th>mean_time</th>\n",
" <th>num_items</th>\n",
" <th>mean_time_diff</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2019-05-21</td>\n",
" <td>18:50:44</td>\n",
" <td>search_in_list</td>\n",
" <td>master</td>\n",
" <td>3a1e755</td>\n",
" <td>2019-05-20</td>\n",
" <td>initial benchmark search</td>\n",
" <td>0.1022</td>\n",
" <td>0.1125</td>\n",
" <td>10000.0</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2019-05-21</td>\n",
" <td>18:51:02</td>\n",
" <td>search_in_list</td>\n",
" <td>master</td>\n",
" <td>3a1e755</td>\n",
" <td>2019-05-20</td>\n",
" <td>million items</td>\n",
" <td>9.6100</td>\n",
" <td>10.1782</td>\n",
" <td>1000000.0</td>\n",
" <td>10.0657</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2019-05-21</td>\n",
" <td>18:50:47</td>\n",
" <td>search_in_set</td>\n",
" <td>master</td>\n",
" <td>3a1e755</td>\n",
" <td>2019-05-20</td>\n",
" <td>initial benchmark search</td>\n",
" <td>0.0030</td>\n",
" <td>0.0041</td>\n",
" <td>10000.0</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2019-05-21</td>\n",
" <td>18:51:06</td>\n",
" <td>search_in_set</td>\n",
" <td>master</td>\n",
" <td>3a1e755</td>\n",
" <td>2019-05-20</td>\n",
" <td>million items</td>\n",
" <td>0.0032</td>\n",
" <td>0.0043</td>\n",
" <td>1000000.0</td>\n",
" <td>0.0002</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2019-05-21</td>\n",
" <td>18:50:44</td>\n",
" <td>total</td>\n",
" <td>master</td>\n",
" <td>3a1e755</td>\n",
" <td>2019-05-20</td>\n",
" <td>initial benchmark search</td>\n",
" <td>0.1052</td>\n",
" <td>0.1166</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2019-05-21</td>\n",
" <td>18:51:02</td>\n",
" <td>total</td>\n",
" <td>master</td>\n",
" <td>3a1e755</td>\n",
" <td>2019-05-20</td>\n",
" <td>million items</td>\n",
" <td>9.6132</td>\n",
" <td>10.1825</td>\n",
" <td>NaN</td>\n",
" <td>10.0659</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" date time name branch commit commit_date \\\n",
"0 2019-05-21 18:50:44 search_in_list master 3a1e755 2019-05-20 \n",
"1 2019-05-21 18:51:02 search_in_list master 3a1e755 2019-05-20 \n",
"0 2019-05-21 18:50:47 search_in_set master 3a1e755 2019-05-20 \n",
"1 2019-05-21 18:51:06 search_in_set master 3a1e755 2019-05-20 \n",
"0 2019-05-21 18:50:44 total master 3a1e755 2019-05-20 \n",
"1 2019-05-21 18:51:02 total master 3a1e755 2019-05-20 \n",
"\n",
" comment best_time mean_time num_items mean_time_diff \n",
"0 initial benchmark search 0.1022 0.1125 10000.0 NaN \n",
"1 million items 9.6100 10.1782 1000000.0 10.0657 \n",
"0 initial benchmark search 0.0030 0.0041 10000.0 NaN \n",
"1 million items 0.0032 0.0043 1000000.0 0.0002 \n",
"0 initial benchmark search 0.1052 0.1166 NaN NaN \n",
"1 million items 9.6132 10.1825 NaN 10.0659 "
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"benchmark_df"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"run `benchmark_analyze` from command line (without `!` in the real terminal):"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r\n",
"File: \u001b[36mbenchmark_time.jsonl\u001b[0m\r\n",
"Function: \u001b[36msearch_in_list\u001b[0m\r\n",
" date time branch commit commit_date comment best_time mean_time num_items mean_time_diff\r\n",
"\u001b[32m0 2019-05-21 18:50:44 master 3a1e755 2019-05-20 initial benchmark search 0.1022 0.1125 10000 NaN\u001b[0m\r\n",
"\u001b[31m1 2019-05-21 18:51:02 master 3a1e755 2019-05-20 million items 9.6100 10.1782 1000000 10.0657\u001b[0m\r\n",
"-------------------------------------------------------------------------------------------------------------------------------\r\n",
"\r\n",
"File: \u001b[36mbenchmark_time.jsonl\u001b[0m\r\n",
"Function: \u001b[36msearch_in_set\u001b[0m\r\n",
" date time branch commit commit_date comment best_time mean_time num_items mean_time_diff\r\n",
"\u001b[32m0 2019-05-21 18:50:47 master 3a1e755 2019-05-20 initial benchmark search 0.0030 0.0041 10000 NaN\u001b[0m\r\n",
"\u001b[31m1 2019-05-21 18:51:06 master 3a1e755 2019-05-20 million items 0.0032 0.0043 1000000 0.0002\u001b[0m\r\n",
"-------------------------------------------------------------------------------------------------------------------------------\r\n",
"\r\n",
"File: \u001b[36mbenchmark_time.jsonl\u001b[0m\r\n",
"\u001b[36mTotal\u001b[0m\r\n",
" date time branch commit commit_date comment best_time mean_time mean_time_diff\r\n",
"\u001b[32m0 2019-05-21 18:50:44 master 3a1e755 2019-05-20 initial benchmark search 0.1052 0.1166 NaN\u001b[0m\r\n",
"\u001b[31m1 2019-05-21 18:51:02 master 3a1e755 2019-05-20 million items 9.6132 10.1825 10.0659\u001b[0m\r\n",
"--------------------------------------------------------------------------------------------------------------------\r\n"
]
}
],
"source": [
"!benchmark_analyze /tmp/benchmark_time.jsonl --extra_fields num_items"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Benchmark model"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.datasets import load_iris\n",
"from sklearn.linear_model import LogisticRegression"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"MODEL_BENCHMARK_SAVE_FILE = '/tmp/benchmark_model.jsonl'"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"x, y = load_iris(return_X_y=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Only parameters, passed to the decorated function `log_regression` will be saved (regularization parameter `C` and `fit_intercept`).\n",
"\n",
"In order to save results, decorated function `log_regression` should return `dict` with the results that need to be saved."
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"@benchmark(save_params=True, save_output=True)\n",
"def log_regression(C=1.0, fit_intercept=True):\n",
" clf = LogisticRegression(\n",
" random_state=0, \n",
" solver='lbfgs', \n",
" multi_class='multinomial', \n",
" C=C,\n",
" fit_intercept=fit_intercept,\n",
" )\n",
" clf.fit(x, y)\n",
" score = clf.score(x, y)\n",
" return {'score': score}"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"File: \u001b[36mbenchmark_model.jsonl\u001b[0m\n",
"Function: \u001b[36mlog_regression\u001b[0m\n",
" date time branch commit commit_date comment best_time mean_time C fit_intercept score score_diff\n",
"0 2019-05-21 18:51:35 master 3a1e755 2019-05-20 baseline model 26.2866 26.2866 1.0 True 0.973333 NaN\n",
"------------------------------------------------------------------------------------------------------------------------------------\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/mnt/ubuntu_storage/home/vitaliy/scoutbee/benchmarkit/env/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
" \"of iterations.\", ConvergenceWarning)\n"
]
}
],
"source": [
"model_benchmark_results = benchmark_run(\n",
" log_regression,\n",
" MODEL_BENCHMARK_SAVE_FILE,\n",
" comment='baseline model',\n",
" extra_fields=['C', 'fit_intercept'],\n",
" metric='score',\n",
" bigger_is_better=True,\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Change hyperparameter `C`"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"File: \u001b[36mbenchmark_model.jsonl\u001b[0m\n",
"Function: \u001b[36mlog_regression\u001b[0m\n",
" date time branch commit commit_date comment best_time mean_time C fit_intercept score score_diff\n",
"\u001b[32m0 2019-05-21 18:51:35 master 3a1e755 2019-05-20 baseline model 26.2866 26.2866 1.0 True 0.973333 NaN\u001b[0m\n",
"\u001b[31m1 2019-05-21 18:51:37 master 3a1e755 2019-05-20 stronger regularization 22.2981 22.2981 0.5 True 0.966667 -0.006667\u001b[0m\n",
"---------------------------------------------------------------------------------------------------------------------------------------------\n"
]
}
],
"source": [
"@benchmark(save_params=True, save_output=True)\n",
"def log_regression(C=0.5, fit_intercept=True):\n",
" clf = LogisticRegression(\n",
" random_state=0, \n",
" solver='lbfgs', \n",
" multi_class='multinomial', \n",
" C=C,\n",
" fit_intercept=fit_intercept,\n",
" )\n",
" clf.fit(x, y)\n",
" score = clf.score(x, y)\n",
" return {'score': score}\n",
"\n",
"model_benchmark_results = benchmark_run(\n",
" log_regression,\n",
" MODEL_BENCHMARK_SAVE_FILE,\n",
" comment='stronger regularization',\n",
" extra_fields=['C', 'fit_intercept'],\n",
" metric='score',\n",
" bigger_is_better=True,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[{'name': 'log_regression',\n",
" 'best_time': 22.2981,\n",
" 'mean_time': 22.2981,\n",
" 'date': '2019-05-21',\n",
" 'time': '18:51:37',\n",
" 'branch': 'master',\n",
" 'commit': '3a1e755',\n",
" 'commit_date': '2019-05-20',\n",
" 'C': 0.5,\n",
" 'fit_intercept': True,\n",
" 'score': 0.9666666666666667,\n",
" 'comment': 'stronger regularization',\n",
" '_id': 'f3c3f2a0-fd11-4790-a58a-34aac1f44f5a'}]"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model_benchmark_results"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"File: \u001b[36mbenchmark_model.jsonl\u001b[0m\n",
"Function: \u001b[36mlog_regression\u001b[0m\n",
" date time branch commit commit_date comment best_time mean_time C fit_intercept score score_diff\n",
"\u001b[32m0 2019-05-21 18:51:35 master 3a1e755 2019-05-20 baseline model 26.2866 26.2866 1.0 True 0.973333 NaN\u001b[0m\n",
"\u001b[31m1 2019-05-21 18:51:37 master 3a1e755 2019-05-20 stronger regularization 22.2981 22.2981 0.5 True 0.966667 -0.006667\u001b[0m\n",
"---------------------------------------------------------------------------------------------------------------------------------------------\n"
]
}
],
"source": [
"model_benchmark_df = benchmark_analyze(MODEL_BENCHMARK_SAVE_FILE, metric='score', bigger_is_better=True, extra_fields=['C', 'fit_intercept'])"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>date</th>\n",
" <th>time</th>\n",
" <th>name</th>\n",
" <th>branch</th>\n",
" <th>commit</th>\n",
" <th>commit_date</th>\n",
" <th>comment</th>\n",
" <th>best_time</th>\n",
" <th>mean_time</th>\n",
" <th>C</th>\n",
" <th>fit_intercept</th>\n",
" <th>score</th>\n",
" <th>score_diff</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>2019-05-21</td>\n",
" <td>18:51:35</td>\n",
" <td>log_regression</td>\n",
" <td>master</td>\n",
" <td>3a1e755</td>\n",
" <td>2019-05-20</td>\n",
" <td>baseline model</td>\n",
" <td>26.2866</td>\n",
" <td>26.2866</td>\n",
" <td>1.0</td>\n",
" <td>True</td>\n",
" <td>0.973333</td>\n",
" <td>NaN</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2019-05-21</td>\n",
" <td>18:51:37</td>\n",
" <td>log_regression</td>\n",
" <td>master</td>\n",
" <td>3a1e755</td>\n",
" <td>2019-05-20</td>\n",
" <td>stronger regularization</td>\n",
" <td>22.2981</td>\n",
" <td>22.2981</td>\n",
" <td>0.5</td>\n",
" <td>True</td>\n",
" <td>0.966667</td>\n",
" <td>-0.006667</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" date time name branch commit commit_date \\\n",
"0 2019-05-21 18:51:35 log_regression master 3a1e755 2019-05-20 \n",
"1 2019-05-21 18:51:37 log_regression master 3a1e755 2019-05-20 \n",
"\n",
" comment best_time mean_time C fit_intercept \\\n",
"0 baseline model 26.2866 26.2866 1.0 True \n",
"1 stronger regularization 22.2981 22.2981 0.5 True \n",
"\n",
" score score_diff \n",
"0 0.973333 NaN \n",
"1 0.966667 -0.006667 "
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model_benchmark_df"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"run `benchmark_analyze` from command line (without `!` in the real terminal):"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\r\n",
"File: \u001b[36mbenchmark_model.jsonl\u001b[0m\r\n",
"Function: \u001b[36mlog_regression\u001b[0m\r\n",
" date time branch commit commit_date comment best_time mean_time C fit_intercept score score_diff\r\n",
"\u001b[32m0 2019-05-21 18:51:35 master 3a1e755 2019-05-20 baseline model 26.2866 26.2866 1.0 True 0.973333 NaN\u001b[0m\r\n",
"\u001b[31m1 2019-05-21 18:51:37 master 3a1e755 2019-05-20 stronger regularization 22.2981 22.2981 0.5 True 0.966667 -0.006667\u001b[0m\r\n",
"---------------------------------------------------------------------------------------------------------------------------------------------\r\n"
]
}
],
"source": [
"!benchmark_analyze /tmp/benchmark_model.jsonl --metric score --bigger_is_better --extra_fields C fit_intercept"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Source: https://github.com/vgrabovets/benchmarkit"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}